Advertisement
Guest User

Untitled

a guest
Jul 27th, 2017
52
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.69 KB | None | 0 0
  1. with beam.Pipeline(argv=pipeline_args) as p:
  2.  
  3. from apache_beam.io.gcp.internal.clients import bigquery # pylint: disable=wrong-import-order, wrong-import-position
  4.  
  5. table_schema = bigquery.TableSchema()
  6.  
  7. # Fields that use standard types.
  8. alpha_schema = bigquery.TableFieldSchema()
  9. alpha_schema.name = 'alpha'
  10. alpha_schema.type = 'string'
  11. alpha_schema.mode = 'nullable'
  12. table_schema.fields.append(alpha_schema)
  13.  
  14. # A nested field
  15.  
  16. # beta_schema
  17. # |-- delta
  18. # |-- gama
  19.  
  20. beta_schema = bigquery.TableFieldSchema()
  21. beta_schema.name = 'beta'
  22. beta_schema.type = 'record'
  23. beta_schema.mode = 'nullable'
  24.  
  25. delta = bigquery.TableFieldSchema()
  26. delta.name = 'delta'
  27. delta.type = 'integer'
  28. delta.mode = 'nullable'
  29. beta_schema.fields.append(delta) # Append data to beta
  30.  
  31. gamma = bigquery.TableFieldSchema()
  32. gamma.name = 'gamma'
  33. gamma.type = 'integer'
  34. gamma.mode = 'nullable'
  35.  
  36. beta_schema.fields.append(gamma) # Append data to beta
  37. table_schema.fields.append(beta_schema) # Append the nested fields to the table_schema
  38.  
  39. # A repeated field.
  40. children_schema = bigquery.TableFieldSchema()
  41. children_schema.name = 'children'
  42. children_schema.type = 'string'
  43. children_schema.mode = 'repeated'
  44. table_schema.fields.append(children_schema)
  45.  
  46. # pylint: disable=expression-not-assigned
  47.  
  48. output_data | 'WriteToBigQuery' >> beam.io.Write(
  49. beam.io.BigQuerySink(
  50. known_args.output,
  51. schema=table_schema, # Pass the defined table_schema
  52. create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
  53. write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement