Advertisement
Guest User

Untitled

a guest
Feb 20th, 2019
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.61 KB | None | 0 0
  1. import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
  2. import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  3. import com.google.api.client.http.HttpTransport;
  4. import com.google.api.client.json.JsonFactory;
  5. import com.google.api.client.json.JsonFactory;
  6. import com.google.api.client.json.jackson2.JacksonFactory;
  7. import com.google.api.services.bigquery.Bigquery;
  8. import com.google.api.services.bigquery.BigqueryScopes;
  9. import com.google.api.client.util.Data;
  10. import com.google.api.services.bigquery.model.*;
  11.  
  12. /* your class starts here */
  13.  
  14. private String projectId = ""; /* fill in the project id here */
  15. private String query = ""; /* enter your query here */
  16. private Bigquery bigQuery;
  17. private Job insert;
  18. private TableDataList tableDataList;
  19. private Iterator<TableRow> rowsIterator;
  20. private List<TableRow> rows;
  21. private long maxResults = 100000L; /* max number of rows in a page */
  22.  
  23. /* run query */
  24. public void open() throws Exception {
  25. HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
  26. JsonFactory jsonFactory = new JacksonFactory();
  27. GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
  28. if (credential.createScopedRequired())
  29. credential = credential.createScoped(BigqueryScopes.all());
  30. bigQuery = new Bigquery.Builder(transport, jsonFactory, credential).setApplicationName("my app").build();
  31.  
  32. JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(query);
  33. JobConfiguration jobConfig = new JobConfiguration().setQuery(queryConfig);
  34. Job job = new Job().setConfiguration(jobConfig);
  35. insert = bigQuery.jobs().insert(projectId, job).execute();
  36. JobReference jobReference = insert.getJobReference();
  37.  
  38. while (true) {
  39. Job poll = bigQuery.jobs().get(projectId, jobReference.getJobId()).execute();
  40. String state = poll.getStatus().getState();
  41. if ("DONE".equals(state)) {
  42. ErrorProto errorResult = poll.getStatus().getErrorResult();
  43. if (errorResult != null)
  44. throw new Exception("Error running job: " + poll.getStatus().getErrors().get(0));
  45. break;
  46. }
  47. Thread.sleep(10000);
  48. }
  49.  
  50. tableDataList = getPage();
  51. rows = tableDataList.getRows();
  52. rowsIterator = rows != null ? rows.iterator() : null;
  53. }
  54.  
  55. /* read data row by row */
  56. public /* your data object here */ read() throws Exception {
  57. if (rowsIterator == null) return null;
  58.  
  59. if (!rowsIterator.hasNext()) {
  60. String pageToken = tableDataList.getPageToken();
  61. if (pageToken == null) return null;
  62. tableDataList = getPage(pageToken);
  63. rows = tableDataList.getRows();
  64. if (rows == null) return null;
  65. rowsIterator = rows.iterator();
  66. }
  67.  
  68. TableRow row = rowsIterator.next();
  69. for (TableCell cell : row.getF()) {
  70. Object value = cell.getV();
  71. /* extract the data here */
  72. }
  73.  
  74. /* return the data */
  75. }
  76.  
  77. private TableDataList getPage() throws IOException {
  78. return getPage(null);
  79. }
  80.  
  81. private TableDataList getPage(String pageToken) throws IOException {
  82. TableReference sourceTable = insert
  83. .getConfiguration()
  84. .getQuery()
  85. .getDestinationTable();
  86. if (sourceTable == null)
  87. throw new IllegalArgumentException("Source table not available. Please check the query syntax.");
  88. return bigQuery.tabledata()
  89. .list(projectId, sourceTable.getDatasetId(), sourceTable.getTableId())
  90. .setPageToken(pageToken)
  91. .setMaxResults(maxResults)
  92. .execute();
  93. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement