Guest User

Untitled

a guest
Jul 20th, 2018
83
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.58 KB | None | 0 0
  1. // Sample logging-quickstart writes a log entry to Stackdriver Logging.
  2. package main
  3.  
  4. import (
  5. "log"
  6. "cloud.google.com/go/logging"
  7. "golang.org/x/net/context"
  8. "cloud.google.com/go/logging/logadmin"
  9. "google.golang.org/api/iterator"
  10. "fmt"
  11. "time"
  12. "cloud.google.com/go/bigquery"
  13. )
  14.  
  15. type StackDriverQuery struct {
  16. projectID string
  17. appName string
  18. filter string
  19. }
  20.  
  21. func (q StackDriverQuery) getQuery() string {
  22. //return fmt.Sprintf(`logName:%s %s`, q.appName, q.filter)
  23. return fmt.Sprintf(`logName = "projects/%s/logs/%s" %s`, q.projectID, q.appName, q.filter)
  24. }
  25.  
  26. func main() {
  27.  
  28. tryCount := 20
  29.  
  30. query := StackDriverQuery{
  31. projectID: "tsdn091",
  32. appName: "event-exporter",
  33. filter: `textPayload: watch`,
  34. }
  35.  
  36. fmt.Println("=== Call stackdriver API directly ===")
  37. for i:=1; i< tryCount+ 1; i++ {
  38. testProcessingTime(query, i*10)
  39. }
  40. fmt.Println("")
  41.  
  42. query = StackDriverQuery{
  43. projectID: "tsdn091",
  44. appName: "event-exporter",
  45. filter: `resource.labels.project_id = "tsdn091"`,
  46. }
  47.  
  48. fmt.Println("=== Call BigQuery API ===")
  49. for i:=1; i< tryCount+1; i++ {
  50. testProcessingTimeWithBigQuery(query, i*10)
  51. }
  52. fmt.Println("")
  53.  
  54.  
  55. }
  56.  
  57. func testProcessingTime(query StackDriverQuery, count int) {
  58.  
  59. startTime := time.Now()
  60. entries, err := getLogEntries(query.projectID, query.getQuery(), count)
  61. if err != nil {
  62. log.Fatalf("Failed to get log entries: %v", err)
  63. }
  64. endTime := time.Now()
  65. fmt.Printf("count: %d(%d), time: %s \n", count, len(entries), endTime.Sub(startTime))
  66. }
  67.  
  68. func testProcessingTimeWithBigQuery(query StackDriverQuery, count int) {
  69.  
  70. startTime := time.Now()
  71. entries, err := getLogEntriesWithBigQuery(
  72. query.projectID,
  73. query.appName,
  74. "20180720",
  75. query.filter,
  76. count)
  77. if err != nil {
  78. log.Fatalf("Failed to get log entries: %v", err)
  79. }
  80. endTime := time.Now()
  81. fmt.Printf("count: %d(%d), time: %s \n", count, len(entries), endTime.Sub(startTime))
  82. }
  83.  
  84. func getLogEntries(projectID, filter string, maxRecord int) ([]*logging.Entry, error) {
  85. ctx := context.Background()
  86.  
  87. // Sets your Google Cloud Platform project ID.
  88.  
  89. // Creates a client.
  90. client, err := logadmin.NewClient(ctx, projectID)
  91. if err != nil {
  92. log.Fatalf("Failed to create client: %v", err)
  93. }
  94.  
  95. // Get Iterable.
  96. iter := client.Entries(ctx,
  97. // Only get entries from the log-example log.
  98. logadmin.Filter(filter),
  99. //logadmin.Filter(""),
  100. // Get most recent entries first.
  101. logadmin.NewestFirst(),
  102. )
  103.  
  104. // Fetch entries.
  105. var entries []*logging.Entry
  106. for len(entries) < maxRecord {
  107. entry, err := iter.Next()
  108. if err == iterator.Done {
  109. return entries, nil
  110. }
  111.  
  112. //log.Print(entry)
  113. if err != nil {
  114. return nil, err
  115. }
  116. entries = append(entries, entry)
  117. }
  118. return entries, nil
  119.  
  120. }
  121.  
  122. func getLogEntriesWithBigQuery(projectID, appName, date, filter string, maxRecord int) ([]*bigquery.Value, error) {
  123. ctx := context.Background()
  124.  
  125. // Sets your Google Cloud Platform project ID.
  126.  
  127. // Creates a client.
  128. client, err := bigquery.NewClient(ctx, projectID)
  129. if err != nil {
  130. log.Fatalf("Failed to create client: %v", err)
  131. }
  132.  
  133. // Get Iterable.
  134. queryFmt := `
  135. SELECT textPayload
  136. FROM %s.event_exporter.%s_%s
  137. WHERE %s
  138. LIMIT %d`
  139. queryStr := fmt.Sprintf(queryFmt, projectID, appName, date, filter, maxRecord)
  140. q := client.Query(queryStr)
  141. it, err := q.Read(ctx)
  142. if err != nil {
  143. log.Fatalf("Failed to read records: %v", err)
  144. }
  145.  
  146. // Fetch entries.
  147. var values []*bigquery.Value
  148. var value []bigquery.Value
  149. for {
  150. err := it.Next(&value)
  151. if err == iterator.Done {
  152. return values, nil
  153. }
  154. if err != nil {
  155. log.Fatalf("Failed to get next iteration: %v", err)
  156. }
  157. values = append(values, &value[0])
  158. }
  159. return values, nil
  160. }
Add Comment
Please, Sign In to add comment