Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- // Sample logging-quickstart writes a log entry to Stackdriver Logging.
- package main
- import (
- "log"
- "cloud.google.com/go/logging"
- "golang.org/x/net/context"
- "cloud.google.com/go/logging/logadmin"
- "google.golang.org/api/iterator"
- "fmt"
- "time"
- "cloud.google.com/go/bigquery"
- )
- type StackDriverQuery struct {
- projectID string
- appName string
- filter string
- }
- func (q StackDriverQuery) getQuery() string {
- //return fmt.Sprintf(`logName:%s %s`, q.appName, q.filter)
- return fmt.Sprintf(`logName = "projects/%s/logs/%s" %s`, q.projectID, q.appName, q.filter)
- }
- func main() {
- tryCount := 20
- query := StackDriverQuery{
- projectID: "tsdn091",
- appName: "event-exporter",
- filter: `textPayload: watch`,
- }
- fmt.Println("=== Call stackdriver API directly ===")
- for i:=1; i< tryCount+ 1; i++ {
- testProcessingTime(query, i*10)
- }
- fmt.Println("")
- query = StackDriverQuery{
- projectID: "tsdn091",
- appName: "event-exporter",
- filter: `resource.labels.project_id = "tsdn091"`,
- }
- fmt.Println("=== Call BigQuery API ===")
- for i:=1; i< tryCount+1; i++ {
- testProcessingTimeWithBigQuery(query, i*10)
- }
- fmt.Println("")
- }
- func testProcessingTime(query StackDriverQuery, count int) {
- startTime := time.Now()
- entries, err := getLogEntries(query.projectID, query.getQuery(), count)
- if err != nil {
- log.Fatalf("Failed to get log entries: %v", err)
- }
- endTime := time.Now()
- fmt.Printf("count: %d(%d), time: %s \n", count, len(entries), endTime.Sub(startTime))
- }
- func testProcessingTimeWithBigQuery(query StackDriverQuery, count int) {
- startTime := time.Now()
- entries, err := getLogEntriesWithBigQuery(
- query.projectID,
- query.appName,
- "20180720",
- query.filter,
- count)
- if err != nil {
- log.Fatalf("Failed to get log entries: %v", err)
- }
- endTime := time.Now()
- fmt.Printf("count: %d(%d), time: %s \n", count, len(entries), endTime.Sub(startTime))
- }
- func getLogEntries(projectID, filter string, maxRecord int) ([]*logging.Entry, error) {
- ctx := context.Background()
- // Sets your Google Cloud Platform project ID.
- // Creates a client.
- client, err := logadmin.NewClient(ctx, projectID)
- if err != nil {
- log.Fatalf("Failed to create client: %v", err)
- }
- // Get Iterable.
- iter := client.Entries(ctx,
- // Only get entries from the log-example log.
- logadmin.Filter(filter),
- //logadmin.Filter(""),
- // Get most recent entries first.
- logadmin.NewestFirst(),
- )
- // Fetch entries.
- var entries []*logging.Entry
- for len(entries) < maxRecord {
- entry, err := iter.Next()
- if err == iterator.Done {
- return entries, nil
- }
- //log.Print(entry)
- if err != nil {
- return nil, err
- }
- entries = append(entries, entry)
- }
- return entries, nil
- }
- func getLogEntriesWithBigQuery(projectID, appName, date, filter string, maxRecord int) ([]*bigquery.Value, error) {
- ctx := context.Background()
- // Sets your Google Cloud Platform project ID.
- // Creates a client.
- client, err := bigquery.NewClient(ctx, projectID)
- if err != nil {
- log.Fatalf("Failed to create client: %v", err)
- }
- // Get Iterable.
- queryFmt := `
- SELECT textPayload
- FROM %s.event_exporter.%s_%s
- WHERE %s
- LIMIT %d`
- queryStr := fmt.Sprintf(queryFmt, projectID, appName, date, filter, maxRecord)
- q := client.Query(queryStr)
- it, err := q.Read(ctx)
- if err != nil {
- log.Fatalf("Failed to read records: %v", err)
- }
- // Fetch entries.
- var values []*bigquery.Value
- var value []bigquery.Value
- for {
- err := it.Next(&value)
- if err == iterator.Done {
- return values, nil
- }
- if err != nil {
- log.Fatalf("Failed to get next iteration: %v", err)
- }
- values = append(values, &value[0])
- }
- return values, nil
- }
Add Comment
Please, Sign In to add comment