Guest User

Untitled

a guest
Oct 20th, 2018
90
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.78 KB | None | 0 0
  1. package main
  2.  
  3. import (
  4. "bufio"
  5. "fmt"
  6. "io"
  7. "log"
  8. "os"
  9. "strings"
  10. "unsafe"
  11.  
  12. "github.com/satyrius/gonx"
  13. )
  14.  
  15. func scanner(initialBuf, maxBuf int, noallocate bool) {
  16. f, err := os.Open("./testdata/NASA_access_log_Aug95")
  17. if err != nil {
  18. log.Println(err.Error())
  19. return
  20. }
  21. defer f.Close()
  22. // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
  23.  
  24. scanner := bufio.NewScanner(f)
  25. buf := make([]byte, initialBuf)
  26. scanner.Buffer(buf, maxBuf)
  27. // dis := ioutil.Discard
  28.  
  29. for scanner.Scan() {
  30. if !noallocate {
  31. s := scanner.Text()
  32. _ = strings.Split(s, " ")
  33. // for _, v := range sp {
  34. // dis.Write([]byte(v))
  35. // }
  36. // dis.Write([]byte(s))
  37. } else {
  38. b := scanner.Bytes()
  39. s := *(*string)(unsafe.Pointer(&b))
  40. _ = strings.Split(s, " ")
  41. // for _, v := range sp {
  42. // dis.Write([]byte(v))
  43. // }
  44. // dis.Write(b)
  45. }
  46. }
  47.  
  48. if err := scanner.Err(); err != nil {
  49. fmt.Printf("Scanner error: %q\n", err)
  50. }
  51. }
  52.  
  53. func scannerWithIndex(initialBuf, maxBuf int, noallocate bool) {
  54. f, err := os.Open("./testdata/NASA_access_log_Aug95")
  55. if err != nil {
  56. log.Println(err.Error())
  57. return
  58. }
  59. defer f.Close()
  60. // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
  61.  
  62. scanner := bufio.NewScanner(f)
  63. buf := make([]byte, initialBuf)
  64. scanner.Buffer(buf, maxBuf)
  65. // dis := ioutil.Discard
  66.  
  67. for scanner.Scan() {
  68. if !noallocate {
  69. s := scanner.Text()
  70. _, err := splitWithIndex(s)
  71. if err != nil {
  72. fmt.Println(err.Error())
  73. return
  74. }
  75. // for _, v := range sp {
  76. // dis.Write([]byte(v))
  77. // }
  78. } else {
  79. b := scanner.Bytes()
  80. s := *(*string)(unsafe.Pointer(&b))
  81. _, err := splitWithIndex(s)
  82. if err != nil {
  83. fmt.Println(err.Error())
  84. return
  85. }
  86. // for _, v := range sp {
  87. // dis.Write([]byte(v))
  88. // }
  89. }
  90. }
  91.  
  92. if err := scanner.Err(); err != nil {
  93. fmt.Printf("Scanner error: %q\n", err)
  94. }
  95. }
  96.  
  97. func splitWithIndex(s string) ([]string, error) {
  98. sp := make([]string, 0, 32)
  99. if len(s) == 0 {
  100. return sp, nil
  101. }
  102. end := false
  103. for {
  104. i := strings.IndexByte(s, ' ')
  105. if i < 0 {
  106. end = true
  107. } else {
  108. ss := s[:i]
  109. sp = append(sp, ss)
  110. }
  111. s = s[i+1:]
  112. if end {
  113. break
  114. }
  115. }
  116. return sp, nil
  117. }
  118. func scannerWithGonx() {
  119. f, err := os.Open("./testdata/NASA_access_log_Aug95")
  120. if err != nil {
  121. log.Println(err.Error())
  122. return
  123. }
  124. defer f.Close()
  125. // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
  126. format := "$remote_addr - - [$time_local] \"$request\" $in $in2"
  127. // parser := gonx.NewParser(format)
  128. reader := gonx.NewReader(f, format)
  129.  
  130. // dis := ioutil.Discard
  131. var count int
  132.  
  133. for {
  134. _, err := reader.Read()
  135. if err == io.EOF {
  136. break
  137. } else if err != nil {
  138. panic(err)
  139. }
  140. // fmt.Printf("%+v\n", rec)
  141. count++
  142. }
  143.  
  144. fmt.Printf("%v lines readed\n", count)
  145. }
Add Comment
Please, Sign In to add comment