Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package main
- import (
- "bufio"
- "fmt"
- "io"
- "log"
- "os"
- "strings"
- "unsafe"
- "github.com/satyrius/gonx"
- )
- func scanner(initialBuf, maxBuf int, noallocate bool) {
- f, err := os.Open("./testdata/NASA_access_log_Aug95")
- if err != nil {
- log.Println(err.Error())
- return
- }
- defer f.Close()
- // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
- scanner := bufio.NewScanner(f)
- buf := make([]byte, initialBuf)
- scanner.Buffer(buf, maxBuf)
- // dis := ioutil.Discard
- for scanner.Scan() {
- if !noallocate {
- s := scanner.Text()
- _ = strings.Split(s, " ")
- // for _, v := range sp {
- // dis.Write([]byte(v))
- // }
- // dis.Write([]byte(s))
- } else {
- b := scanner.Bytes()
- s := *(*string)(unsafe.Pointer(&b))
- _ = strings.Split(s, " ")
- // for _, v := range sp {
- // dis.Write([]byte(v))
- // }
- // dis.Write(b)
- }
- }
- if err := scanner.Err(); err != nil {
- fmt.Printf("Scanner error: %q\n", err)
- }
- }
- func scannerWithIndex(initialBuf, maxBuf int, noallocate bool) {
- f, err := os.Open("./testdata/NASA_access_log_Aug95")
- if err != nil {
- log.Println(err.Error())
- return
- }
- defer f.Close()
- // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
- scanner := bufio.NewScanner(f)
- buf := make([]byte, initialBuf)
- scanner.Buffer(buf, maxBuf)
- // dis := ioutil.Discard
- for scanner.Scan() {
- if !noallocate {
- s := scanner.Text()
- _, err := splitWithIndex(s)
- if err != nil {
- fmt.Println(err.Error())
- return
- }
- // for _, v := range sp {
- // dis.Write([]byte(v))
- // }
- } else {
- b := scanner.Bytes()
- s := *(*string)(unsafe.Pointer(&b))
- _, err := splitWithIndex(s)
- if err != nil {
- fmt.Println(err.Error())
- return
- }
- // for _, v := range sp {
- // dis.Write([]byte(v))
- // }
- }
- }
- if err := scanner.Err(); err != nil {
- fmt.Printf("Scanner error: %q\n", err)
- }
- }
- func splitWithIndex(s string) ([]string, error) {
- sp := make([]string, 0, 32)
- if len(s) == 0 {
- return sp, nil
- }
- end := false
- for {
- i := strings.IndexByte(s, ' ')
- if i < 0 {
- end = true
- } else {
- ss := s[:i]
- sp = append(sp, ss)
- }
- s = s[i+1:]
- if end {
- break
- }
- }
- return sp, nil
- }
- func scannerWithGonx() {
- f, err := os.Open("./testdata/NASA_access_log_Aug95")
- if err != nil {
- log.Println(err.Error())
- return
- }
- defer f.Close()
- // in := strings.NewReader("1st line\n" + strings.Repeat("X", 65536) + "\n3rd line\n")
- format := "$remote_addr - - [$time_local] \"$request\" $in $in2"
- // parser := gonx.NewParser(format)
- reader := gonx.NewReader(f, format)
- // dis := ioutil.Discard
- var count int
- for {
- _, err := reader.Read()
- if err == io.EOF {
- break
- } else if err != nil {
- panic(err)
- }
- // fmt.Printf("%+v\n", rec)
- count++
- }
- fmt.Printf("%v lines readed\n", count)
- }
Add Comment
Please, Sign In to add comment