rege

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"github.com/dgraph-io/badger"
	//"io/ioutil"
	//"os"
)

//struct for Story
type Story struct {
	Title       string `json:"title"`
	Url         string `json:"url"`
	Text        string `json:"text"`
	Dead       bool  `json:"dead"`
	By          string `json:"by"`
	Score       string `json:"score"`
	Time        string `json:"time"`
	Timestamp   string `json:"timestamp"`
	Type        string `json:"type"`
	Id          string `json:"id"`
	Parent      string `json:"parent"`
	Descendants string `json:"descendants"`
	Ranking     string `json:"ranking"`
	Deleted     bool `json:"deleted"`
}

func (h Story) encodeStory() []byte {
	data, err := json.Marshal(h)
	if err != nil {
		panic(err)
	}

	return data
}

func decodeStory(data []byte) (Story, error) {
	var h Story
	err := json.Unmarshal(data, &h)
	return h, err
}

type Ids struct {
	Ids []int `json:"ids"`
}

func (h Ids) encodeIds() []byte {
	data, err := json.Marshal(h)
	if err != nil {
		panic(err)
	}

	return data
}

func decodeIds(data []byte) (Ids, error) {
	var h Ids
	err := json.Unmarshal(data, &h)
	return h, err
}

func main() {

	// Open the Badger database located in the /tmp/badger directory.
	// Will be created if it doesn't exist.

	db, err := badger.Open(badger.DefaultOptions("./data/badger"))
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	dbTest, err := badger.Open(badger.DefaultOptions("./data/regex2"))
	if err != nil {
		log.Fatal(err)
	}
	defer dbTest.Close()

	err = dbTest.DropAll()
	if err != nil {
		fmt.Println("Error while dropping the database")
	}

	var batch int = 100000

	var keyCount int = 0

	idMap := make(map[string]Ids)

	reg, err := regexp.Compile("[^a-zA-Z0-9]+")
	if err != nil {
		log.Println(err)
	}

	txn := db.NewTransaction(true)
	defer txn.Discard()

	err = db.View(func(txn *badger.Txn) error {

		opts := badger.DefaultIteratorOptions
		opts.PrefetchSize = 10
		opts.Prefix=[]byte("story:")
		it := txn.NewIterator(opts)
		defer it.Close()
		for it.Rewind(); it.Valid(); it.Next() {

			item := it.Item()

			err := item.Value(func(v []byte) error {
				var s Story
				s, err1 := decodeStory(v)

				//check for errors
				if err1 != nil {
					log.Fatal(err)
				}

				//fmt.Println(s.Title)

				word := strings.Fields(s.Title)

				//var ids []string

				for _, v := range word {

					v := reg.ReplaceAllString(v, "")

					err = dbTest.View(func(txn *badger.Txn) error {
						item, err := txn.Get([]byte("_index::title::" + strings.ToLower(v)))
						if err != nil {
								return err
						}

						//if val found retrieve val

						val, err := item.ValueCopy(nil)
						if err != nil {
							return err
						}

						ids, err1 := decodeIds(val)

						if err1 != nil {
							log.Fatal("Error decoding")
						}

						idInt, err1 := strconv.Atoi(s.Id)

						if err1 != nil {
							fmt.Println("error")
						}

						ids.Ids = append(ids.Ids, idInt)

						//make a single transaction

						// Start a writable transaction.
						txn1 := dbTest.NewTransaction(true)
						defer txn1.Discard()

						// Use the transaction...
						err = txn1.Set([]byte("_index::title::"+strings.ToLower(v)), ids.encodeIds())
						if err != nil {
							return err
						}

						// Commit the transaction and check for error.
						if err := txn1.Commit(); err != nil {
							if err != nil {
								fmt.Println(err)
							}
						}

						return nil
					})

					//if val not found set a new key
					if err != nil {

						newIDInt, err2 := strconv.Atoi(s.Id)

						if err2 != nil {
							fmt.Println(err2)
						}

						var id []int

						id = append(id, newIDInt)

						idMap["_index::title::"+strings.ToLower(v)] = Ids{
							Ids: id,
						}

						keyCount++

						if keyCount > batch {
							keyCount = 0
							//write batch
							wb := dbTest.NewWriteBatch()
							defer wb.Cancel()

							for k, v := range idMap {

								err := wb.Set([]byte(k), v.encodeIds())

								if err != nil {
									log.Fatal(err)
								}
							}
							fmt.Println("batch commit")

							err1 := wb.Flush()

							if err1 != nil {
								log.Fatal(err1)
							}

							//reinitialize idMap
							idMap = make(map[string]Ids)

						}

					}

				}

				return nil
			})

			if err != nil {
				return err
			}

		}
		//write batch for remaining words
		wb := dbTest.NewWriteBatch()
		defer wb.Cancel()

		for k, v := range idMap {

			err := wb.Set([]byte(k), v.encodeIds())

			if err != nil {
				log.Fatal(err)
			}
		}
		fmt.Println("batch commit remaining")
		err1 := wb.Flush()

		if err1 != nil {
			log.Fatal(err1)
		}

		return nil
	})

	//error handle for View
	if err != nil {
		log.Fatal(err)
	}

	wb := dbTest.NewWriteBatch()
	defer wb.Cancel()

	for k, v := range idMap {

		err := wb.Set([]byte(k), v.encodeIds())

		if err != nil {
			log.Fatal(err)
		}
	}

	/*
	   err = dbTest.View(func(txn *badger.Txn) error {
	       item, err := txn.Get([]byte("_index::title::nuclear"))
	       if err != nil {
	           return err
	       }
	       val, err := item.ValueCopy(nil)
	       if err != nil {
	           return err
	       }
	       fmt.Printf("%s\n", string(val))
	       return nil
	   })

	   if err != nil {
	       fmt.Println("NotFound")
	   }
	*/

}