API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Nov 26th, 2015
68
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Go 16.30 KB | None | 0 0
raw download clone embed print report
  1. package main
  2. /**
  3.     Twitter Accounts, Links and Hashtags Searcher
  4.     v 0.2.2
  5.     by CosmicCat
  6.     cosmiccat@i2pmail.org
  7.     how to use: https://ghostbin.com/paste/sopbw
  8.  */
  9. import (
  10.     "net/http"
  11.     "os"
  12.     "fmt"
  13.     "net/url"
  14.     "io/ioutil"
  15.     "io"
  16.     "bytes"
  17.     "gopkg.in/xmlpath.v2"
  18.     "strings"
  19.     "golang.org/x/net/html"
  20.     "encoding/json"
  21.     "runtime/debug"
  22.     "runtime"
  23.     "bufio"
  24.     "math/rand"
  25. )
  26.  
  27. var (
  28.     domain = "https://twitter.com/"
  29.     configFile = "./config.json"
  30.     detailedDir = "detailed"
  31.     accountDir = "by_accounts"
  32.     hashtagDir = "by_hashtags"
  33.     resultsDir = "results"
  34.     sortedDir = "sorted"
  35.     unsortedDir = "unsorted"
  36.     hashtagsFile = "hashtags.txt"
  37.     externalLinksFile = "external_links.txt"
  38.     userAccountsFile = "accounts.txt"
  39.     config Config
  40.     hashtagsToSearch map[string]struct{}
  41.     accountsToSearch map[string]struct{}
  42.     findedAccountsToSearch map[string]struct{}
  43.     hashtagsByAccount map[string]map[string]struct{}
  44.     totalHashtags map[string]struct{}
  45.     externalLinksByAccount map[string]map[string]struct{}
  46.     totalExternalLinks map[string]struct{}
  47.     userAccountsByAccount map[string]map[string]struct{}
  48.     totalUserAccounts map[string]struct{}
  49.     hashtagsByHashtag map[string]map[string]struct{}
  50.     userAccountsByHashtag map[string]map[string]struct{}
  51.     externalLinksByHashtag map[string]map[string]struct{}
  52. )
  53.  
  54. type Config struct {
  55.     DetailedData    bool
  56.     ProxyHost       string
  57.     ProxyPort       string
  58.     HashtagsFile    string
  59.     AccountsFile    string
  60.     Shuffle         bool
  61.     RecursiveSearch bool
  62.     Threads         int
  63.     UseThreads      bool
  64. }
  65.  
  66. func main() {
  67.     prepares()
  68.     if config.UseThreads {
  69.         goSearchHashtags()
  70.         goSearchAccounts(accountsToSearch)
  71.         goSearchAccounts(findedAccountsToSearch)
  72.     } else {
  73.         for _, hashtag := range (getShuffledValues(hashtagsToSearch)) {
  74.             loadHashtag(hashtag)
  75.         }
  76.         for _, account := range (getShuffledValues(accountsToSearch)) {
  77.             loadAccount(account)
  78.         }
  79.         if config.RecursiveSearch {
  80.             for account, _ := range (findedAccountsToSearch) {
  81.                 loadAccount(account)
  82.             }
  83.         }
  84.     }
  85.     saveTotal()
  86.     fmt.Println("Finished!")
  87. }
  88.  
  89. func goSearchHashtags() {
  90.     sem := make(chan bool, config.Threads)
  91.     for hashtag, _ := range (hashtagsToSearch) {
  92.         sem <- true
  93.         go func(ht string) {
  94.             defer func() {
  95.                 <-sem
  96.                 runtime.GC()
  97.                 debug.FreeOSMemory()
  98.             }()
  99.             loadHashtag(ht)
  100.         }(hashtag)
  101.     }
  102.     for i := 0; i < cap(sem); i++ {
  103.         sem <- true
  104.     }
  105. }
  106.  
  107. func goSearchAccounts(accounts map[string]struct{}) {
  108.     fmt.Println("start search accounts")
  109.     sem := make(chan bool, config.Threads)
  110.     for account, _ := range (accounts) {
  111.         sem <- true
  112.         go func(acc string) {
  113.             defer func() {
  114.                 <-sem
  115.                 runtime.GC()
  116.                 debug.FreeOSMemory()
  117.             }()
  118.             loadAccount(acc)
  119.         }(account)
  120.     }
  121.     for i := 0; i < cap(sem); i++ {
  122.         sem <- true
  123.     }
  124. }
  125.  
  126. func loadHashtags() {
  127.     if config.HashtagsFile != "" {
  128.         if _, err := os.Stat(config.HashtagsFile); !os.IsNotExist(err) {
  129.             inFile, _ := os.Open(config.HashtagsFile)
  130.             defer inFile.Close()
  131.             scanner := bufio.NewScanner(inFile)
  132.             scanner.Split(bufio.ScanLines)
  133.             for scanner.Scan() {
  134.                 hashtagsToSearch[strings.Replace(scanner.Text(), "#", "", -1)] = struct{}{}
  135.             }
  136.         }
  137.     }
  138. }
  139.  
  140. func loadAccounts() {
  141.     if config.AccountsFile != "" {
  142.         if _, err := os.Stat(config.AccountsFile); !os.IsNotExist(err) {
  143.             inFile, _ := os.Open(config.AccountsFile)
  144.             defer inFile.Close()
  145.             scanner := bufio.NewScanner(inFile)
  146.             scanner.Split(bufio.ScanLines)
  147.             for scanner.Scan() {
  148.                 accountsToSearch[strings.Replace(strings.Replace(scanner.Text(), "@", "", -1), domain, "", -1)] = struct{}{}
  149.             }
  150.         }
  151.     }
  152. }
  153.  
  154. func saveTotal() {
  155.     fullSortedDir := resultsDir + "/" + sortedDir + "/"
  156.     for hashtag := range (totalHashtags) {
  157.         writeToFile(fullSortedDir + hashtagsFile, "#" + hashtag)
  158.     }
  159.     totalHashtags = make(map[string]struct{})
  160.     for extLink := range (totalExternalLinks) {
  161.         writeToFile(fullSortedDir + externalLinksFile, extLink)
  162.     }
  163.     totalExternalLinks = make(map[string]struct{})
  164.     for account := range (totalUserAccounts) {
  165.         writeToFile(fullSortedDir + userAccountsFile, "@" + account)
  166.     }
  167.     totalUserAccounts = make(map[string]struct{})
  168.     runtime.GC()
  169.     debug.FreeOSMemory()
  170. }
  171.  
  172. func prepares() {
  173.     checkConfig()
  174.     proxyUrl, _ := url.Parse("http://" + config.ProxyHost + ":" + config.ProxyPort)
  175.     http.DefaultTransport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
  176.     checkAndMakeDir(resultsDir)
  177.     if config.DetailedData {
  178.         checkAndMakeDir(resultsDir + "/" + detailedDir)
  179.         checkAndMakeDir(resultsDir + "/" + detailedDir + "/" + accountDir)
  180.         checkAndMakeDir(resultsDir + "/" + detailedDir + "/" + hashtagDir)
  181.     }
  182.     checkAndMakeDir(resultsDir + "/" + sortedDir)
  183.     checkAndMakeDir(resultsDir + "/" + unsortedDir)
  184.     hashtagsByAccount = make(map[string]map[string]struct{})
  185.     totalHashtags = make(map[string]struct{})
  186.     externalLinksByAccount = make(map[string]map[string]struct{})
  187.     totalExternalLinks = make(map[string]struct{})
  188.     userAccountsByAccount = make(map[string]map[string]struct{})
  189.     totalUserAccounts = make(map[string]struct{})
  190.     accountsToSearch = make(map[string]struct{})
  191.     hashtagsToSearch = make(map[string]struct{})
  192.     findedAccountsToSearch = make(map[string]struct{})
  193.     userAccountsByHashtag = make(map[string]map[string]struct{})
  194.     externalLinksByHashtag = make(map[string]map[string]struct{})
  195.     hashtagsByHashtag = make(map[string]map[string]struct{})
  196.     loadHashtags()
  197.     loadAccounts()
  198. }
  199.  
  200. func getShuffledValues(src map[string]struct{}) (dest []string) {
  201.     keys := make([]string, 0, len(src))
  202.     for k := range src {
  203.         keys = append(keys, k)
  204.     }
  205.     if config.Shuffle {
  206.         dest = make([]string, len(keys))
  207.         perm := rand.Perm(len(keys))
  208.         for i, v := range perm {
  209.             dest[v] = keys[i]
  210.         }
  211.     } else {
  212.         dest = keys
  213.     }
  214.     return
  215. }
  216.  
  217. func checkConfig() {
  218.     if _, err := os.Stat(configFile); os.IsNotExist(err) {
  219.         default_conf := &Config{
  220.             UseThreads: false,
  221.             Threads: 4,
  222.             RecursiveSearch: true,
  223.             Shuffle: true,
  224.             HashtagsFile: "./tags.txt",
  225.             AccountsFile: "./targets.txt",
  226.             ProxyHost: "127.0.0.1",
  227.             ProxyPort: "8118",
  228.             DetailedData: true}
  229.         jsonStr, _ := json.Marshal(default_conf)
  230.         out, _ := os.Create(configFile)
  231.         io.Copy(out, bytes.NewReader(jsonStr))
  232.         log("Default config saved. Check and change config file \"" + configFile + "\"!")
  233.         os.Exit(0)
  234.     }
  235.     file, _ := ioutil.ReadFile(configFile)
  236.     config = Config{}
  237.     json.Unmarshal(file, &config)
  238.     log("Config loaded")
  239. }
  240.  
  241. func loadHashtag(hashtag string) {
  242.     rootNode := getRootNode(domain + "search?q=" + url.QueryEscape("#" + hashtag))
  243.     if rootNode == nil {
  244.         log("Root node is absend for #" + hashtag)
  245.         return
  246.     }
  247.     hashtagsByHashtag[hashtag] = make(map[string]struct{})
  248.     externalLinksByHashtag[hashtag] = make(map[string]struct{})
  249.     userAccountsByHashtag[hashtag] = make(map[string]struct{})
  250.     handleTweets(rootNode, hashtag, false)
  251.     min_position_path := xmlpath.MustCompile("//div[@id='timeline']//div[contains(@class,'stream-container')]/@data-min-position")
  252.     min_position, ok := min_position_path.String(rootNode)
  253.     if !ok {
  254.         log("Skipping page of #" + hashtag)
  255.     } else {
  256.         loadTweetsByHashtag(min_position, hashtag)
  257.         fullDir := resultsDir + "/" + detailedDir + "/" + hashtagDir + "/" + hashtag
  258.         if config.DetailedData {
  259.             checkAndMakeDir(fullDir)
  260.             for hashtag := range (hashtagsByHashtag[hashtag]) {
  261.                 writeToFile(fullDir + "/" + hashtagsFile, "#" + hashtag)
  262.             }
  263.             for externalLink := range (externalLinksByHashtag[hashtag]) {
  264.                 writeToFile(fullDir + "/" + externalLinksFile, externalLink)
  265.             }
  266.         }
  267.         for userAccount := range (userAccountsByHashtag[hashtag]) {
  268.             if config.DetailedData {
  269.                 writeToFile(fullDir + "/" + userAccountsFile, "@" + userAccount)
  270.             }
  271.             if config.RecursiveSearch {
  272.                 if _, ok := accountsToSearch[userAccount]; !ok {
  273.                     findedAccountsToSearch[userAccount] = struct{}{};
  274.                 }
  275.             }
  276.         }
  277.     }
  278.     delete(hashtagsByHashtag, hashtag)
  279.     delete(externalLinksByHashtag, hashtag)
  280.     delete(userAccountsByHashtag, hashtag)
  281.     runtime.GC()
  282.     debug.FreeOSMemory()
  283. }
  284.  
  285. func loadAccount(account string) {
  286.     log("Start search account " + domain + account)
  287.     rootNode := getRootNode(domain + account)
  288.     if rootNode == nil {
  289.         log("Root node is absend for @" + account)
  290.         return
  291.     }
  292.     hashtagsByAccount[account] = make(map[string]struct{})
  293.     externalLinksByAccount[account] = make(map[string]struct{})
  294.     userAccountsByAccount[account] = make(map[string]struct{})
  295.     handleTweets(rootNode, account, true)
  296.     min_position_path := xmlpath.MustCompile("//div[@id='timeline']//div[contains(@class,'stream-container')]/@data-min-position")
  297.     min_position, ok := min_position_path.String(rootNode)
  298.     if !ok {
  299.         log("Skipping page of @" + account)
  300.     } else {
  301.         loadTweets(min_position, account)
  302.         fullDir := resultsDir + "/" + detailedDir + "/" + accountDir + "/" + account
  303.         if config.DetailedData {
  304.             checkAndMakeDir(fullDir)
  305.             for hashtag := range (hashtagsByAccount[account]) {
  306.                 writeToFile(fullDir + "/" + hashtagsFile, "#" + hashtag)
  307.             }
  308.             for externalLink := range (externalLinksByAccount[account]) {
  309.                 writeToFile(fullDir + "/" + externalLinksFile, externalLink)
  310.             }
  311.         }
  312.         for userAccount := range (userAccountsByAccount[account]) {
  313.             if config.DetailedData {
  314.                 writeToFile(fullDir + "/" + userAccountsFile, "@" + userAccount)
  315.             }
  316.             if config.RecursiveSearch {
  317.                 if _, ok := accountsToSearch[userAccount]; !ok {
  318.                     findedAccountsToSearch[userAccount] = struct{}{};
  319.                 }
  320.             }
  321.         }
  322.     }
  323.     delete(hashtagsByAccount, account)
  324.     delete(externalLinksByAccount, account)
  325.     delete(userAccountsByAccount, account)
  326.     runtime.GC()
  327.     debug.FreeOSMemory()
  328. }
  329.  
  330. func loadTweetsByHashtag(max_tweet string, hashtag string) {
  331.     urlToLoad := "https://twitter.com/i/search/timeline?vertical=default&q=%23" + hashtag + "&src=typd&include_available_features=1&include_entities=1&last_note_ts=7456&max_position=" + max_tweet
  332.     req, err := http.NewRequest("GET", urlToLoad, nil)
  333.     if err != nil {
  334.         log("Error1: " + err.Error())
  335.     } else {
  336.         response, err := http.DefaultTransport.RoundTrip(req)
  337.         if err != nil {
  338.             log("Error2: " + err.Error())
  339.         } else {
  340.             defer response.Body.Close()
  341.             content, err := ioutil.ReadAll(response.Body)
  342.             if err != nil {
  343.                 log("Error3: " + err.Error())
  344.             } else {
  345.                 var data interface{}
  346.                 if err := json.Unmarshal(content, &data); err != nil {
  347.                     log("Error4: " + err.Error())
  348.                 } else {
  349.                     m := data.(map[string]interface{})
  350.                     innerHtml := m["items_html"]
  351.                     has_more_i := m["has_more_items"]
  352.                     min_position_i := m["min_position"]
  353.                     str, _ := innerHtml.(string)
  354.                     has_more, _ := has_more_i.(bool)
  355.                     min_position, _ := min_position_i.(string)
  356.                     root, err := html.Parse(bytes.NewReader([]byte(str)))
  357.                     if err != nil {
  358.                         log("Error5: " + err.Error())
  359.                     }
  360.                     var b bytes.Buffer
  361.                     html.Render(&b, root)
  362.                     fixedHtml := b.String()
  363.                     reader := strings.NewReader(fixedHtml)
  364.                     rootNode, err := xmlpath.ParseHTML(reader)
  365.                     if err != nil {
  366.                         log("Error6: " + err.Error())
  367.                     } else {
  368.                         handleTweets(rootNode, hashtag, false)
  369.                         if has_more == true {
  370.                             loadTweetsByHashtag(min_position, hashtag)
  371.                         }
  372.                     }
  373.                 }
  374.             }
  375.         }
  376.     }
  377. }
  378.  
  379. func loadTweets(max_position string, account string) {
  380.     log("Loading tweets for @" + account + "... [" + max_position + "]")
  381.     urlToLoad := "https://twitter.com/i/profiles/show/" + account + "/timeline?include_available_features=1&include_entities=1&reset_error_state=false&max_position=" + max_position
  382.     req, err := http.NewRequest("GET", urlToLoad, nil)
  383.     if err != nil {
  384.         log("Error1: " + err.Error())
  385.     } else {
  386.         response, err := http.DefaultTransport.RoundTrip(req)
  387.         if err != nil {
  388.             log("Error2: " + err.Error())
  389.         } else {
  390.             defer response.Body.Close()
  391.             content, err := ioutil.ReadAll(response.Body)
  392.             if err != nil {
  393.                 log("Error3: " + err.Error())
  394.             } else {
  395.                 var data interface{}
  396.                 if err := json.Unmarshal(content, &data); err != nil {
  397.                     log("Error4: " + err.Error())
  398.                 } else {
  399.                     m := data.(map[string]interface{})
  400.                     innerHtml := m["items_html"]
  401.                     has_more_i := m["has_more_items"]
  402.                     min_position_i := m["min_position"]
  403.                     str, _ := innerHtml.(string)
  404.                     has_more, _ := has_more_i.(bool)
  405.                     min_position, _ := min_position_i.(string)
  406.                     root, err := html.Parse(bytes.NewReader([]byte(str)))
  407.                     if err != nil {
  408.                         log("Error5: " + err.Error())
  409.                     }
  410.                     var b bytes.Buffer
  411.                     html.Render(&b, root)
  412.                     fixedHtml := b.String()
  413.                     reader := strings.NewReader(fixedHtml)
  414.                     rootNode, err := xmlpath.ParseHTML(reader)
  415.                     if err != nil {
  416.                         log("Error6: " + err.Error())
  417.                     } else {
  418.                         handleTweets(rootNode, account, true)
  419.                         if has_more == true && min_position != "" {
  420.                             loadTweets(min_position, account)
  421.                         }
  422.                     }
  423.                 }
  424.             }
  425.         }
  426.     }
  427. }
  428.  
  429. func handleTweets(rootNode *xmlpath.Node, what string, is_account bool) {
  430.     handleHashtags(rootNode, what, is_account)
  431.     handleExternalLinks(rootNode, what, is_account)
  432.     handleAccounts(rootNode, what, is_account)
  433. }
  434.  
  435. func handleAccounts(rootNode *xmlpath.Node, what string, is_account bool) {
  436.     accountsNodes := xmlpath.MustCompile("//div[contains(@class,'stream-container')]//span[contains(@class,'username')]//b")
  437.     iter := accountsNodes.Iter(rootNode)
  438.     for iter.Next() {
  439.         accountLink := iter.Node().String()
  440.         if config.DetailedData {
  441.             if is_account {
  442.                 userAccountsByAccount[what][accountLink] = struct{}{}
  443.             } else {
  444.                 userAccountsByHashtag[what][accountLink] = struct{}{}
  445.             }
  446.         }
  447.         totalUserAccounts[accountLink] = struct{}{}
  448.         writeToFile(resultsDir + "/" + unsortedDir + "/" + userAccountsFile, "@" + accountLink)
  449.     }
  450. }
  451.  
  452. func handleExternalLinks(rootNode *xmlpath.Node, what string, is_account bool) {
  453.     linksNodes := xmlpath.MustCompile("//div[contains(@class,'stream-container')]//a[contains(@class,'twitter-timeline-link')]/@data-expanded-url")
  454.     iter := linksNodes.Iter(rootNode)
  455.     for iter.Next() {
  456.         externalLink := iter.Node().String()
  457.         if config.DetailedData {
  458.             if is_account {
  459.                 externalLinksByAccount[what][externalLink] = struct{}{}
  460.             } else {
  461.                 externalLinksByHashtag[what][externalLink] = struct{}{}
  462.             }
  463.         }
  464.         totalExternalLinks[externalLink] = struct{}{}
  465.         writeToFile(resultsDir + "/" + unsortedDir + "/" + externalLinksFile, externalLink)
  466.     }
  467. }
  468.  
  469. func handleHashtags(rootNode *xmlpath.Node, what string, is_account bool) {
  470.     hashtagsNodes := xmlpath.MustCompile("//a[contains(@class,'twitter-hashtag')]//b")
  471.     iter := hashtagsNodes.Iter(rootNode)
  472.     for iter.Next() {
  473.         hashtag := iter.Node().String()
  474.         if config.DetailedData {
  475.             if is_account {
  476.                 hashtagsByAccount[what][hashtag] = struct{}{}
  477.             } else {
  478.                 hashtagsByHashtag[what][hashtag] = struct{}{}
  479.             }
  480.         }
  481.         totalHashtags[hashtag] = struct{}{}
  482.         writeToFile(resultsDir + "/" + unsortedDir + "/" + hashtagsFile, "#" + hashtag)
  483.     }
  484. }
  485.  
  486. func checkAndMakeDir(dir string) {
  487.     if _, err := os.Stat(dir); os.IsNotExist(err) {
  488.         os.Mkdir(dir, 0600)
  489.     }
  490. }
  491.  
  492. func getRootNode(urlToOpen string) (*xmlpath.Node) {
  493.     req, err := http.NewRequest("GET", urlToOpen, nil)
  494.     if (err != nil) {
  495.         log("error1: " + err.Error())
  496.     } else {
  497.         response, err := http.DefaultTransport.RoundTrip(req)
  498.         if (err != nil) {
  499.             log("Error 3: " + err.Error())
  500.         } else {
  501.             defer response.Body.Close()
  502.             content, err := ioutil.ReadAll(response.Body)
  503.             if err != nil {
  504.                 log("Error 2: " + err.Error())
  505.             } else {
  506.                 root, err := html.Parse(bytes.NewReader(content))
  507.  
  508.                 if err != nil {
  509.                     log("Parse error: " + err.Error())
  510.                 }
  511.  
  512.                 var b bytes.Buffer
  513.                 html.Render(&b, root)
  514.                 fixedHtml := b.String()
  515.                 reader := strings.NewReader(fixedHtml)
  516.                 rootNode, err := xmlpath.ParseHTML(reader)
  517.                 if err != nil {
  518.                     log("Error 4: " + err.Error())
  519.                 } else {
  520.                     return rootNode
  521.                 }
  522.             }
  523.         }
  524.     }
  525.     return nil
  526. }
  527.  
  528. func log(text string) {
  529.     f, err := os.OpenFile("log.txt", os.O_APPEND | os.O_WRONLY | os.O_CREATE, 0600)
  530.     if err != nil {
  531.         panic(err)
  532.     }
  533.     defer f.Close()
  534.     fmt.Println(text)
  535.     text = text + "\r\n\r\n"
  536.     if _, err = f.WriteString(text); err != nil {
  537.         panic(err)
  538.     }
  539. }
  540.  
  541. func writeToFile(file string, text string) {
  542.     f, err := os.OpenFile(file, os.O_APPEND | os.O_WRONLY | os.O_CREATE, 0600)
  543.     if err != nil {
  544.         log("Error 1 on write file \"" + file + "\"")
  545.     }
  546.     defer f.Close()
  547.     text = text + "\r\n"
  548.     if _, err = f.WriteString(text); err != nil {
  549.         log("Error 2 on write file \"" + file + "\"")
  550.     }
  551. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!