Advertisement
Guest User

HALP !1!

a guest
Apr 1st, 2020
123
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Go 2.39 KB | None | 0 0
  1. package main
  2.  
  3. import (
  4.     "fmt"
  5.     "sync"
  6. )
  7.  
  8. type SafeMap struct {
  9.     v   map[string]string
  10.     mux sync.Mutex
  11. }
  12.  
  13. func (c *SafeMap) Add(key, value string) {
  14.     c.mux.Lock()
  15.     // Lock so only one goroutine at a time can access the map c.v.
  16.     c.v[key] = value
  17.     c.mux.Unlock()
  18. }
  19.  
  20. func (c *SafeMap) Value(key string) (res string,ok bool) {
  21.     c.mux.Lock()
  22.     // Lock so only one goroutine at a time can access the map c.v.
  23.     defer c.mux.Unlock()
  24.     res, ok = c.v[key]
  25.     return
  26. }
  27.  
  28.  
  29. type Fetcher interface {
  30.     // Fetch returns the body of URL and
  31.     // a slice of URLs found on that page.
  32.     Fetch(url string) (body string, urls []string, err error)
  33. }
  34.  
  35. var theMap = SafeMap{v: make(map[string]string)}
  36.  
  37. // Crawl uses fetcher to recursively crawl
  38. // pages starting with url, to a maximum of depth.
  39. func Crawl(url string, depth int, fetcher Fetcher) {
  40.    
  41.     // TODO: Fetch URLs in parallel.
  42.     // TODO: Don't fetch the same URL twice.
  43.     // This implementation doesn't do either:
  44.     _, ok := theMap.Value(url)
  45.    
  46.     if ok || depth == 0 {
  47.         return
  48.     }
  49.    
  50.     body, urls, err := fetcher.Fetch(url)
  51.     theMap.Add(url, body)
  52.    
  53.     if err != nil {
  54.         fmt.Println(err)
  55.         return
  56.     } else {
  57.         fmt.Printf("Found: url=%s : %s \n", url , body)
  58.     }
  59.    
  60.     for _,v := range urls {
  61.         go Crawl(v, depth -1 , fetcher)
  62.     }
  63. }
  64.  
  65. func main() {
  66.     Crawl("https://golang.org/", 4, fetcher)
  67. }
  68.  
  69. // fakeFetcher is Fetcher that returns canned results.
  70. type fakeFetcher map[string]*fakeResult
  71.  
  72. type fakeResult struct {
  73.     body string
  74.     urls []string
  75. }
  76.  
  77. func (f fakeFetcher) Fetch(url string) (string, []string, error) {
  78.     if res, ok := f[url]; ok {
  79.         return res.body, res.urls, nil
  80.     }
  81.     return "", nil, fmt.Errorf("Not found: %s", url)
  82. }
  83.  
  84. // fetcher is a populated fakeFetcher.
  85. var fetcher = fakeFetcher{
  86.     "https://golang.org/": &fakeResult{
  87.         "The Go Programming Language",
  88.         []string{
  89.             "https://golang.org/pkg/",
  90.             "https://golang.org/cmd/",
  91.         },
  92.     },
  93.     "https://golang.org/pkg/": &fakeResult{
  94.         "Packages",
  95.         []string{
  96.             "https://golang.org/",
  97.             "https://golang.org/cmd/",
  98.             "https://golang.org/pkg/fmt/",
  99.             "https://golang.org/pkg/os/",
  100.         },
  101.     },
  102.     "https://golang.org/pkg/fmt/": &fakeResult{
  103.         "Package fmt",
  104.         []string{
  105.             "https://golang.org/",
  106.             "https://golang.org/pkg/",
  107.         },
  108.     },
  109.     "https://golang.org/pkg/os/": &fakeResult{
  110.         "Package os",
  111.         []string{
  112.             "https://golang.org/",
  113.             "https://golang.org/pkg/",
  114.         },
  115.     },
  116. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement