Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package main
- import (
- "fmt"
- "sync"
- )
- type SafeMap struct {
- v map[string]string
- mux sync.Mutex
- }
- func (c *SafeMap) Add(key, value string) {
- c.mux.Lock()
- // Lock so only one goroutine at a time can access the map c.v.
- c.v[key] = value
- c.mux.Unlock()
- }
- func (c *SafeMap) Value(key string) (res string,ok bool) {
- c.mux.Lock()
- // Lock so only one goroutine at a time can access the map c.v.
- defer c.mux.Unlock()
- res, ok = c.v[key]
- return
- }
- type Fetcher interface {
- // Fetch returns the body of URL and
- // a slice of URLs found on that page.
- Fetch(url string) (body string, urls []string, err error)
- }
- var theMap = SafeMap{v: make(map[string]string)}
- // Crawl uses fetcher to recursively crawl
- // pages starting with url, to a maximum of depth.
- func Crawl(url string, depth int, fetcher Fetcher) {
- // TODO: Fetch URLs in parallel.
- // TODO: Don't fetch the same URL twice.
- // This implementation doesn't do either:
- _, ok := theMap.Value(url)
- if ok || depth == 0 {
- return
- }
- body, urls, err := fetcher.Fetch(url)
- theMap.Add(url, body)
- if err != nil {
- fmt.Println(err)
- return
- } else {
- fmt.Printf("Found: url=%s : %s \n", url , body)
- }
- for _,v := range urls {
- go Crawl(v, depth -1 , fetcher)
- }
- }
- func main() {
- Crawl("https://golang.org/", 4, fetcher)
- }
- // fakeFetcher is Fetcher that returns canned results.
- type fakeFetcher map[string]*fakeResult
- type fakeResult struct {
- body string
- urls []string
- }
- func (f fakeFetcher) Fetch(url string) (string, []string, error) {
- if res, ok := f[url]; ok {
- return res.body, res.urls, nil
- }
- return "", nil, fmt.Errorf("Not found: %s", url)
- }
- // fetcher is a populated fakeFetcher.
- var fetcher = fakeFetcher{
- "https://golang.org/": &fakeResult{
- "The Go Programming Language",
- []string{
- "https://golang.org/pkg/",
- "https://golang.org/cmd/",
- },
- },
- "https://golang.org/pkg/": &fakeResult{
- "Packages",
- []string{
- "https://golang.org/",
- "https://golang.org/cmd/",
- "https://golang.org/pkg/fmt/",
- "https://golang.org/pkg/os/",
- },
- },
- "https://golang.org/pkg/fmt/": &fakeResult{
- "Package fmt",
- []string{
- "https://golang.org/",
- "https://golang.org/pkg/",
- },
- },
- "https://golang.org/pkg/os/": &fakeResult{
- "Package os",
- []string{
- "https://golang.org/",
- "https://golang.org/pkg/",
- },
- },
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement