hellerve/crawl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: reformat

Browse Source
This commit is contained in:
Veit Heller
2018-05-29 12:32:24 +02:00
parent 546188fb7d
commit af61b5fd75
1 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawler/crawl.go
View File

@@ -25,7 +25,6 @@ var Visited = struct {
visited map[string]bool visited map[string]bool
}{visited: make(map[string]bool)} }{visited: make(map[string]bool)}
// visitNode inspects the current node and, if it contains a link we haven’t // visitNode inspects the current node and, if it contains a link we haven’t
// visited it yet, will spawn a goroutine for it. It will also return that link, // visited it yet, will spawn a goroutine for it. It will also return that link,
// because we have to add it to our list of linked nodes. // because we have to add it to our list of linked nodes.
@@ -119,9 +118,10 @@ func parseRequest(body io.ReadCloser, parent, url string, wg *sync.WaitGroup) {
// don’t exit prematurely, since this is all concurrent. // don’t exit prematurely, since this is all concurrent.
func doCrawl(toVisit string, parent string, wg *sync.WaitGroup) { func doCrawl(toVisit string, parent string, wg *sync.WaitGroup) {
wg.Add(1) wg.Add(1)
resp, err := http.Get(toVisit)
defer wg.Done() defer wg.Done()
resp, err := http.Get(toVisit)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
return return