我有一個包含大約 10k 個 URL 的 CSV 文件，我需要通過 HTTP 獲取。將 Go 例程的并發限制為一次不超過 16 個的最簡單方法是什么？func getUrl(url string) { request := gorequest.New() resp, body, errs := request.Get(each[1]).End() _ = resp _ = body _ = errs}func main() { csvfile, err := os.Open("urls.csv") defer csvfile.Close() reader := csv.NewReader(csvfile) reader.FieldsPerRecord = -1 rawCSVdata, err := reader.ReadAll() completed := 0 for _, each := range rawCSVdata { go getUrl(each[1]) completed++ }}
1 回答
當年話下
TA貢獻1890條經驗 獲得超9個贊
生產者-消費者模式：
package main
import (
"encoding/csv"
"os"
"sync"
"github.com/parnurzeal/gorequest"
)
const workersCount = 16
func getUrlWorker(urlChan chan string) {
for url := range urlChan {
request := gorequest.New()
resp, body, errs := request.Get(url).End()
_ = resp
_ = body
_ = errs
}
}
func main() {
csvfile, err := os.Open("urls.csv")
if err != nil {
panic(err)
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
reader.FieldsPerRecord = -1
rawCSVdata, err := reader.ReadAll()
var wg sync.WaitGroup
urlChan := make(chan string)
wg.Add(workersCount)
for i := 0; i < workersCount; i++ {
go func() {
getUrlWorker(urlChan)
wg.Done()
}()
}
completed := 0
for _, each := range rawCSVdata {
urlChan <- each[1]
completed++
}
close(urlChan)
wg.Wait()
}
- 1 回答
- 0 關注
- 192 瀏覽
添加回答
舉報
0/150
提交
取消