当前位置：首页 > news >正文

golang并发http请求几种方式

news 2024/5/15 7:37:27

概述

go中方便的协程写法, 让并发变得很简单, 让开发者很多时候不需要关心协程(线程)管理的问题, 不过也是有值得注意的地方.
下面以一个http请求的例子, 请求的服务是进制数据-手机号码归属地查询.(找了很久才找到方便学demo的免费API, 想跑demo的朋友注册一下在代码中填上自己的appkey即可)

并发方式

平时常用到3种方式做并发请求.

简单粗暴, 一次请求开一个线程, 线程中完成请求与响应处理.
优雅一点, 限制线程池, 以master-worker的方式处理并发, 最后又把响应统一处理.
优雅地限制线程池(2020.7.15更新)

1. 简单并发

代码

package mainimport ("fmt""io/ioutil""net/http""sync""time"
)
func main()  {var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439","13438522395"}client = &http.Client{}beg := time.Now()wg := &sync.WaitGroup{}for _, keyword := range numberTasks{wg.Add(1)go func(keyword string, group *sync.WaitGroup) {body, err := NumberQueryRequest(keyword)if err != nil{fmt.Printf("error occurred in query keyword: %s, error: %s\n",keyword, err.Error() )}else{fmt.Printf("search %s success, data size is %d\n, body is %s\n",keyword, len(body), string(body))}group.Done()}(keyword, wg)}wg.Wait() //waitGroup在主线程等待所有任务完成fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}var client *http.Client//GET请求的通用写法, 注意复用client即可.
func NumberQueryRequest(keyword string)(body []byte, err error){url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)req, err := http.NewRequest("GET", url, nil)if err != nil{return nil, err}resp, err := client.Get(url)if err != nil{return nil, err}if resp.StatusCode != http.StatusOK{data, _ := ioutil.ReadAll(resp.Body)return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))}if resp != nil && resp.Body != nil{defer resp.Body.Close()}body, err = ioutil.ReadAll(resp.Body)if err != nil{return nil, err}return body, nil
}

解析

在for循环中有多少个任务就go出去多少个协程,没有限制,在查询量固定或者不大的时候没什么问题.
没有用到 channel的特性, 仅利用了多核调度.

2. master-worker形式

代码

package mainimport ("fmt""io/ioutil""net/http""sync""time"
)
const (routineCountTotal = 5 //限制线程数
)func main() {var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439", "13438522395"}client = &http.Client{}beg := time.Now()wg := &sync.WaitGroup{}tasks := make(chan string)results := make(chan string)//receiver接受响应并处理的函数块, 也可以单独写在一个函数go func() {for result := range results{if result == ""{close(results)}else{fmt.Println("result:", result)}}}()for i:= 0; i < routineCountTotal; i++{wg.Add(1)go worker(wg, tasks, results)}//分发任务for _, task := range numberTasks{tasks <- task}tasks <- "" //worker结束标志wg.Wait() //同步结束results <- "" // result结束标志fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}func worker(group *sync.WaitGroup, tasks chan string, result chan string){for task := range tasks{if task == ""{close(tasks)}else{respBody, err := NumberQueryRequest(task)if err != nil{fmt.Printf("error occurred in NumberQueryRequest: %s\n", task)result <- err.Error()}else{result <- string(respBody)}}}group.Done()
}var client *http.Clientfunc NumberQueryRequest(keyword string) (body []byte, err error) {url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)req, err := http.NewRequest("GET", url, nil)if err != nil {return nil, err}req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")resp, err := client.Get(url)if err != nil {return nil, err}if resp.StatusCode != http.StatusOK {data, _ := ioutil.ReadAll(resp.Body)return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))}if resp != nil && resp.Body != nil {defer resp.Body.Close()}body, err = ioutil.ReadAll(resp.Body)if err != nil {return nil, err}return body, nil
}

输出

result: {"status":0,"msg":"ok","result":{"shouji":"13434343439","province":"广东","city":"广州","company":"中国移动","cardtype":null,"areacode":"020"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13438522395","province":"四川","city":"攀枝花","company":"中国移动","cardtype":null,"areacode":"0812"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13419317885","province":"四川","city":"广安","company":"中国移动","cardtype":null,"areacode":"0826"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13456755448","province":"浙江","city":"杭州","company":"中国移动","cardtype":null,"areacode":"0571"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13419385751","province":"四川","city":"遂宁","company":"中国移动","cardtype":null,"areacode":"0825"}}
time consumed: 0.399975s

3. 使用带缓冲区的chan控制

完整代码见: https://github.com/chrispinkyang/golimitDemo

先上代码

package golimittype Glimit struct {Num intC   chan struct{}
}func NewG(num int) *Glimit {return &Glimit{Num: num,C : make(chan struct{}, num),}
}func (g *Glimit) Run(f func()){g.C <- struct{}{}go func() {f()<-g.C}()
}

使用方式

func main() {number := 10// 限制线程数2g := NewG(2)wg := sync.WaitGroup{}for i := 0; i < number; i++ {wg.Add(1)value :=i // 闭包传值方式goFunc := func() {// 做一些业务逻辑处理fmt.Printf("go func: %d\n", value)time.Sleep(time.Second)wg.Done()}g.Run(goFunc)}wg.Wait()
}

并发请求的用法

package mainimport ("fmt""io/ioutil""net/http""sync""time""./golimit"
)const (routineCountTotal = 5 //限制线程数
)func main() {var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439", "13438522395"}g := golimit.NewG(routineCountTotal)wg := &sync.WaitGroup{}client = &http.Client{}beg := time.Now()for i := 0; i < len(numberTasks); i++ {wg.Add(1)task := numberTasks[i]g.Run(func() {respBody, err := NumberQueryRequest(task)if err != nil {fmt.Printf("error occurred in NumberQueryRequest: %s\n", task)} else {fmt.Printf("response data: %s\n", string(respBody))}wg.Done()})}wg.Wait()fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}
var client *http.Clientfunc NumberQueryRequest(keyword string) (body []byte, err error) {url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)req, err := http.NewRequest("GET", url, nil)if err != nil {return nil, err}req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")resp, err := client.Get(url)if err != nil {return nil, err}if resp.StatusCode != http.StatusOK {data, _ := ioutil.ReadAll(resp.Body)return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))}if resp != nil && resp.Body != nil {defer resp.Body.Close()}body, err = ioutil.ReadAll(resp.Body)if err != nil {return nil, err}return body, nil
}

输出

response data: {"status":0,"msg":"ok","result":{"shouji":"13456755448","province":"浙江","city":"杭州","company":"中国移动","cardtype":null,"areacode":"0571"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13438522395","province":"四川","city":"攀枝花","company":"中国移动","cardtype":null,"areacode":"0812"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13434343439","province":"广东","city":"广州","company":"中国移动","cardtype":null,"areacode":"020"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13419385751","province":"四川","city":"遂宁","company":"中国移动","cardtype":null,"areacode":"0825"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13419317885","province":"四川","city":"广安","company":"中国移动","cardtype":null,"areacode":"0826"}}
time consumed: 0.937952s%