We are running core go server in production under load balancer and other stuff.
Now when goroutine count(with pprof trace) is more than 2500, our HTTP client is returning nil as result.
Here is my code for HTTP client.
client := &http.Client{
Timeout: time.Second * 130,
Transport: netTransport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
go func() {
select {
case <-ctx.Done():
log.Error(ctx.Err()) // prints "context deadline exceeded"
return
}
}()
vars := mux.Vars(r)
validateTokenURL = authURL + "/" + vars["resource"] + "/authorized" + "?url=" + path
req, _ := http.NewRequest("GET", validateTokenURL, nil)
req.Header = r.Header
req = req.WithContext(ctx)
res, error := client.Do(req)
if error != nil {
log.Error("The group's number increased tremendously!")
return nil, nil, false
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Error(err.Error())
}
json.Unmarshal(body, &responseBody)
res.Body.Close()
if _, ok := responseBody["data"]; ok && strings.Compare(reflect.TypeOf(responseBody["data"]).String(), "string") == 0 && strings.Compare(responseBody["data"].(string), "ok") == 0 {
return responseBody, res, true
}
return responseBody, res, false
}
I have set ulimit to approx 8 GB, plus we have 16 GB as RAM. Memory uses is less than 20%.
Can anyone help us to figure it out what we are missing?