打开的文件太多/运行执行并发请求的go程序时没有此类主机错误

I have a golang program which is supposed to call an API with different payloads, the web application is a drop wizard application which is running on localhost, and the go program is below

package main

import (
"bufio"
"encoding/json"
"log"
"net"
"net/http"
"os"
"strings"
"time"
)

type Data struct {
PersonnelId  string `json:"personnel_id"`
DepartmentId string `json:"department_id"`
}

type PersonnelEvent struct {
EventType string `json:"event_type"`
Data      `json:"data"`
}

const (
maxIdleConnections        = 20
maxIdleConnectionsPerHost = 20
timeout                   = time.Duration(5 * time.Second)
)

var transport = http.Transport{
Dial:                dialTimeout,
MaxIdleConns:        maxIdleConnections,
MaxIdleConnsPerHost: 20,
}

var client = &http.Client{
Transport: &transport,
}

func dialTimeout(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, timeout)
}

func makeRequest(payload string) {
req, _ := http.NewRequest("POST", "http://localhost:9350/v1/provider- 
location-personnel/index", strings.NewReader(payload))
req.Header.Set("X-App-Token", "TESTTOKEN1")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
    log.Println("Api invocation returned an error ", err)
} else {
    defer resp.Body.Close()
    log.Println(resp.Body)
}
}

func indexPersonnels(personnelPayloads []PersonnelEvent) {
for _, personnelEvent := range personnelPayloads {
    payload, err := json.Marshal(personnelEvent)
    if err != nil {
        log.Println("Error while marshalling payload ", err)
    }
    log.Println(string(payload))
    // go makeRequest(string(payload))
  }
}

func main() {
ch := make(chan PersonnelEvent)
for i := 0; i < 20; i++ {
    go func() {
        for personnelEvent := range ch {
            payload, err := json.Marshal(personnelEvent)
            if err != nil {
                log.Println("Error while marshalling payload", err)
            }
            go makeRequest(string(payload))
            //log.Println("Payload ", string(payload))
        }
    }()
}

file, err := os.Open("/Users/tmp/Desktop/personnels.txt")
defer file.Close()
if err != nil {
    log.Fatalf("Error opening personnel id file %v", err)
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
    go func() {
        ch <- PersonnelEvent{EventType: "provider_location_department_personnel_linked", Data: Data{DepartmentId: "2a8d9687-aea8-4a2c-bc08-c64d7716d973", PersonnelId: scanner.Text()}}
    }()
}

}

Its reading some ids from a file and then creating a payload out of it and invoking a post request on the web server, but when i run the program it gives too many open file errors/no such host errors, i feel that the program is too much concurrent how to make it run gracefully?

inside your 20 goroutines started in main(), "go makeRequest(...)" again created one goroutine for each event. you don't need start extra goroutine there. Besides, I think you don't need start goroutine in your scan loop, either. buffered channel is enough，because bottleneck should be at doing http requests.

You can use a buffered channel, A.K.A. counting semaphore, to limit the parallelism.

// The capacity of the buffered channel is 10,
// which means you can have 10 goroutines to 
// run the makeRequest function in parallel.
var tokens = make(chan struct{}, 10)

func makeRequest(payload string) {
    tokens <- struct{}{}            // acquire the token or block here
    defer func() { <-tokens }()     // release the token to awake another goroutine
    // other code...
}