I'm experiencing a problem to communicate between GRPC client and server after switching network namespace in my Golang project. To zoom in on this issue, I modified the GRPC example program Hello World accordingly with the same problem as a result. Are there any known limitations when using GRPC after switching namespace in a golang application?
I have read about issues related to switching namespace in Golang, but I guess it depends on the GRPC behaviour if this could lead to a problem or not.
Does the GRPC client spawn any additional goroutines? Such a goroutine would not necessarily execute in the same namespace as the original goroutine locked the thread due to the namespace switch.
I assume that the GRPC server spawns goroutines for each client, but does it create a socket in the original goroutine before a new is spawned? This question is also related to that the spawned goroutines doesn't have to execute in the same namespace as where grpc.Serve() was called (namespace switch requires runtime.LockOSThread()).
Communication works in case server and client are started in matching namespaces (ip netns exec...), but fails in case namespace switch is performed internally in the client. Communication also works when namespace switch is performed internally in the server, so the problem should be in the client.
greeter_client/main.go:
package main
import (
"fmt"
"log"
"os"
"runtime"
"syscall"
"time"
pb "grpctest/helloworld/helloworld"
"github.com/vishvananda/netns"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
const (
defaultName = "world"
defaultAddress = "localhost:50051"
nsEnv = "NAMESPACE"
addressEnv = "ADDRESS"
blockEnv = "DIALBLOCK"
)
func main() {
fmt.Printf("* Client thread id before runtime.LockOSThread(): %d
", syscall.Gettid())
runtime.LockOSThread()
defer runtime.UnlockOSThread()
fmt.Printf("* Client thread id after runtime.LockOSThread(): %d
", syscall.Gettid())
var dialOpts []grpc.DialOption
dialOpts = append(dialOpts, grpc.WithInsecure())
_, ok := os.LookupEnv(blockEnv)
if ok == true {
dialOpts = append(dialOpts, grpc.WithBlock())
fmt.Printf("* Dial in blocked mode
")
} else {
fmt.Printf("* Dial in unblocked mode
")
}
address, ok := os.LookupEnv(addressEnv)
if ok == false {
address = defaultAddress
}
fmt.Printf("* Talk to server at %s
", address)
var origns netns.NsHandle
namespace, ok := os.LookupEnv(nsEnv)
if ok {
fmt.Printf("* Switch namespace to %s
", namespace)
origns, err := netns.Get()
if err != nil {
log.Fatal("failed to get current namespace")
}
defer origns.Close()
newns, err := netns.GetFromName(namespace)
if err != nil {
log.Fatalf("failed to get new namespace: %s", namespace)
}
err = netns.Set(newns)
if err != nil {
log.Fatalf("failed to set new namespace: %s", namespace)
}
defer newns.Close()
}
fmt.Printf("* Client thread id before grpc.Dial(): %d
", syscall.Gettid())
// Set up a connection to the server.
conn, err := grpc.Dial(address, dialOpts...)
if err != nil {
log.Fatalf("did not connect: %v", err)
}
defer conn.Close()
fmt.Printf("* Client thread id before pb.NewGreeterClient(): %d
", syscall.Gettid())
c := pb.NewGreeterClient(conn)
fmt.Printf("* Client thread id after pb.NewGreeterClient(): %d
", syscall.Gettid())
// Contact the server and print out its response.
name := defaultName
if len(os.Args) > 1 {
name = os.Args[1]
}
r, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: name})
if err != nil {
fmt.Printf("could not greet: %v", err)
select {}
log.Fatalf("could not greet: %v", err)
}
fmt.Printf("* Client thread id after c.SayHello(): %d
", syscall.Gettid())
log.Printf("Greeting: %s", r.Message)
time.Sleep(5 * time.Second)
if namespace != "" {
netns.Set(origns)
}
fmt.Printf("* Client thread id at exit: %d
", syscall.Gettid())
}
greeter_server/main.go:
package main
import (
"fmt"
"log"
"net"
"os"
"runtime"
"syscall"
pb "grpctest/helloworld/helloworld"
"github.com/vishvananda/netns"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
)
const (
port = ":50051"
nsEnv = "NAMESPACE"
)
// server is used to implement helloworld.GreeterServer.
type server struct{}
// SayHello implements helloworld.GreeterServer
func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
fmt.Printf("* RPC call server thread id: %d
", syscall.Gettid())
return &pb.HelloReply{Message: "Hello " + in.Name}, nil
}
func main() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
var origns netns.NsHandle
namespace := os.Getenv(nsEnv)
if namespace != "" {
fmt.Printf("* Switch namespace to %s
", namespace)
origns, err := netns.Get()
if err != nil {
log.Fatal("failed to get current namespace")
}
defer origns.Close()
newns, err := netns.GetFromName(namespace)
if err != nil {
log.Fatalf("failed to get new namespace: %s", namespace)
}
err = netns.Set(newns)
if err != nil {
log.Fatalf("failed to set new namespace: %s", namespace)
}
defer newns.Close()
}
fmt.Printf("* Main server thread id: %d
", syscall.Gettid())
lis, err := net.Listen("tcp", port)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
pb.RegisterGreeterServer(s, &server{})
// Register reflection service on gRPC server.
reflection.Register(s)
if err := s.Serve(lis); err != nil {
log.Fatalf("failed to serve: %v", err)
}
if namespace != "" {
netns.Set(origns)
}
fmt.Printf("* Main server exits (thread id: %d)
", syscall.Gettid())
}