Go Grpc部署到 k8s【端口共享】

書接上文Go Grpc部署到 k8s【負載均衡】 grpc server端我們暴露了9090和8080端口，這次我們的http服務用iris,並且綁定到9090端口。

GO文件

服務端代碼main.go：

package main

import (
    "context"
    "fmt"
    pb "grpcdemo/protos"
    "net"

    "github.com/kataras/iris/v12"
    "github.com/kataras/iris/v12/middleware/logger"
    "github.com/kataras/iris/v12/middleware/recover"
    "github.com/soheilhy/cmux"
    "google.golang.org/grpc"
)

func main() {
    l, err := net.Listen("tcp", ":9090")
    fmt.Println(err)
    m := cmux.New(l)

    // grpc
    grpcL := m.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"))
    grpcServer := grpc.NewServer()                  // 創建GRPC
    pb.RegisterGreeterServer(grpcServer, &server{}) // 在GRPC服務端注冊服務

    go grpcServer.Serve(grpcL)

    // http
    app := iris.New()
    app.Use(recover.New())
    app.Use(logger.New())

    app.Handle("GET", "/", func(ctx iris.Context) {
        ctx.WriteString("pong")
    })

    go func() {
        httpL := m.Match(cmux.HTTP1Fast())
        app.Run(iris.Listener(httpL))
    }()

    // Start serving!
    m.Serve()
}

type server struct{}

func NewServer() *server {
    return &server{}
}

func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    msg := "Resuest By:" + in.Name + " Response By :" + LocalIp()
    fmt.Println("GRPC Send: ", msg)
    return &pb.HelloReply{Message: msg}, nil
}

func LocalIp() string {
    addrs, _ := net.InterfaceAddrs()
    var ip string = "localhost"
    for _, address := range addrs {
        if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
            if ipnet.IP.To4() != nil {
                ip = ipnet.IP.String()
            }
        }
    }
    return ip
}

服務端代碼deploy.yaml：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
    version: v1
spec:
  replicas: 1
  minReadySeconds: 10 
  selector:
    matchLabels:
      app: grpcserver
      version: v1
  template:
    metadata:
      labels:
        app: grpcserver
        version: v1
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: grpcserver
        image: 192.168.100.30:8080/go/grpcserver:2022
        ports:
        - containerPort: 9090
        imagePullPolicy: Always

---
apiVersion: v1 
kind: Service 
metadata:
  name: grpcserver
  namespace: go 
  labels:
    app: grpcserver
    version: v1
spec:
  ports:
    - port: 9090 
      targetPort: 9090 
      name: grpcserver
      protocol: TCP
  selector:
    app: grpcserver

服務端build.sh

#!/bin/bash
#cd $WORKSPACE
export GOPROXY=https://goproxy.io
 
 #根據 go.mod 文件來處理依賴關系。
go mod tidy
 
# linux環境編譯
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 構建docker鏡像，項目中需要在當前目錄下有dockerfile，否則構建失敗

docker build -t grpcserver .
docker tag  grpcserver 192.168.100.30:8080/go/grpcserver:2022

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/grpcserver
 
docker rmi  grpcserver
docker rmi 192.168.100.30:8080/go/grpcserver:2022

服務端Dockerfile

FROM golang:1.15.6

RUN mkdir -p /app

WORKDIR /app
 
ADD main /app/main

EXPOSE 9090
 
CMD ["./main"]

客戶端 default.go

package controllers

import (
    "context"
    "fmt"
    pb "grpcclient/protos"
    "io/ioutil"
    "net"
    "net/http"
    "time"

    "github.com/astaxie/beego"
    "google.golang.org/grpc"
)

type MainController struct {
    beego.Controller
}

func (c *MainController) Get() {
    address := "grpcserver:9090"
    // GRPC
    conn, _ := grpc.Dial(address, grpc.WithInsecure())
    defer conn.Close()
    grpcClient := pb.NewGreeterClient(conn)
    req := pb.HelloRequest{Name: "gavin_" + LocalIp()}

    res, err := grpcClient.SayHello(context.Background(), &req)
    if err != nil {
        msg := fmt.Sprintf("grpc client  client.SayHello  has err:%v\r\n", err)
        c.Ctx.WriteString(msg)
        return
    }

    //http
    var httpClient = http.Client{
        Timeout: 10 * time.Second,
    }
    resp, err := httpClient.Get("http://" + address)
    if err != nil {
        msg := fmt.Sprintf("http get  has err:%v\r\n", err)
        c.Ctx.WriteString(msg)
        return
    }

    defer resp.Body.Close()
    bytes, _ := ioutil.ReadAll(resp.Body)
    message := fmt.Sprintf("GRPC Clinet Received:%v ,Http Get:%v ", res.Message, string(bytes))
    //
    c.Ctx.WriteString(message)
}

func LocalIp() string {
    addrs, _ := net.InterfaceAddrs()
    var ip string = "localhost"
    for _, address := range addrs {
        if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
            if ipnet.IP.To4() != nil {
                ip = ipnet.IP.String()
            }
        }
    }
    return ip
}

客戶端 deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcclient
  namespace: go
  labels:
    app: grpcclient
    version: v1
spec:
  replicas: 1
  minReadySeconds: 10 
  selector:
    matchLabels:
      app: grpcclient
      version: v1
  template:
    metadata:
      labels:
        app: grpcclient
        version: v1
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: grpcclient
        image: 192.168.100.30:8080/go/grpcclient:2022
        ports:
        - containerPort: 8080
        imagePullPolicy: Always

---

apiVersion: v1 
kind: Service 
metadata:
  name: grpcclient
  namespace: go 
  labels:
    app: grpcclient
    version: v1
spec:
  type: ClusterIP
  ports:
    - port: 8080 
      targetPort: 8080 
      protocol: TCP
  selector:
    app: grpcclient

客戶端 build.sh

#!/bin/bash
#cd $WORKSPACE
export GOPROXY=https://goproxy.io
 
 #根據 go.mod 文件來處理依賴關系。
go mod tidy
 
# linux環境編譯
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 構建docker鏡像，項目中需要在當前目錄下有dockerfile，否則構建失敗

docker build -t grpcserver .
docker tag  grpcserver 192.168.100.30:8080/go/grpcserver:2022

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/grpcserver
 
docker rmi  grpcserver
docker rmi 192.168.100.30:8080/go/grpcserver:2022

客戶端 Dockerfile

FROM golang:1.15.6

RUN mkdir -p /app
RUN mkdir -p /app/conf
RUN mkdir -p /app/logs

WORKDIR /app
 
ADD main /app/main

EXPOSE 8080
 
CMD ["./main"]

Deploy

默認的k8s 部署運行結果：

Istio部署

​

需要先啟用攔截 kubectl label namespace go istio-injection=enabled

Istio想要啟用的話 就必須修改svc的 端口名稱如：

 但是這樣http返回失敗：

 所以用一般的方式是不能能搞定grpc和http協議.我后面嘗試了很多方法 還是沒搞定。。。。，估計只能放棄了

截圖中的name 建議不要用 http 和http2 以及grpc-  打頭， 他們有特別的意義， 同樣gateway那邊的name也要注意， istio會有helth檢測【比如我的name以http或者http2 可以過健康檢測， 但是grpc沒法用， 如果用grpc作為name ， 健康檢測都過不了， 當然實際 http 和egrpc是不通的】

記錄以下我的錯誤記錄吧， 上面的name,我試過grpcserver,grpc-port,http-port，http2,test[http2 和test是我gateway定義的]，結果pod啟動不起來錯誤信息：

Readiness probe failed: Get "http://10.244.1.128:15021/healthz/ready": dial tcp 10.244.1.128:15021: connect: connection refused

 

 把name改為http后pod可以啟動【我先說以下的圖是虛假的吧】

 

我的gateway.yaml如下：【9090是我增加的】

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: grpcserver-gateway
  namespace: go
spec:
  selector:
    istio: ingressgateway # use istio default controller
  servers:
  - port:
      number: 9090  
      name: test
      protocol: TCP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: grpcserver
  namespace: go
spec:
  hosts:
  - "*"
  gateways:
  - grpcserver-gateway
  http:
  - match:
    - port: 9090
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port:
          number: 9090

查看地址信息：

export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="test")].nodePort}')
export INGRESS_HOST=$(kubectl get po -l istio=ingressgateway -n istio-system -o 'jsonpath={.items[0].status.hostIP}')
echo $INGRESS_HOST:$INGRESS_PORT

結果是失敗的

 為了證明是是istio的問題，我的gateway.yaml文件不變，新將deploy2.yaml文件【沿用上文的grpc服務，8080和9090端口】：

apiVersion: v1
kind: Service
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
spec:
  ports:
    - name: grpc-port
      port: 9090
    - name: http-port
      port: 8080
  selector:
    app: grpcserver    
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
    version: v1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grpcserver
      version: v1
  template:
    metadata:
      labels:
        app: grpcserver
        version: v1
    spec:
      containers:
      - name: grpcserver
        image: 192.168.100.30:8080/go/grpcserver:2021
        imagePullPolicy: IfNotPresent
        ports:
          - containerPort: 8080
          - containerPort: 9090

gateway2.yaml文件如下：

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: grpcserver-gateway
  namespace: go 
spec:
  selector:
    istio: ingressgateway # use istio default controller
  servers:
  - port:
      number: 80  
      name: http
      protocol: HTTP
    hosts:
    - "*"
  - port:
      number: 9090  
      name: test
      protocol: HTTP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: grpcserver
  namespace: go 
spec:
  hosts:
  - "*"
  gateways:
  - grpcserver-gateway
  http:
  - match:
    - port: 80
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port:
          number: 8080
  - match:
    - port: 9090
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port: