bug: routing not working properly to pods with appropriate labels about apisix-ingress-controller

[louis2557]

this is my crd config

Upstream config

Namespace:    staging
Labels:       <none>
Annotations:  <none>
API Version:  apisix.apache.org/v2
Kind:         ApisixUpstream
Metadata:
  Creation Timestamp:  2023-12-20T09:41:45Z
  Generation:          1
  Managed Fields:
    API Version:  apisix.apache.org/v2
    Fields Type:  FieldsV1
    fieldsV1:
      f:spec:
        f:discovery:
          f:serviceName:
          f:type:
        f:subsets:
    Manager:      Terraform
    Operation:    Apply
    Time:         2023-12-20T09:41:45Z
    API Version:  apisix.apache.org/v2
    Fields Type:  FieldsV1
    fieldsV1:
      f:status:
        .:
        f:conditions:
    Manager:         apisix-ingress-controller
    Operation:       Update
    Subresource:     status
    Time:            2023-12-20T09:41:45Z
  Resource Version:  1096002412
  UID:               0a11cfdf-9c9c-4677-8094-5699bac81f5b
Spec:
  Discovery:
    Service Name:  staging/moovshop-api-writer-svc:http
    Type:          kubernetes
  Subsets:
    Labels:
      Version:  add-cart
    Name:       add-cart
    Labels:
      Version:  active-cart
    Name:       active-cart

Route config

Namespace:    staging
Labels:       <none>
Annotations:  <none>
API Version:  apisix.apache.org/v2
Kind:         ApisixRoute
Metadata:
  Creation Timestamp:  2023-12-20T09:41:50Z
  Generation:          1
  Managed Fields:
    API Version:  apisix.apache.org/v2
    Fields Type:  FieldsV1
    fieldsV1:
      f:spec:
        f:http:
    Manager:      Terraform
    Operation:    Apply
    Time:         2023-12-20T09:41:50Z
    API Version:  apisix.apache.org/v2
    Fields Type:  FieldsV1
    fieldsV1:
      f:status:
        .:
        f:conditions:
    Manager:         apisix-ingress-controller
    Operation:       Update
    Subresource:     status
    Time:            2023-12-20T09:41:50Z
  Resource Version:  1096002517
  UID:               dad3cd39-a116-46f8-bef1-6007fa0e2245
Spec:
  Http:
    Backends:
      Resolve Granularity:  service
      Service Name:         moovshop-api-writer-svc
      Service Port:         80
      Subset:               add-cart
    Match:
      Hosts:
        uat-api.moovshop.com
      Paths:
        /api/cart/item/add
    Name:  add-cart
    Plugins:
      Config:
        allow_credential:  true
        allow_headers:     X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods:     GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins:     https://uat.moovshop.com
        expose_headers:    X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      Enable:              true
      Name:                cors
      Config:
        Functions:
                  return function(conf, ctx)
            function string.starts(String, Start)
                return string.sub(String,1,string.len(Start))==Start
            end

            -- Import neccessary libraries
            local http = require("resty.http")
            local core = require("apisix.core")
            local path = ctx.curr_req_matched._path
            local checkDatetime = "true"

            if string.starts(path, "/api") and path ~= "/api/login" and string.starts(path, "/api/admin") ~= true then
                local pageToken = core.request.header(ctx, "X-Page-Token")
                local clientIp = core.request.header(ctx, "X-Forwarded-For")

                if not pageToken or pageToken == ""  then
                    core.response.exit(403, { error_msg = "pageToken is invalid" })
                end

                if not clientIp or clientIp == ""  then
                    core.response.exit(403, { error_msg = "clientIp is invalid" })
                end

                core.log.info(core.request.headers(ctx))
                core.log.info("check token: " .. pageToken .. ", client Ip: " .. clientIp)

                -- Send request to extract page token
                local httpc = http.new()
                local res, err = httpc:request_uri("http://moovshop-extract-token.staging.svc.cluster.local", {
                    method = "GET",
                    path = "/getPageTokenPayload?clientIp=" .. clientIp .. "&checkDatetime=" .. checkDatetime,
                    headers = {
                        ["X-Page-Token"] = pageToken
                    },
                })

                if not res then
                    core.response.exit(403, { error_msg = "extract page token failed" })
                end

                if res.status ~= 200 then
                    core.response.exit(403, { error_msg = "extract token failed, code: " .. res.status .. ", reason: " .. res.reason .. ", clientIp: " .. clientIp })
                else
                    local sessionId = string.sub(res.body, 2, -2)
                    core.log.info("session id from extract token: " .. sessionId)
                    core.request.add_header(ctx, "X-Session-Id", sessionId)
                    core.request.set_header(ctx, "X-Page-Token", "")
                end
            end
        end
      Enable:  true
      Name:    serverless-pre-function
    Backends:
      Resolve Granularity:  service
      Service Name:         moovshop-api-writer-svc
      Service Port:         80
      Subset:               active-cart
    Match:
      Hosts:
        uat-api.moovshop.com
      Paths:
        /api/cart/active
    Name:  active-cart
    Plugins:
      Config:
        allow_credential:  true
        allow_headers:     X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods:     GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins:     https://uat.moovshop.com
        expose_headers:    X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      Enable:              true
      Name:                cors
      Config:
        Functions:
                  return function(conf, ctx)
            function string.starts(String, Start)
                return string.sub(String,1,string.len(Start))==Start
            end

            -- Import neccessary libraries
            local http = require("resty.http")
            local core = require("apisix.core")
            local path = ctx.curr_req_matched._path
            local checkDatetime = "true"

            if string.starts(path, "/api") and path ~= "/api/login" and string.starts(path, "/api/admin") ~= true then
                local pageToken = core.request.header(ctx, "X-Page-Token")
                local clientIp = core.request.header(ctx, "X-Forwarded-For")

                if not pageToken or pageToken == ""  then
                    core.response.exit(403, { error_msg = "pageToken is invalid" })
                end

                if not clientIp or clientIp == ""  then
                    core.response.exit(403, { error_msg = "clientIp is invalid" })
                end

                core.log.info(core.request.headers(ctx))
                core.log.info("check token: " .. pageToken .. ", client Ip: " .. clientIp)

                -- Send request to extract page token
                local httpc = http.new()
                local res, err = httpc:request_uri("http://moovshop-extract-token.staging.svc.cluster.local", {
                    method = "GET",
                    path = "/getPageTokenPayload?clientIp=" .. clientIp .. "&checkDatetime=" .. checkDatetime,
                    headers = {
                        ["X-Page-Token"] = pageToken
                    },
                })

                if not res then
                    core.response.exit(403, { error_msg = "extract page token failed" })
                end

                if res.status ~= 200 then
                    core.response.exit(403, { error_msg = "extract token failed, code: " .. res.status .. ", reason: " .. res.reason .. ", clientIp: " .. clientIp })
                else
                    local sessionId = string.sub(res.body, 2, -2)
                    core.log.info("session id from extract token: " .. sessionId)
                    core.request.add_header(ctx, "X-Session-Id", sessionId)
                    core.request.set_header(ctx, "X-Page-Token", "")
                end
            end
        end
      Enable:  true
      Name:    serverless-pre-function

[Revolyssup]

@louis2557 You dont need to add

  Discovery:
    Service Name:  staging/moovshop-api-writer-svc:http
    Type:          kubernetes

in Upstream configuration. Can you try removing that?

[louis2557]

@louis2557 You dont need to add

  Discovery:
    Service Name:  staging/moovshop-api-writer-svc:http
    Type:          kubernetes

in Upstream configuration. Can you try removing that?

seems not working, I also make some changes. Since I used one service with multiple endpoints, I removed "Resolve Granularity: service" but it is not able to get the endpoints ip for the upstream. The config attribute "nodes" of all upstreams are empty array. dun know how to config

[Revolyssup]

@louis2557 Can you show the output of
kubectl get service moovshop-api-writer-svc -n staging
kubectl get apisixupstream <name of your upstream> -n staging
kubectl get apisixroute <name of your route> -n staging
after you made the above change.

[louis2557]

@louis2557 Can you show the output of kubectl get service moovshop-api-writer-svc -n staging kubectl get apisixupstream <name of your upstream> -n staging kubectl get apisixroute <name of your route> -n staging after you made the above change.

Oh, I found the problem, apisixupstream name must be the same as k8s service name. However, I still got the error in ingress controller, looks like the upstream name is automatically generated which were included subset name but the ingress connect to the upstream name without subset name, how can i to solve it?

the log like this
2023-12-27T16:00:03+08:00 error apisix/apisix_upstream.go:332 failed to get upstream staging_moovshop-api-writer-svc_80: not found 2023-12-27T16:00:03+08:00 warn apisix/apisix_upstream.go:488 sync ApisixUpstream failed, will retry {"object": {"Type":1,"Object":{"Key":"staging/moovshop-api-writer-svc","OldObject":null,"GroupVersion":"apisix.apache.org/v2"},"OldObject":null,"Tombstone":null}, "error": "not found"}

kubectl get service moovshop-api-writer-svc -n staging

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
moovshop-api-writer-svc ClusterIP 172.20.107.19 80/TCP,443/TCP 69d

kubectl get service moovshop-api-writer-svc -n staging

NAME AGE
moovshop-api-writer-svc 6m59s

kubectl get apisixroute moovshop-writer-route -n staging

NAME HOSTS URIS AGE
moovshop-writer-route ["uat-api.moovshop.com"] ["/api/cart/item/remove"] 7m51s

[Revolyssup]

@louis2557 Yes the upstream name should match the service name. The configuration you pasted didn't have upstream name so I couldn't see that. In my last message I forgot to mention -o yaml. Can you show the output of the above kubectl commands that I asked but with -o yaml. I want to confirm the state of these resources when you get this error.

[louis2557]

@louis2557 Yes the upstream name should match the service name. The configuration you pasted didn't have upstream name so I couldn't see that. In my last message I forgot to mention -o yaml. Can you show the output of the above kubectl commands that I asked but with -o yaml. I want to confirm the state of these resources when you get this error.

service

apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"argocd.argoproj.io/instance":"moovshop-api"},"name":"moovshop-api-writer-svc","namespace":"staging"},"spec":{"ports":[{"name":"http","port":80,"protocol":"TCP","targetPort":80},{"name":"https","port":443,"protocol":"TCP","targetPort":443}],"selector":{"app":"moovshop-api-writer"},"type":"ClusterIP"}}
  creationTimestamp: "2023-10-19T07:31:24Z"
  labels:
    argocd.argoproj.io/instance: moovshop-api
  name: moovshop-api-writer-svc
  namespace: staging
  resourceVersion: "990315141"
  uid: 42b5910e-bf03-4340-9f89-edbac0d720eb
spec:
  clusterIP: 172.20.107.19
  clusterIPs:
  - 172.20.107.19
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
  - name: https
    port: 443
    protocol: TCP
    targetPort: 443
  selector:
    app: moovshop-api-writer
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}

apisixupstream

apiVersion: apisix.apache.org/v2
kind: ApisixUpstream
metadata:
  creationTimestamp: "2023-12-27T07:55:58Z"
  generation: 1
  name: moovshop-api-writer-svc
  namespace: staging
  resourceVersion: "1107274261"
  uid: 54510349-a106-479f-a806-3a260aebf478
spec:
  subsets:
  - labels:
      version: add-cart
    name: add-cart
  - labels:
      version: active-cart
    name: active-cart
  - labels:
      version: checkout
    name: checkout
  - labels:
      version: live
    name: live
status:
  conditions:
  - message: not found
    observedGeneration: 1
    reason: ResourceSyncAborted
    status: "False"
    type: ResourcesAvailable

apisixroute

apiVersion: apisix.apache.org/v2
kind: ApisixRoute
metadata:
  creationTimestamp: "2023-12-27T07:56:01Z"
  generation: 2
  name: moovshop-writer-route
  namespace: staging
  resourceVersion: "1107366295"
  uid: 44ddcadf-8d51-47c7-ab85-f76b2551afe6
spec:
  http:
  - match:
      hosts:
      - uat-api.moovshop.com
      paths:
      - /api/cart/item/remove
    name: remove-cart
    plugins:
    - config:
        allow_credential: true
        allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins: https://uat.moovshop.com
        expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      enable: true
      name: cors
    - config:
        functions:
        - |2-
                ...
      enable: true
      name: serverless-pre-function
    upstreams:
    - name: moovshop-api-writer-svc
  - match:
      hosts:
      - uat-api.moovshop.com
      paths:
      - /api/cart/item/add
    name: add-cart
    plugins:
    - config:
        allow_credential: true
        allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins: https://uat.moovshop.com
        expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      enable: true
      name: cors
    - config:
        functions:
        - |2-
                  ...
      enable: true
      name: serverless-pre-function
    upstreams:
    - name: moovshop-api-writer-svc
  - match:
      hosts:
      - uat-api.moovshop.com
      paths:
      - /api/cart/active
    name: active-cart
    plugins:
    - config:
        allow_credential: true
        allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins: https://uat.moovshop.com
        expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      enable: true
      name: cors
    - config:
        functions:
        - |2-
                  ...
      enable: true
      name: serverless-pre-function
    upstreams:
    - name: moovshop-api-writer-svc
  - match:
      hosts:
      - uat-api.moovshop.com
      paths:
      - /api/cart/checkout
    name: checkout
    plugins:
    - config:
        allow_credential: true
        allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
        allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
        allow_origins: https://uat.moovshop.com
        expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
      enable: true
      name: cors
    - config:
        functions:
        - |2-
                  ...
      enable: true
      name: serverless-pre-function
    upstreams:
    - name: moovshop-api-writer-svc
status:
  conditions:
  - message: "4 errors occurred:\n\t* unexpected status code 400; error message: {\"error_msg\":\"failed
      to fetch upstream info by upstream id [fb04cf71], response code: 404\"}\n\n\t*
      unexpected status code 400; error message: {\"error_msg\":\"failed to fetch
      upstream info by upstream id [fb04cf71], response code: 404\"}\n\n\t* unexpected
      status code 400; error message: {\"error_msg\":\"failed to fetch upstream info
      by upstream id [fb04cf71], response code: 404\"}\n\n\t* unexpected status code
      400; error message: {\"error_msg\":\"failed to fetch upstream info by upstream
      id [fb04cf71], response code: 404\"}\n\n\n"
    observedGeneration: 2
    reason: ResourceSyncAborted
    status: "False"
    type: ResourcesAvailable

[Revolyssup]

@louis2557 Did you remove

  Discovery:
    Service Name:  staging/moovshop-api-writer-svc:http
    Type:          kubernetes

in Upstream configuration.
Also did you change the name of ApisixUpstream? Can you delete and create the ApisixUpstream yaml again?

[Revolyssup]

@louis2557 Can you show the output of kubectl get service moovshop-api-writer-svc -n staging kubectl get apisixupstream <name of your upstream> -n staging kubectl get apisixroute <name of your route> -n staging after you made the above change.

Oh, I found the problem, apisixupstream name must be the same as k8s service name. However, I still got the error in ingress controller, looks like the upstream name is automatically generated which were included subset name but the ingress connect to the upstream name without subset name, how can i to solve it?

the log like this 2023-12-27T16:00:03+08:00 error apisix/apisix_upstream.go:332 failed to get upstream staging_moovshop-api-writer-svc_80: not found 2023-12-27T16:00:03+08:00 warn apisix/apisix_upstream.go:488 sync ApisixUpstream failed, will retry {"object": {"Type":1,"Object":{"Key":"staging/moovshop-api-writer-svc","OldObject":null,"GroupVersion":"apisix.apache.org/v2"},"OldObject":null,"Tombstone":null}, "error": "not found"}

kubectl get service moovshop-api-writer-svc -n staging

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE moovshop-api-writer-svc ClusterIP 172.20.107.19 80/TCP,443/TCP 69d

kubectl get service moovshop-api-writer-svc -n staging

NAME AGE moovshop-api-writer-svc 6m59s

kubectl get apisixroute moovshop-writer-route -n staging

NAME HOSTS URIS AGE moovshop-writer-route ["uat-api.moovshop.com"] ["/api/cart/item/remove"] 7m51s

@louis2557 Please also show the output of kubectl get apisixupstream -n staging -o yaml