googlecloudplatform / gke-autoneg-controller Goto Github PK

annotations:
    cloud.google.com/neg: '{"exposed_ports": {"80":{}}}'
    controller.autoneg.dev/neg: '{"backend_services":{"80":[{"name":"gke-development-ingress-backend-web","zones":["us-central1-c"]}]}}'

Events:                                                                                                                                                                                                                                                                               
│   Type     Reason        Age                From                Message                                                                                                                                                                                                               │
│   ----     ------        ----               ----                -------                                                                                                                                                                                                               │
│   Normal   Sync          32s                autoneg-controller  Synced NEGs for "traefik/traefik" as backends to backend service "gke-development-ingress-backend-web" (port 80)                                                                                                      │
│   Normal   Create        25s (x2 over 15m)  neg-controller      Created NEG "k8s1-c278397d-traefik-traefik-80-230eaa9a" for traefik/traefik-k8s1-c278397d-traefik-traefik-80-230eaa9a--web/80-web-GCE_VM_IP_PORT-L7 in "us-central1-b".                                               │
│   Warning  BackendError  25s (x9 over 32s)  autoneg-controller  googleapi: Error 404: The resource 'projects/feltboard-terraform/zones/us-central1-c/networkEndpointGroups/k8s1-c278397d-traefik-traefik-80-230eaa9a' was not found, notFound                                         │
│   Normal   Create        13s (x2 over 15m)  neg-controller      Created NEG "k8s1-c278397d-traefik-traefik-80-230eaa9a" for traefik/traefik-k8s1-c278397d-traefik-traefik-80-230eaa9a--web/80-web-GCE_VM_IP_PORT-L7 in "us-central1-c".                                               │
│   Normal   Attach        10s (x2 over 14m)  neg-controller      Attach 1 network endpoint(s) (NEG "k8s1-c278397d-traefik-traefik-80-230eaa9a" in zone "us-central1-b")                                                                                                                │
│   Warning  BackendError  3s (x7 over 11m)   autoneg-controller  googleapi: Error 400: Invalid value for field 'resource.backends[0]': '{  "resourceGroup": "https://www.googleapis.com/compute/v1/projects/xx/zones/us-cen...'. CONNECTION balancing mode is not supported for protocol HTTP, invalid     

          command:
            - /manager
          args:
            - --metrics-bind-address=:8080
            - --health-probe-bind-address=:8081
            - --leader-elect
            - --zap-devel=false
            - --zap-encoder=json
            - --zap-time-encoding=rfc3339

I0718 03:39:16.676513       1 request.go:601] Waited for 1.004034269s due to client-side throttling, not priority and fairness, request: GET:https://10.30.180.1:443/apis/vulnerabilities.protect.gke.io/v1?timeout=32s
{"level":"info","ts":"2023-07-18T03:39:17Z","logger":"controller-runtime.metrics","msg":"Metrics server is starting to listen","addr":":8080"}
{"level":"info","ts":"2023-07-18T03:39:17Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2023-07-18T03:39:17Z","msg":"Starting server","path":"/metrics","kind":"metrics","addr":"[::]:8080"}
{"level":"info","ts":"2023-07-18T03:39:17Z","msg":"Starting server","kind":"health probe","addr":"[::]:8081"}
I0718 03:39:17.791176       1 leaderelection.go:248] attempting to acquire leader lease autoneg-system/9fe89c94.controller.autoneg.dev...
E0718 03:39:17.801515       1 leaderelection.go:330] error retrieving resource lock autoneg-system/9fe89c94.controller.autoneg.dev: leases.coordination.k8s.io "9fe89c94.controller.autoneg.dev" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "autoneg-system"
E0718 03:39:21.244438       1 leaderelection.go:330] error retrieving resource lock autoneg-system/9fe89c94.controller.autoneg.dev: leases.coordination.k8s.io "9fe89c94.controller.autoneg.dev" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "autoneg-system"
E0718 03:39:24.292654       1 leaderelection.go:330] error retrieving resource lock autoneg-system/9fe89c94.controller.autoneg.dev: leases.coordination.k8s.io "9fe89c94.controller.autoneg.dev" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "autoneg-system"
E0718 03:39:28.356441       1 leaderelection.go:330] error retrieving resource lock autoneg-system/9fe89c94.controller.autoneg.dev: leases.coordination.k8s.io "9fe89c94.controller.autoneg.dev" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "autoneg-system"
E0718 03:39:32.195193       1 leaderelection.go:330] error retrieving resource lock autoneg-system/9fe89c94.controller.autoneg.dev: leases.coordination.k8s.io "9fe89c94.controller.autoneg.dev" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "autoneg-system"

securityContext:        
  privileged: false        
  allowPrivilegeEscalation: false        
  runAsUser: 1002        
  runAsNonRoot: true

curl -k https://10.1.41.10:8443/metrics
Unauthorized

E0727 18:09:24.847396       1 reflector.go:126] pkg/mod/k8s.io/[email protected]+incompatible/tools/cache/reflector.go:94: Failed to list *v1.Service: services is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot list resource "services" in API group "" at the cluster scope
E0727 18:09:24.891173       1 leaderelection.go:306] error retrieving resource lock autoneg-system/controller-leader-election-helper: configmaps "controller-leader-election-helper" is forbidden: User "system:serviceaccount:autoneg-system:autoneg" cannot get resource "configmaps" in API group "" in the namespace "autoneg-system"

apiVersion: v1
kind: Service
metadata:
  name: frontend-svc
  annotations:
    cloud.google.com/neg: '{"exposed_ports": {"443":{}}}'
    controller.autoneg.dev/neg: '{"backend_services":{"443":[{"name":"https-be","max_connections_per_endpoint":1000}]}}'
spec:
  selector:
    app: frontend-app
  type: NodePort
  ports:
    - protocol: TCP
      port: 443
      targetPort: 3000

Events:
  Type     Reason        Age                 From                Message
  ----     ------        ----                ----                -------
  Normal   Sync          32s                 autoneg-controller  Synced NEGs for "default/frontend-svc" as backends to backend service "https-be" (port 443)
  Normal   Create        23s                 neg-controller      Created NEG "k8s1-1f4ed5c4-default-frontend-svc-443-9757dbe8" for default/frontend-svc-k8s1-1f4ed5c4-default-frontend-svc-443-9757dbe8--/443-3000-GCE_VM_IP_PORT-L7 in "us-central1-f".
  Warning  BackendError  11s (x13 over 32s)  autoneg-controller  googleapi: Error 403: Request had insufficient authentication scopes.
Details:
[
  {
    "@type": "type.googleapis.com/google.rpc.ErrorInfo",
    "domain": "googleapis.com",
    "metadatas": {
      "method": "compute.v1.BackendServicesService.Get",
      "service": "compute.googleapis.com"
    },
    "reason": "ACCESS_TOKEN_SCOPE_INSUFFICIENT"
  }
]

$ gcloud iam roles describe autoneg --project=$PROJECT_ID
etag: REDACTED
includedPermissions:
- compute.backendServices.get
- compute.backendServices.update
- compute.healthChecks.useReadOnly
- compute.networkEndpointGroups.use
- compute.regionBackendServices.get
- compute.regionBackendServices.update
- compute.regionHealthChecks.useReadOnly
name: projects/${PROJECT_ID}/roles/autoneg
stage: ALPHA
title: autoneg

DEBUG	events	Normal	{"object": {"kind":"Service","namespace":"test","name":"test-marianh","uid":"d362ac9e-f5dd-48ee-a167-91abf6c1be0a","apiVersion":"v1","resourceVersion":"14606392"}, "reason": "Delete", "message": "Deregistered NEGs for \"test/test-marianh\" from backend service \"autoneg-w1\" (port 80)"}"

$ gcloud compute backend-services list --project=anthos-mesh-sandbox-1b827f9b 
NAME        BACKENDS                                                                                                                                                                                                                                            PROTOCOL
autoneg-w1  europe-west1-b/networkEndpointGroups/k8s1-6b097e81-test-test-marianh-80-1330420d,europe-west1-c/networkEndpointGroups/k8s1-6b097e81-test-test-marianh-80-1330420d,europe-west1-d/networkEndpointGroups/k8s1-6b097e81-test-test-marianh-80-1330420d  TCP

$ gcloud compute network-endpoint-groups list --project=anthos-mesh-sandbox-1b827f9b  | grep marianh
k8s1-6b097e81-test-test-marianh-80-1330420d                      europe-west1-b  GCE_VM_IP_PORT  0
k8s1-6b097e81-test-test-marianh-80-1330420d                      europe-west1-c  GCE_VM_IP_PORT  0
k8s1-6b097e81-test-test-marianh-80-1330420d                      europe-west1-d  GCE_VM_IP_PORT  1

Received unexpected error:
            	FatalError{Underlying: error while running command: exit status 1; �[31m╷�[0m�[0m
            	�[31m│�[0m �[0m�[1m�[31mError: �[0m�[0m�[1mError creating the custom project role projects/<project-id>/roles/autonegRegional: googleapi: Error 400: You can't create a role_id (autonegRegional) which has been marked for deletion., failedPrecondition�[0m
            	�[31m│�[0m �[0m
            	�[31m│�[0m �[0m�[0m  with module.dwam_test.module.autoneg[0].module.gcp.google_project_iam_custom_role.autoneg,
            	�[31m│�[0m �[0m  on .terraform/modules/dwam_test.autoneg/terraform/gcp/main.tf line 57, in resource "google_project_iam_custom_role" "autoneg":
            	�[31m│�[0m �[0m  57: resource "google_project_iam_custom_role" "autoneg" �[4m{�[0m�[0m
            	�[31m│�[0m �[0m
            	�[31m╵�[0m�[0m}

message: "pkg/mod/k8s.io/[email protected]+incompatible/tools/cache/reflector.go:94: Failed to list *v1.Service: services is forbidden: User "system:serviceaccount:autoneg-per-ns:default" cannot list resource "services" in API group "" at the cluster scope"
source: "reflector.go:126"

​ Type     Reason        Age                  From                Message
  ----     ------        ----                 ----                -------
  Normal   Create        7m54s                neg-controller      Created NEG "k8s1-85a2e695-default-app-catfood-80-fb89a260" for default/app-catfood-k8s1-85a2e695-default-app-catfood-80-fb89a260--web/80-80-GCE_VM_IP_PORT-L7 in "europe-west1-b".
  Normal   Create        7m51s                neg-controller      Created NEG "k8s1-85a2e695-default-app-catfood-80-fb89a260" for default/app-catfood-k8s1-85a2e695-default-app-catfood-80-fb89a260--web/80-80-GCE_VM_IP_PORT-L7 in "europe-west1-c".
  Normal   Create        7m49s                neg-controller      Created NEG "k8s1-85a2e695-default-app-catfood-80-fb89a260" for default/app-catfood-k8s1-85a2e695-default-app-catfood-80-fb89a260--web/80-80-GCE_VM_IP_PORT-L7 in "europe-west1-d".
  Normal   Attach        7m45s (x2 over 27h)  neg-controller      Attach 1 network endpoint(s) (NEG "k8s1-85a2e695-default-app-catfood-80-fb89a260" in zone "europe-west1-b")
Warning  BackendError  7m44s (x2 over 26h)  autoneg-controller  Operation cannot be fulfilled on services "app-catfood": the object has been modified; please apply your changes to the latest version and try again

Normal Sync 1s (x2 over 26h) autoneg-controller Synced NEGs for "default/app-catfood" as backends to backend service "catfood"

You're using deprecated Ginkgo functionality:
=============================================
Ginkgo 2.0 is under active development and will introduce several new features, improvements, and a small handful of breaking changes.
A release candidate for 2.0 is now available and 2.0 should GA in Fall 2021.  Please give the RC a try and send us feedback!
  - To learn more, view the migration guide at https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md
  - For instructions on using the Release Candidate visit https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md#using-the-beta
  - To comment, chime in at https://github.com/onsi/ginkgo/issues/711

  You are using a custom reporter.  Support for custom reporters will likely be removed in V2.  Most users were using them to generate junit or teamcity reports and this functionality will be merged into the core reporter.  In addition, Ginkgo 2.0 will support emitting a JSON-formatted report that users can then manipulate to generate custom reports.

  If this change will be impactful to you please leave a comment on https://github.com/onsi/ginkgo/issues/711
  Learn more at: https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md#removed-custom-reporters

To silence deprecations that can be silenced set the following environment variable:
  ACK_GINKGO_DEPRECATIONS=1.16.5

COPY . .

2021-10-22T17:55:52.092Z	INFO	controllers.Service	Applying intended status	{"service": "envoy/envoy", "status": {"backend_services":{"8000":{"myproduct":{"name":"myproduct","max_connections_per_endpoint":1000}}},"network_endpoint_groups":{"8000":"k8s1-4fd3dc4c-envoy-envoy-8000-44f9746b","8001":"k8s1-4fd3dc4c-envoy-envoy-8001-7508741b"},"zones":["europe-west1-b","europe-west1-c","europe-west1-d"]}}
2021-10-22T17:55:52.762Z	ERROR	controller-runtime.controller	Reconciler error	{"controller": "service", "request": "envoy/envoy", "error": "googleapi: Error 404: The resource 'projects/myproduct-dev/zones/europe-west1-c/networkEndpointGroups/k8s1-4fd3dc4c-envoy-envoy-8000-44f9746b' was not found, notFound"}

func (b Backends) isEqual(ob Backends) bool {
	if b.name != ob.name {
		return false
	}
	newB := map[string]compute.Backend{}
	for _, be := range b.backends {
		rb := relevantCopy(be)
		if _, ok := newB[rb.Group]; ok {
			return false
		}
		newB[rb.Group] = rb
	}
	newOB := map[string]compute.Backend{}
	for _, be := range ob.backends {
		rb := relevantCopy(be)
		if _, ok := newOB[rb.Group]; ok {
			return false
		}
		newOB[rb.Group] = rb
	}
	return reflect.DeepEqual(newB, newOB)
}

DEBUG	controller-runtime.controller	Successfully Reconciled	{"controller": "service", "request": "foo/bar}

apiVersion: v1
kind: Service
metadata:
  name: neg-demo-svc
  annotations:
    cloud.google.com/neg: '{"exposed_ports": {"80":{}}}'
    controller.autoneg.dev/neg: '{"backend_services":{"80":[{"name":"some-backend-service","region":"europe-west2","max_rate_per_endpoint":100}]}}'
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
  - port: 80
    protocol: TCP

2022-02-25T16:45:12.987+0100    DEBUG   controller-runtime.manager.events       Normal  {"object": {"kind":"Service","namespace":"default","name":"neg-demo-svc","uid":"affa73d3-10e3-4698-ae02-d1c1c22ad748","apiVersion":"v1","resourceVersion":"420760"}, "reason": "Delete", "message": "Deregistered NEGs for \"default/neg-demo-svc\" from backend service \"argon-gke-general-blue-01-psc-backend-service\" (port 80)"}

autoneg-controller-manager manager 2019-12-10T19:58:09.561Z    INFO    controllers.Service     Applying intended status        {"service": "foo/bar", "status": {"name":"bar","max_rate_per_endpoint":1000,"network_endpoint_groups":{"80":"k8s1-foo-bar-80-12345a"},"zones":["us-central1-a"]}}
autoneg-controller-manager manager 2019-12-10T19:58:09.990Z    ERROR   controller-runtime.controller   Reconciler error        {"controller": "service", "request": "foo/bar", "error": "googleapi: Error 412: Precondition Failed, conditionNotMet"}

Events:
  Type     Reason     Age                   From               Message
  ----     ------     ----                  ----               -------
  Normal   Scheduled  19m                   default-scheduler  Successfully assigned autoneg-system/autoneg-controller-manager-758bc7f94b-tzvz9 to gke-d003-sb2-k8s-euwe1-node-pool-1-e0234f93-8c4q
  Normal   Pulling    19m                   kubelet            Pulling image "gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0"
  Normal   Started    19m                   kubelet            Started container kube-rbac-proxy
  Normal   Pulled     19m                   kubelet            Successfully pulled image "gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0" in 1.270493418s
  Normal   Created    19m                   kubelet            Created container kube-rbac-proxy
  Warning  Failed     18m (x3 over 19m)     kubelet            Failed to pull image "docker.pkg.github.com/googlecloudplatform/gke-autoneg-controller/gke-autoneg-controller:0.9.1": rpc error: code = Unknown desc = Error response from daemon: Get https://docker.pkg.github.com/v2/googlecloudplatform/gke-autoneg-controller/gke-autoneg-controller/manifests/0.9.1: no basic auth credentials

2022-07-27T18:52:41.739Z	ERROR	controller-runtime.controller	Reconciler error	{"controller": "service", "request": "<name_space>/<service>", "error": "googleapi: Error 401: Invalid Credentials, authError"}
github.com/go-logr/zapr.(*zapLogger).Error
	/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:218
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:192
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
	/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:171
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152
k8s.io/apimachinery/pkg/util/wait.JitterUntil
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153
k8s.io/apimachinery/pkg/util/wait.Until
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

$ git clone [email protected]:GoogleCloudPlatform/gke-autoneg-controller.git
$ cd gke-autoneg-controller
$ docker build -tx .
------
 > [builder 7/9] COPY api/ api/:
------
failed to compute cache key: "/api" not found: not found