2023-06-20 14:23:10,004 ERROR scheduling-1 core.service.task.GeaflowTaskSubmitter:101 - Task ["1686902126697882985"] Submit Waiting Timeout
java.util.concurrent.TimeoutException: null
at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1784) ~[?:1.8.0_372]
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1928) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.submit(GeaflowTaskSubmitter.java:98) ~[geaflow-console-core-service-0.1.jar!/:?]
at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) ~[spring-context-5.3.26.jar!/:5.3.26]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-5.3.26.jar!/:5.3.26]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:95) ~[spring-context-5.3.26.jar!/:5.3.26]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_372]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_372]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
2023-06-20 14:23:51,446 ERROR pool-3-thread-11 common.util.exception.GeaflowLogException:24 - Start task 1686902126697882985 failed
com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
... 16 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
... 16 more
2023-06-20 14:23:51,446 ERROR pool-3-thread-11 core.service.task.GeaflowTaskOperator:109 - Submit task 1686902126697882985 failed
com.antgroup.geaflow.console.common.util.exception.GeaflowLogException: Start task 1686902126697882985 failed
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:96) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
2023-06-20 14:23:51,449 ERROR pool-3-thread-11 core.service.task.GeaflowTaskSubmitter:86 - task 1686902126697882985 submit error: Start task 1686902126697882985 failed
com.antgroup.geaflow.console.common.util.exception.GeaflowLogException: Start task 1686902126697882985 failed
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:96) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687270930023-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687270930023-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
2023-06-20 14:31:50,010 INFO http-nio-8081-exec-10 core.service.file.RemoteFileStorage:73 - Start upload file, url=http://xxx:8081/api/tasks/%s/files?path=/tmp/geaflow/packages/1686898993810030898/release-20.zip
2023-06-20 14:31:50,011 INFO http-nio-8081-exec-10 core.service.file.RemoteFileStorage:75 - Upload success, url=http://xxx:8081/api/tasks/%s/files?path=/tmp/geaflow/packages/1686898993810030898/release-20.zip
2023-06-20 14:31:54,819 INFO http-nio-8081-exec-1 core.service.file.RemoteFileStorage:73 - Start upload file, url=http://xxx:8081/api/tasks/%s/files?path=/tmp/geaflow/packages/1686898993810030898/release-20.zip
2023-06-20 14:31:54,820 INFO http-nio-8081-exec-1 core.service.file.RemoteFileStorage:75 - Upload success, url=http://xxx:8081/api/tasks/%s/files?path=/tmp/geaflow/packages/1686898993810030898/release-20.zip
2023-06-20 14:31:54,821 WARN http-nio-8081-exec-1 common.dal.dao.ReleaseDao:179 - SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@2bc8ded8] Transaction not enabled
2023-06-20 14:31:55,163 INFO http-nio-8081-exec-5 biz.shared.impl.TaskManagerImpl:146 - submit task successfully, waiting for scheduling. id: 1686902126697882985
2023-06-20 14:32:00,001 INFO scheduling-1 core.service.task.GeaflowTaskSubmitter:61 - task submitter start, task size: 1
2023-06-20 14:32:00,019 INFO pool-3-thread-12 core.service.task.GeaflowTaskSubmitter:75 - task 1686902126697882985 submit start, curr status: WAITING
2023-06-20 14:33:00,002 ERROR scheduling-1 core.service.task.GeaflowTaskSubmitter:101 - Task ["1686902126697882985"] Submit Waiting Timeout
java.util.concurrent.TimeoutException: null
at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1784) ~[?:1.8.0_372]
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1928) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.submit(GeaflowTaskSubmitter.java:98) ~[geaflow-console-core-service-0.1.jar!/:?]
at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) ~[spring-context-5.3.26.jar!/:5.3.26]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-5.3.26.jar!/:5.3.26]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:95) ~[spring-context-5.3.26.jar!/:5.3.26]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_372]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_372]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_372]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
2023-06-20 14:33:40,749 ERROR pool-3-thread-12 common.util.exception.GeaflowLogException:24 - Start task 1686902126697882985 failed
com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
... 16 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
... 16 more
2023-06-20 14:33:40,749 ERROR pool-3-thread-12 core.service.task.GeaflowTaskOperator:109 - Submit task 1686902126697882985 failed
com.antgroup.geaflow.console.common.util.exception.GeaflowLogException: Start task 1686902126697882985 failed
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:96) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
2023-06-20 14:33:40,752 ERROR pool-3-thread-12 core.service.task.GeaflowTaskSubmitter:86 - task 1686902126697882985 submit error: Start task 1686902126697882985 failed
com.antgroup.geaflow.console.common.util.exception.GeaflowLogException: Start task 1686902126697882985 failed
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:96) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.start(K8sRuntime.java:55) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskOperator.start(GeaflowTaskOperator.java:100) ~[geaflow-console-core-service-0.1.jar!/:?]
at com.antgroup.geaflow.console.core.service.task.GeaflowTaskSubmitter.lambda$submit$0(GeaflowTaskSubmitter.java:83) ~[geaflow-console-core-service-0.1.jar!/:?]
at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_372]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_372]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:96) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: com.antgroup.geaflow.common.exception.GeaflowRuntimeException: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:41) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: POST at: https://xxx:6443/api/v1/namespaces/tugraph/pods. Message: pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found. Received status: Status(apiVersion=v1, code=403, details=StatusDetails(causes=[], group=null, kind=pods, name=geaflow1686902126697882985-1687271520022-client, retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status, message=pods "geaflow1686902126697882985-1687271520022-client" is forbidden: error looking up service account tugraph/geaflow: serviceaccount "geaflow" not found, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Forbidden, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:639) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:578) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:543) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:504) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:292) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:893) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:372) ~[?:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:86) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.lambda$createPod$6(GeaflowKubeClient.java:101) ~[?:?]
at com.antgroup.geaflow.common.utils.RetryCommand.run(RetryCommand.java:36) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.runWithRetries(GeaflowKubeClient.java:125) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.clustermanager.GeaflowKubeClient.createPod(GeaflowKubeClient.java:104) ~[?:?]
at com.antgroup.geaflow.cluster.k8s.client.KubernetesJobClient.submitJob(KubernetesJobClient.java:93) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_372]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_372]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_372]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_372]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.lambda$invoke$0(GeaflowInvocationHandler.java:43) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.LoaderSwitchUtil.call(LoaderSwitchUtil.java:38) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.antgroup.geaflow.console.common.util.proxy.GeaflowInvocationHandler.invoke(GeaflowInvocationHandler.java:35) ~[geaflow-console-common-util-0.1.jar!/:?]
at com.sun.proxy.$Proxy231.submitJob(Unknown Source) ~[?:?]
at com.antgroup.geaflow.console.core.service.runtime.K8sRuntime.doStart(K8sRuntime.java:86) ~[geaflow-console-core-service-0.1.jar!/:?]
... 7 more