connect: connection refused when enable syncCatalog about consul-helm

[dongtianyi]

values.yaml

global:
  enabled: true
  domain: consul
  image: "consul:1.2.3"
  imageK8S: "hashicorp/consul-k8s:0.1.0"
  datacenter: dc1

server:
  enabled: "-"
  image: null
  replicas: 3
  bootstrapExpect: 3 # Should <= replicas count

  storage: 20Gi
  storageClass: null

  connect: true

  resources: {}

  updatePartition: 0

  disruptionBudget:
    enabled: true

    maxUnavailable: null

  extraConfig: |
    {}

  extraVolumes: []

client:
  enabled: "-"
  image: null
  join: null
  grpc: false
  resources: {}
  extraConfig: |
    {}

  extraVolumes: []

dns:
  enabled: "-"

ui:
  enabled: "-"

  service:
    enabled: true
    type: null
syncCatalog:
  enabled: true
  image: null
  toConsul: true
  toK8S: false
  k8sPrefix: null
connectInject:
  enabled: false
  image: null 
  default: false 
  imageConsul: null
  imageEnvoy: null
  namespaceSelector: null
  certs:
    secretName: null
    caBundle: ""
    certName: tls.crt
    keyName: tls.key

[dongtianyi]

consul-y-sync-catalog try to connect hostIP:8500, but I found that consul client run command

/bin/sh -ec CONSUL_FULLNAME="consul-y" exec /bin/consul agent \ -advertise="${POD_IP}" \ -bind=0.0.0.0 \ -client=0.0.0.0 \ -config-dir=/consul/config \ -datacenter=dc1 \ -data-dir=/consul/data \ -retry-join=${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \ -retry-join=${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \ -retry-join=${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \ -domain=consul

-advertise="${POD_IP}

Anything I wrong?

Thanks!!!

[dongtianyi]

Also I run helm :
helm install --name consul-y ./ --namespace=consul

(backend5) tt-939:consul-helm tt$ helm version
Client: &version.Version{SemVer:"v2.11.0", GitCommit:"2e55dbe1fdb5fdb96b75ff144a339489417b146b", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.11.0", GitCommit:"2e55dbe1fdb5fdb96b75ff144a339489417b146b", GitTreeState:"clean"}

(backend5) tt-939:consul-helm tt$ kubectl version
Client Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.2", GitCommit:"bb9ffb1654d4a729bb4cec18ff088eacc153c239", GitTreeState:"clean", BuildDate:"2018-08-08T16:31:16Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"darwin/amd64"}
Server Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.2", GitCommit:"bb9ffb1654d4a729bb4cec18ff088eacc153c239", GitTreeState:"clean", BuildDate:"2018-08-07T23:08:19Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"}

[dongtianyi]

(backend5) tt-939:consul-helm tt$ ku describe deployment consul-y-sync-catalog --namespace=consul
Name:                   consul-y-sync-catalog
Namespace:              consul
CreationTimestamp:      Thu, 06 Dec 2018 02:27:16 +0900
Labels:                 app=consul
                        chart=consul-0.1.0
                        heritage=Tiller
                        release=consul-y
Annotations:            deployment.kubernetes.io/revision=1
Selector:               app=consul,chart=consul-0.1.0,component=sync-catalog,release=consul-y
Replicas:               1 desired | 1 updated | 1 total | 0 available | 1 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:       app=consul
                chart=consul-0.1.0
                component=sync-catalog
                release=consul-y
  Annotations:  consul.hashicorp.com/connect-inject=false
  Containers:
   consul-sync-catalog:
    Image:      hashicorp/consul-k8s:0.1.0
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -ec
      consul-k8s sync-catalog \
  -http-addr=${HOST_IP}:8500 \
  -to-k8s=false \
  -consul-domain=consul \
  -k8s-write-namespace=${NAMESPACE}

    Environment:
      HOST_IP:     (v1:status.hostIP)
      NAMESPACE:   (v1:metadata.namespace)
    Mounts:       <none>
  Volumes:        <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      False   MinimumReplicasUnavailable
  Progressing    False   ProgressDeadlineExceeded
OldReplicaSets:  <none>
NewReplicaSet:   consul-y-sync-catalog-5d557fbfdf (1/1 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  24m   deployment-controller  Scaled up replica set consul-y-sync-catalog-5d557fbfdf to 1

[dongtianyi]

(backend5) tt-939:consul-helm tt$ ku describe statefulset consul-y-server --namespace=consul
Name:               consul-y-server
Namespace:          consul
CreationTimestamp:  Thu, 06 Dec 2018 02:27:16 +0900
Selector:           app=consul,chart=consul-0.1.0,component=server,hasDNS=true,release=consul-y
Labels:             app=consul
                    chart=consul-0.1.0
                    heritage=Tiller
                    release=consul-y
Annotations:        <none>
Replicas:           3 desired | 3 total
Update Strategy:    RollingUpdate
Pods Status:        3 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:       app=consul
                chart=consul-0.1.0
                component=server
                hasDNS=true
                release=consul-y
  Annotations:  consul.hashicorp.com/connect-inject=false
  Containers:
   consul:
    Image:       consul:1.2.3
    Ports:       8500/TCP, 8301/TCP, 8302/TCP, 8300/TCP, 8600/TCP, 8600/UDP
    Host Ports:  0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/UDP
    Command:
      /bin/sh
      -ec
      CONSUL_FULLNAME="consul-y"

exec /bin/consul agent \
  -advertise="${POD_IP}" \
  -bind=0.0.0.0 \
  -bootstrap-expect=3 \
  -client=0.0.0.0 \
  -config-dir=/consul/config \
  -datacenter=dc1 \
  -data-dir=/consul/data \
  -domain=consul \
  -hcl="connect { enabled = true }" \
  -ui \
  -retry-join=${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \
  -retry-join=${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \
  -retry-join=${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \
  -server

    Readiness:  exec [/bin/sh -ec curl http://127.0.0.1:8500/v1/status/leader 2>/dev/null | \
grep -E '".+"'
] delay=5s timeout=5s period=3s #success=1 #failure=2
    Environment:
      POD_IP:      (v1:status.podIP)
      NAMESPACE:   (v1:metadata.namespace)
    Mounts:
      /consul/config from config (rw)
      /consul/data from data (rw)
  Volumes:
   config:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      consul-y-server-config
    Optional:  false
Volume Claims:
  Name:          data
  StorageClass:  
  Labels:        <none>
  Annotations:   <none>
  Capacity:      20Gi
  Access Modes:  [ReadWriteOnce]
Events:
  Type    Reason            Age   From                    Message
  ----    ------            ----  ----                    -------
  Normal  SuccessfulCreate  26m   statefulset-controller  create Claim data-consul-y-server-0 Pod consul-y-server-0 in StatefulSet consul-y-server success
  Normal  SuccessfulCreate  26m   statefulset-controller  create Pod consul-y-server-0 in StatefulSet consul-y-server successful
  Normal  SuccessfulCreate  26m   statefulset-controller  create Claim data-consul-y-server-1 Pod consul-y-server-1 in StatefulSet consul-y-server success
  Normal  SuccessfulCreate  26m   statefulset-controller  create Pod consul-y-server-1 in StatefulSet consul-y-server successful
  Normal  SuccessfulCreate  26m   statefulset-controller  create Claim data-consul-y-server-2 Pod consul-y-server-2 in StatefulSet consul-y-server success
  Normal  SuccessfulCreate  26m   statefulset-controller  create Pod consul-y-server-2 in StatefulSet consul-y-server successful

[miaowing]

How to resolve it @dongtianyi, I have the same issue.

[csayler]

how did you solve did you use podIP? I have the same issue with one consul-server and no volume