Hi, I have some issues to configure the Docker flow monitor in my swarm cluster. From some reason I don't get any targets in the prometheus.
i'm not sure why the scrape_configs is not exist and why I don't have any jobs or targets under it.
here is some of my swarm docker compose which include the proxy,monitor and exporters:
version: '3.3'
volumes:
prometheus_data:
grafana_data:
swarm-endpoints:
txt_file_exporter_data:
networks:
monitor:
external: true
proxy:
external: true
prod:
external: true
configs:
alert_manager_config:
file: ./monitor/alertmanager/config.yml
blackbox_exporter_config:
file: ./monitor/blackbox-exporter/blackbox.yml
grafana_ini_config:
file: ./monitor/grafana/grafana.ini
grafana_dashboard_allhosts_config:
file: ./monitor/grafana/dashboards/monitor_all_hosts_rev1.json
grafana_dashboard_application_config:
file: ./monitor/grafana/dashboards/application_monitoring_rev1.json
grafana_dashboard_system_config:
file: ./monitor/grafana/dashboards/system_docker_monitoring_rev2.json
grafana_provisioning_dashboard_config:
file: ./monitor/grafana/provisioning/dashboards/provisioning_config_file.yml
grafana_provisioning_datasources_config:
file: ./monitor/grafana/provisioning/datasources/datasource.yml
secrets:
prometheus_scraps_config:
file: ./monitor/prometheus/scrape_swarm_prometheus.yml
services:
proxy:
image: dockerflow/docker-flow-proxy:18.07.18-74
ports:
- "80:80"
- "443:443"
#- "3001:3001"
networks:
proxy:
aliases:
- proxy
environment:
- LISTENER_ADDRESS=swarm-listener
- MODE=swarm
- DEBUG=true
deploy:
# replicas: 1
mode: global
placement:
constraints: [node.role == manager]
restart_policy:
delay: 5s
logging:
options:
max-size: 1g
swarm-listener:
image: dockerflow/docker-flow-swarm-listener:18.07.03-28
privileged: true
networks:
- proxy
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- DF_NOTIFY_CREATE_SERVICE_URL=http://proxy:8080/v1/docker-flow-proxy/reconfigure
- DF_NOTIFY_REMOVE_SERVICE_URL=http://proxy:8080/v1/docker-flow-proxy/remove
- DF_NOTIFY_CREATE_SERVICE_URL=http://monitor:8080/v1/docker-flow-monitor/reconfigure
- DF_NOTIFY_REMOVE_SERVICE_URL=http://monitor:8080/v1/docker-flow-monitor/remove
- DF_NOTIFY_CREATE_NODE_URL=http://monitor:8080/v1/docker-flow-monitor/node/reconfigure
- DF_NOTIFY_REMOVE_NODE_URL=http://monitor:8080/v1/docker-flow-monitor/node/remove
- DF_INCLUDE_NODE_IP_INFO=true
deploy:
replicas: 1
placement:
constraints: [node.role == manager]
restart_policy:
delay: 5s
logging:
options:
max-size: 1g
monitor: #This is also include prometheus
image: dockerflow/docker-flow-monitor
environment:
- LISTENER_ADDRESS=swarm-listener
- DF_GET_NODES_URL=http://swarm-listener:8080/v1/docker-flow-swarm-listener/get-nodes
- GLOBAL_SCRAPE_INTERVAL=10s
#- ARG_WEB_ROUTE-PREFIX=/monitor
- ARG_ALERTMANAGER_URL=http://alert-manager:9093
- ARG_CONFIG_FILE=/etc/prometheus/prometheus.yml
- ARG_STORAGE_TSDB_PATH=/prometheus
- ARG_STORAGE_TSDB_RETENTION=10d
- ARG_WEB_ENABLE-LIFECYCLE=
- ARG_WEB_ENABLE-ADMIN-API=
- GLOBAL__SCRAPE_INTERVAL=60s
- GLOBAL__evaluation_interval=60s
- GLOBAL__scrape_timeout=60s
- DF_SCRAPE_TARGET_LABELS=metricType,url_healthcheck
#- DF_NODE_TARGET_LABELS=aws_region,role
secrets:
- source: prometheus_scraps_config
target: /run/secrets/scrape_swarm_prometheus.yml
#uid: "0"
mode: 444
networks:
- monitor
- proxy
ports:
- 9090:9090
deploy:
replicas: 1
placement:
constraints: [node.role == manager]
restart_policy:
delay: 5s
logging:
options:
max-size: 1g
labels:
com.df.notify: 'true'
alert-manager:
image: prom/alertmanager:v0.15.2
configs:
- source: alert_manager_config
target: /etc/alertmanager/config.yml
mode: 444
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
#ports:
# - 9093:9093
networks:
- monitor
environment:
- ADMIN_USER=${ADMIN_USER:-admin}
- ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
logging:
options:
max-size: 1g
deploy:
replicas: 1
placement:
constraints: [node.role == manager]
labels:
com.df.notify: 'true'
grafana:
image: grafana/grafana:5.2.2
volumes:
- grafana_data:/var/lib/grafana:rw
configs:
- source: grafana_ini_config
target: /etc/grafana/grafana.ini
mode: 444
- source: grafana_dashboard_allhosts_config
target: /etc/grafana/dashboards/monitor_all_hosts_rev1.json
mode: 444
- source: grafana_dashboard_application_config
target: /etc/grafana/dashboards/application_monitoring_rev1.json
mode: 444
- source: grafana_dashboard_system_config
target: /etc/grafana/dashboards/system_docker_monitoring_rev2.json
mode: 444
- source: grafana_provisioning_dashboard_config
target: /etc/grafana/provisioning/dashboards/provisioning_config_file.yml
mode: 444
- source: grafana_provisioning_datasources_config
target: /etc/grafana/provisioning/datasources/datasource.yml
mode: 444
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
ports:
- 3001:3001
networks:
- monitor
- proxy
deploy:
replicas: 1
placement:
constraints: [node.role == manager]
logging:
options:
max-size: 1g
labels:
com.df.notify: 'true'
com.df.servicePath: "/monitor"
com.df.reqPathSearchReplace: "/monitor,"
com.df.port: 3001
blackbox:
image: prom/blackbox-exporter:v0.12.0
#ports:
# - "9115:9115"
networks:
- monitor
- prod
configs:
- source: blackbox_exporter_config
target: /config/blackbox.yml
mode: 444
command:
- '--config.file=/config/blackbox.yml'
- '--log.level=debug'
deploy:
replicas: 1
placement:
constraints: [node.role == manager]
resources:
limits:
cpus: '0.1'
memory: '1gb'
logging:
options:
max-size: 1g
labels:
com.df.notify: 'true'
com.df.scrapePort: 9115
com.df.scrapeNetwork: monitor
com.df.metricType: url_healthcheck
nodeexporter:
image: prom/node-exporter:v0.16.0
user: root
privileged: true
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
- /etc/hostname:/etc/host_hostname
- txt_file_exporter_data:/etc/node-exporter:ro
environment:
- HOST_HOSTNAME=/etc/host_hostname
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc|docker|tmpfs)($$|/)'
- '--collector.filesystem.ignored-fs-types=^/(aufs|cgroup|devpts|mqueue|nsfs|sysfs|proc|tmpfs|loop|shm|none|overlay)($$|/)'
- '--collector.textfile.directory=/etc/node-exporter'
restart: always
ports:
- 9100:9100
networks:
- monitor
deploy:
mode: global
restart_policy:
delay: 5s
resources:
limits:
cpus: '0.1'
memory: '1gb'
logging:
options:
max-size: 1g
labels:
com.df.notify: 'true'
com.df.scrapeNetwork: monitor
com.df.scrapePort: 9100
com.df.metricType: system
cadvisor:
image: google/cadvisor:v0.30.2
privileged: true
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
restart: always
ports:
- 9101:9101
networks:
- monitor
command:
- '--port=9101'
deploy:
mode: global
restart_policy:
delay: 5s
resources:
limits:
cpus: '0.1'
memory: '1gb'
logging:
options:
max-size: 1g
labels:
com.df.notify: 'true'
com.df.scrapeNetwork: monitor
com.df.scrapePort: 9101
com.df.metricType: system
nginx:
image: nginx
networks:
proxy:
site01:
aliases:
- nginx-site01.domain.local
volumes:
- /storage:/opt/nginx/html:ro
- /etc/localtime:/etc/localtime:ro
- /etc/timezone:/etc/timezone:ro
deploy:
mode: replicated
replicas: 1
endpoint_mode: dnsrr
placement:
constraints:
- node.labels.site==site01
restart_policy:
delay: 5s
labels:
- com.df.notify='true'
- com.df.healthurl=nginx-site01.domain.local
- com.df.scrapeNetwork=monitor
- com.df.metricType=url_healthcheck
- com.df.alertName=mem_limit
- com.df.alertIf=@service_mem_limit:0.8
- com.df.alertFor=5s
- com.df.scaleMin=2
- com.df.scaleMax=4
- com.df.port=443
- com.df.srcPort=443
- com.df.reqMode=sni
- com.df.pathType-"req.ssl_sni -i -m reg"
- com.df.servicePath="^(nginx-site01\\.)"
global:
scrape_interval: 1m
scrape_timeout: 1m
evaluation_interval: 1m
alerting:
alertmanagers:
- static_configs:
- targets:
- alert-manager:9093
scheme: http
timeout: 10s
rule_files:
- /etc/prometheus/alert.rules
docker logs monitor_monitor.1.l8ydgmdj2iyawtaa89ee35x2l
2018/08/27 11:16:51 Requesting services from Docker Flow Swarm Listener
2018/08/27 11:16:56 Processing: [{"distribute":"true","port":"8080","replicas":"1","reqPathSearchReplace":"\"/site01,\"","serviceName":"site01_app01","servicePath":"\"/site01\""},{"distribute":"true","port":"8080","replicas":"1","reqPathSearchReplace":"\"/site02,\"","serviceName":"site02_app01","servicePath":"\"/site02\""},{"distribute":"true","pathType":"req.ssl_sni -i -m reg","port":"9443","replicas":"1","reqMode":"sni","serviceName":"site01_apigateway","servicePath":"^(apigateway-site01\\.)","srcPort":"9443"},{"distribute":"true","pathType":"req.ssl_sni -i -m reg","port":"9443","replicas":"1","reqMode":"sni","serviceName":"site02_apigateway","servicePath":"^(apigateway-site02\\.)","srcPort":"9443"},{"alertFor":"5s","alertIf":"@service_mem_limit:0.8","alertName":"mem_limit","distribute":"true","healthurl":"nginx-site01.domain.local","metricType":"url_healthcheck","pathType-\"req.ssl_sni -i -m reg\"":"","port":"443","replicas":"1","reqMode":"sni","scaleMax":"4","scaleMin":"2","scrapeNetwork":"monitor","serviceName":"site01_nginx","servicePath":"\"^(nginx-site01\\\\.)\"","srcPort":"443"},{"alertFor":"5s","alertIf":"@service_mem_limit:0.8","alertName":"mem_limit","distribute":"true","healthurl":"nginx-site02.domain.local","metricType":"url_healthcheck","pathType-\"req.ssl_sni -i -m reg\"":"","port":"443","replicas":"1","reqMode":"sni","scaleMax":"4","scaleMin":"2","scrapeNetwork":"monitor","serviceName":"site02_nginx","servicePath":"\"^(nginx-site02\\\\.)\"","srcPort":"443"}]
2018/08/27 11:16:56 Requesting nodes from Docker Flow Swarm Listener
2018/08/27 11:16:56 Processing: [{"address":"10.132.0.10","availability":"active","hostname":"swarm-worker-3","id":"0abczxkaqmgvscwm7r0xafut2","role":"worker","state":"ready","versionIndex":"477591"},{"address":"10.132.0.8","availability":"active","hostname":"swarm-worker-1","id":"4a61xthy8rnd8tu08e48pabx6","role":"worker","state":"ready","versionIndex":"477591"},{"address":"0.0.0.0","availability":"active","hostname":"swarm-manager-2","id":"hn0r5pmdr2gruu6haneqo1b72","role":"manager","state":"ready","versionIndex":"477591"},{"address":"10.132.0.5","availability":"active","hostname":"swarm-manager-1","id":"qg5z99jfzvw1lbcfe7vbqfyom","role":"manager","state":"ready","versionIndex":"477591"},{"address":"10.132.0.7","availability":"active","hostname":"swarm-worker-2","id":"vnjn8uj8mptck9ik291muyqzf","role":"worker","state":"ready","versionIndex":"477591"}]
2018/08/27 11:16:56 Writing to alert.rules
2018/08/27 11:16:56 Writing to prometheus.yml
2018/08/27 11:16:56 Starting Prometheus
2018/08/27 11:16:56 /bin/sh -c prometheus --config.file="/etc/prometheus/prometheus.yml" --storage.tsdb.path="/prometheus" --storage.tsdb.retention="10d" --web.enable-lifecycle --web.console.libraries="/usr/share/prometheus/console_libraries" --web.console.templates="/usr/share/prometheus/consoles"
2018/08/27 11:16:56 Starting Docker Flow Monitor
level=info ts=2018-08-27T11:16:56.48316236Z caller=main.go:222 msg="Starting Prometheus" version="(version=2.3.2, branch=HEAD, revision=71af5e29e815795e9dd14742ee7725682fa14b7b)"
level=info ts=2018-08-27T11:16:56.483243102Z caller=main.go:223 build_context="(go=go1.10.3, user=root@5258e0bd9cc1, date=20180712-14:02:52)"
level=info ts=2018-08-27T11:16:56.483262247Z caller=main.go:224 host_details="(Linux 4.15.0-1018-gcp #19-Ubuntu SMP Thu Aug 16 13:38:55 UTC 2018 x86_64 95303c893d07 (none))"
level=info ts=2018-08-27T11:16:56.483277801Z caller=main.go:225 fd_limits="(soft=1048576, hard=1048576)"
level=info ts=2018-08-27T11:16:56.484625376Z caller=web.go:415 component=web msg="Start listening for connections" address=0.0.0.0:9090
level=info ts=2018-08-27T11:16:56.484599804Z caller=main.go:533 msg="Starting TSDB ..."
level=info ts=2018-08-27T11:16:56.490405807Z caller=main.go:543 msg="TSDB started"
level=info ts=2018-08-27T11:16:56.490472545Z caller=main.go:603 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
level=info ts=2018-08-27T11:16:56.492077907Z caller=main.go:629 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml
level=info ts=2018-08-27T11:16:56.492137538Z caller=main.go:502 msg="Server is ready to receive web requests."
docker service logs management_df-swarm-listener
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Starting Docker Flow: Swarm Listener
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Using Docker Client API version: 1.37
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending notifications for running services and nodes
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Listening to Docker Service Events
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Listening to Docker Node Events
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending node created notification to http://monitor:8080/v1/docker-flow-monitor/node/reconfigure?address=0.0.0.0&availability=active&hostname=swarm-manager-2&id=hn0r5pmdr2gruu6haneqo1b72&role=manager&state=ready&versionIndex=477482
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending node created notification to http://monitor:8080/v1/docker-flow-monitor/node/reconfigure?address=10.132.0.10&availability=active&hostname=swarm-worker-3&id=0abczxkaqmgvscwm7r0xafut2&role=worker&state=ready&versionIndex=477482
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending node created notification to http://monitor:8080/v1/docker-flow-monitor/node/reconfigure?address=10.132.0.7&availability=active&hostname=swarm-worker-2&id=vnjn8uj8mptck9ik291muyqzf&role=worker&state=ready&versionIndex=477482
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending node created notification to http://monitor:8080/v1/docker-flow-monitor/node/reconfigure?address=10.132.0.5&availability=active&hostname=swarm-manager-1&id=qg5z99jfzvw1lbcfe7vbqfyom&role=manager&state=ready&versionIndex=477482
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:28 Sending node created notification to http://monitor:8080/v1/docker-flow-monitor/node/reconfigure?address=10.132.0.8&availability=active&hostname=swarm-worker-1&id=4a61xthy8rnd8tu08e48pabx6&role=worker&state=ready&versionIndex=477482
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?distribute=true&port=8080&replicas=1&reqPathSearchReplace=%22%2Fsite02%2C%22&serviceName=site02_app01&servicePath=%22%2Fsite02%22
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?distribute=true&pathType=req.ssl_sni+-i+-m+reg&port=9443&replicas=1&reqMode=sni&serviceName=site02_apigateway&servicePath=%5E%28apigateway-site02%5C.%29&srcPort=9443
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?alertFor=5s&alertIf=%40service_mem_limit%3A0.8&alertName=mem_limit&distribute=true&healthurl=nginx-site02.domain.local&metricType=url_healthcheck&pathType-%22req.ssl_sni+-i+-m+reg%22=&port=443&replicas=1&reqMode=sni&scaleMax=4&scaleMin=2&scrapeNetwork=monitor&serviceName=site02_nginx&servicePath=%22%5E%28nginx-site02%5C%5C.%29%22&srcPort=443
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?distribute=true&port=8080&replicas=1&reqPathSearchReplace=%22%2Fsite01%2C%22&serviceName=site01_app01&servicePath=%22%2Fsite01%22
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?alertFor=5s&alertIf=%40service_mem_limit%3A0.8&alertName=mem_limit&distribute=true&healthurl=nginx-site01.domain.local&metricType=url_healthcheck&pathType-%22req.ssl_sni+-i+-m+reg%22=&port=443&replicas=1&reqMode=sni&scaleMax=4&scaleMin=2&scrapeNetwork=monitor&serviceName=site01_nginx&servicePath=%22%5E%28nginx-site01%5C%5C.%29%22&srcPort=443
management_df-swarm-listener.1.f0v58zj92u1l@swarm-manager-1 | 2018/08/27 11:06:33 Sending service created notification to http://monitor:8080/v1/docker-flow-monitor/reconfigure?distribute=true&pathType=req.ssl_sni+-i+-m+reg&port=9443&replicas=1&reqMode=sni&serviceName=site01_apigateway&servicePath=%5E%28apigateway-site01%5C.%29&srcPort=9443
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent