502 gateway error on stack deploy (docker swarm) for brief period

1. Caddy version:

v2.6.2 h1:wKoFIxpmOJLGl3QXoo6PNbYvGW4xLEgo32GPBEjWL8o=

2. How I installed, and run Caddy:

Caddy Docker Proxy GitHub - lucaslorentz/caddy-docker-proxy: Caddy as a reverse proxy for Docker

a. System environment:

Docker ubutnu 20.10.22

b. Command:

I run via docker swarm (distributed) stack deploy

c. Service/unit/compose file:

version: '3.9'
services:
  production_healthcheck:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: "http://"
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/production-healthcheck
    build:
      context: "production_healthcheck"
    restart: always
    networks:
      - app_network
      - mail
  production_php_point_of_sale_app:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: "http://*.phppointofsale.com, http://*.phppos.com"
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/production-app
    build:
      context: "production_php_point_of_sale_app"
    restart: always
    env_file:
      - production_php_point_of_sale_app/.env
      - .env
    networks:
      - app_network
      - mail
  production_php_point_of_sale_app_customs:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://custom.phppointofsale.com, http://custom.phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/production-app-custom
    build:
      context: "production_php_point_of_sale_app_customs"
    restart: always
    networks:
      - app_network
      - mail
  production_php_point_of_sale_feedback_website:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://feedback.phppointofsale.com, http://feedback.phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/feedback-website
    build:
      context: "production_php_point_of_sale_feedback_website"
    restart: always
    networks:
      - app_network
      - mail
  production_php_point_of_sale_help_website:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://help.phppointofsale.com, http://help.phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/help-website
    build:
      context: "production_php_point_of_sale_help_website"
    restart: always
    networks:
      - app_network
      - mail
  production_php_point_of_sale_website:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://www.phppointofsale.com, http://www.phppos.com, http://phppointofsale.com, http://phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/production-website
    build:
      context: "production_php_point_of_sale_website"
    restart: always
    env_file: .env
    networks:
      - app_network
      - mail
  staging_php_point_of_sale_app:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: "http://*.phppointofsalestaging.com"
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/staging-app
    build:
      context: "staging_php_point_of_sale_app"
    restart: always
    env_file:
      - staging_php_point_of_sale_app/.env
      - .env
    networks:
      - app_network
      - mail
  staging_php_point_of_sale_website:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://www.phppointofsalestaging.com, http://phppointofsalestaging.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/staging-website
    build:
      context: "staging_php_point_of_sale_website"
    restart: always
    env_file: .env
    networks:
      - app_network
      - mail
  zatca_sdk_api:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://zatca.phppointofsale.com, http://zatca.phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    image: phppointofsale/zatca-sdk-api
    build:
      context: "zatca_sdk_api"
    restart: always
    networks:
      - app_network
      - mail
  newrelic_daemon:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
    image: phppointofsale/newrelic-php-daemon
    restart: always
    build:
      context: "newrelic_daemon"
    networks:
      - app_network
  agent:
    logging:
      driver: "local"
    deploy:
      restart_policy:
        condition: any
      mode: global
    image: phppointofsale/newrelic-agent
    build:
      context: "newrelic_infra"
    cap_add:
      - SYS_PTRACE
    network_mode: host
    pid: host
    privileged: true
    volumes:
      - "/:/host:ro"
      - "/var/run/docker.sock:/var/run/docker.sock"
    restart: unless-stopped

  caddy_server:
    image: lucaslorentz/caddy-docker-proxy:ci-alpine
    ports:
      - 80:80
    networks:
      - caddy_controller
      - app_network
    environment:
      - CADDY_DOCKER_MODE=server
      - CADDY_CONTROLLER_NETWORK=10.200.200.0/24
    volumes:
      - caddy_data:/data
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy_controlled_server:

  caddy_controller:
    image: lucaslorentz/caddy-docker-proxy:ci-alpine
    networks:
      - caddy_controller
      - app_network
    environment:
      - CADDY_DOCKER_MODE=controller
      - CADDY_CONTROLLER_NETWORK=10.200.200.0/24
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    deploy:
      restart_policy:
        condition: any
      placement:
        constraints: [node.role == manager]
  postfix:
    logging:
      driver: "local"
    build:
      context: "postfix"
    deploy:
      restart_policy:
        condition: any
      mode: global
    image: phppointofsale/postfix
    networks:
      - mail
    hostname: postfix
    restart: always
  background_jobs_us_east_1:
    logging:
      driver: "local"
    image: phppointofsale/job-scheduler-us-east-1
    deploy:
      restart_policy:
        condition: any
      placement:
        constraints: [node.hostname == docker-php-pos-web-4]
    restart: always
    env_file: .env
    build:
      context: "job_scheduler"
      args:
          CRONFILE: crontab-us-east-1
  php_my_admin_us_east_1:
    logging:
      driver: "local"
    image: phppointofsale/phpmyadmin-us-east-1
    deploy:
      restart_policy:
        condition: any
      mode: global
      labels:
        caddy: http://mysql.phppointofsale.com, http://mysql.phppos.com
        caddy.reverse_proxy.trusted_proxies: "private_ranges"
        caddy.reverse_proxy: "{{upstreams}}"
    restart: always
    build:
      context: "phpmyadmin-us-east-1"
    networks:
      - app_network
      - mail
  pt-kill-db-1:
    logging:
      driver: "local"
    image: phppointofsale/percona-toolkit
    deploy:
      restart_policy:
        condition: any
      placement:
        constraints: [node.hostname == docker-php-pos-web-4]
    restart: always
    build:
      context: "percona-toolkit" 
    command: /usr/bin/pt-kill --rds --match-command Query --victims all --match-user phppoint --busy-time 30 --kill --print h=php-pos-db.phppointofsale.com,u=master,p=DxPdPt4aACfZr9euHNrEfnuWte3GFz32MKCa,P=3306
  pt-kill-db-2:
    logging:
      driver: "local"
    image: phppointofsale/percona-toolkit
    deploy:
      restart_policy:
        condition: any
      placement:
        constraints: [node.hostname == docker-php-pos-web-4]
    restart: always
    build:
      context: "percona-toolkit"  
    command: /usr/bin/pt-kill --rds --match-command Query --victims all --match-user phppoint --busy-time 30 --kill --print h=php-pos-db-2.phppointofsale.com,u=master,p=DxPdPt4aACfZr9euHNrEfnuWte3GFz32MKCa,P=3306
networks:
  caddy_controller:
    driver: overlay
    ipam:
      driver: default
      config:
        - subnet: "10.200.200.0/24"
  app_network:
        driver: overlay
  mail:
      driver: overlay
volumes:
  caddy_data: {}

d. My complete Caddy config:

{
  "admin": {
    "listen": "tcp/10.200.200.15:2019"
  },
  "apps": {
    "http": {
      "servers": {
        "srv0": {
          "listen": [
            ":80"
          ],
          "routes": [
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.115:80"
                            },
                            {
                              "dial": "10.0.12.135:80"
                            },
                            {
                              "dial": "10.0.12.121:80"
                            },
                            {
                              "dial": "10.0.12.137:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "www.phppointofsalestaging.com",
                    "phppointofsalestaging.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.127:80"
                            },
                            {
                              "dial": "10.0.12.116:80"
                            },
                            {
                              "dial": "10.0.12.128:80"
                            },
                            {
                              "dial": "10.0.12.122:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "feedback.phppointofsale.com",
                    "feedback.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.141:80"
                            },
                            {
                              "dial": "10.0.12.118:80"
                            },
                            {
                              "dial": "10.0.12.142:80"
                            },
                            {
                              "dial": "10.0.12.124:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "custom.phppointofsale.com",
                    "custom.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.89:80"
                            },
                            {
                              "dial": "10.0.12.75:80"
                            },
                            {
                              "dial": "10.0.12.14:80"
                            },
                            {
                              "dial": "10.0.12.77:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "mysql.phppointofsale.com",
                    "mysql.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.132:80"
                            },
                            {
                              "dial": "10.0.12.108:80"
                            },
                            {
                              "dial": "10.0.12.131:80"
                            },
                            {
                              "dial": "10.0.12.112:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "zatca.phppointofsale.com",
                    "zatca.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.109:80"
                            },
                            {
                              "dial": "10.0.12.113:80"
                            },
                            {
                              "dial": "10.0.12.126:80"
                            },
                            {
                              "dial": "10.0.12.120:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "help.phppointofsale.com",
                    "help.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.134:80"
                            },
                            {
                              "dial": "10.0.12.117:80"
                            },
                            {
                              "dial": "10.0.12.140:80"
                            },
                            {
                              "dial": "10.0.12.123:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "www.phppointofsale.com",
                    "www.phppos.com",
                    "phppointofsale.com",
                    "phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.107:80"
                            },
                            {
                              "dial": "10.0.12.133:80"
                            },
                            {
                              "dial": "10.0.12.139:80"
                            },
                            {
                              "dial": "10.0.12.111:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "*.phppointofsalestaging.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.119:80"
                            },
                            {
                              "dial": "10.0.12.136:80"
                            },
                            {
                              "dial": "10.0.12.138:80"
                            },
                            {
                              "dial": "10.0.12.110:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "match": [
                {
                  "host": [
                    "*.phppointofsale.com",
                    "*.phppos.com"
                  ]
                }
              ],
              "terminal": true
            },
            {
              "handle": [
                {
                  "handler": "subroute",
                  "routes": [
                    {
                      "handle": [
                        {
                          "handler": "reverse_proxy",
                          "trusted_proxies": [
                            "192.168.0.0/16",
                            "172.16.0.0/12",
                            "10.0.0.0/8",
                            "127.0.0.1/8",
                            "fd00::/8",
                            "::1"
                          ],
                          "upstreams": [
                            {
                              "dial": "10.0.12.130:80"
                            },
                            {
                              "dial": "10.0.12.125:80"
                            },
                            {
                              "dial": "10.0.12.114:80"
                            },
                            {
                              "dial": "10.0.12.129:80"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ],
              "terminal": true
            }
          ]
        }
      }
    }
  }
}

3. The problem I’m having:

When I run docker the below and application changed

docker --context php-pos-web-1 stack deploy -c docker-compose-us-east-1.yml phppointofsale --with-registry-auth

(I have 3 managers and 1 worker in my swarm)

there is some very brief time for a 502 error while stack is updating. (it is a cloudflare page 502 gateway error)

4. Error messages and/or full log output:

Don’t have this

5. What I already tried:

I looked at:

But I cannot figure out how to apply this (server or controller) and if this will even help.

6. Links to relevant resources:

None.

You can either configure health checks, or tell caddy-docker-proxy to proxy to the swarm service load balancer (instead of each individual task/container).

For the latter, add CADDY_DOCKER_PROXY_SERVICE_TASKS=false to your controllers’ environment vars.
The swarm load balancer updates its routing faster than Caddy, so you shouldn’t experience any http/502 that way.

And if you decide to go with health checks instead, have a look at reverse_proxy (Caddyfile directive) — Caddy Documentation, as the issue you linked to is pretty old and uses by now long deprecated syntax :eyes:

If it proxied to swarm load balancer how does it then know service to go to?

Which method do you recommend?

I was reading at GitHub - lucaslorentz/caddy-docker-proxy: Caddy as a reverse proxy for Docker. I am using deploy right now so I fear that it won’t be able to route

Thank you for what you have posted. I just want sure I exactly understand what CADDY_DOCKER_PROXY_SERVICE_TASKS false is.

See the docs for {{upstreams}}

Returns all addresses for the current Docker resource separated by whitespace.

For services, that would be the service DNS name when proxy-service-tasks is false, or all running tasks IPs when proxy-service-tasks is true.

It becomes swarm’s job to route the traffic to the appropriate container. Swarm knows which containers are for which service, when there’s many replicas of the same service. It’ll pick one according to your swarm config.

I second this – take a look at lb_try_duration, you can delay/suppress the 502 errors by having Caddy retry connecting a few times before giving up.

I think the best bet for me is lb_try_duration of 3 seconds. Where do I put this config? On caddy controller or caddy server? Could you give example of how to set?

Would this be your recommendation also?

It’s Caddyfile config. So you use labels.

I can’t recommend anything in particular, honestly. You’ll have to figure out what works best for you. Your setup is much more complex than I would personally go for, but that’s because my own requirements are probably much different than yours.

Would I do this for each service? Can I do this as global option? If global option where do I put?

Can you show me how to define as label? Would love it I can make global somehow

These are your labels.

See the CDP readme, which explains how to define labels.

Yes, for each service – it’s a reverse_proxy subdirective, similarly to trusted_proxies. Not global.

Thank you! Just to confirm the behavior.

if I change this to five seconds it will retry to connect every 250 ms until it reaches a total of five seconds and the usual just be waiting in the browser. If it reaches five seconds, will have a 502 Gateway error. Otherwise it will connect to backend without the user knowing

Not quite, it will have a 250ms delay between attempts. That means that it waits until the connection was deemed to be a failure before trying again. It depends what type of connection error happens. The default dial_timeout is set to 3 seconds, so it could take 3 seconds to be considered a failed attempt. So with a try duration of 5 seconds, that means there would only be one retry (i.e. after 3s + 250ms the second attempt happens, then 3s later it’s past the 5s deadline so it gives up without trying a third time).

When the error was happening, it would happen very fast to 502 Gateway error(no delay). If this is sent back faster than three seconds, would it try again after 250 ms? The dial time out seems to be some sort of DNS related things based on what I can tell from the documentation.

So I tried without timeout of 5s and it will did a gateway timeout. I am going to try CADDY_DOCKER_PROXY_SERVICE_TASKS false, but I really don’t understand this

So when this is false my caddy file will use DNS names instead of IPs? Do you see any issues with this or anyting else I should know?

I did verify that is the change to below.

it has something like this (I tried on a different swarm just to see). (before it had IP address)

“dial”: “phppointofsale_php_my_admin_us_east_2:80”

So this appears to be the DNS name of the container. How is this different than using IP address? Does the dns name cause it to go though swarm load balancer? Once it does this, does the swarm load balancer know what is healthy for this service and route?

Before did it just go directly to the service on the instance it is on?

CADDY_DOCKER_PROXY_SERVICE_TASKS false seems to work perfectly! Thank you so mcuh. When you have time, can you check to make sure my understanding is correct?

1 Like

This topic was automatically closed 30 days after the last reply. New replies are no longer allowed.