Reverse proxy DoH and DoT with Caddy

1. Output of caddy version:

v2.6.2 h1:wKoFIxpmOJLGl3QXoo6PNbYvGW4xLEgo32GPBEjWL8o=

2. How I run Caddy:

Docker Compose

a. System environment:

Ubuntu 22.04

b. Command:

docker compose up -d

c. Service/unit/compose file:

caddy docker-compose.yml
version: "3.9"
services:
    caddy:
        image: caddy:latest
        restart: unless-stopped
        container_name: caddy
        cap_add:
            - NET_ADMIN
            - CAP_NET_BIND_SERVICE
            - CAP_NET_RAW
        networks:
            - caddy
        ports:
            - "80:80"
            - "443:443"
            - "443:443/udp"
        volumes:
            - $HOME/Docker/caddy/Caddyfile:/etc/caddy/Caddyfile
            - $HOME/Docker/caddy/www:/srv
            - $HOME/Docker/caddy/log/:/var/log/
            - caddy_data:/data
            - caddy_config:/config

volumes:
    caddy_data:
        external: true
    caddy_config:
        external: true
adguard docker-compose.yml:
version: "3.9"
services:
  adguard:
    image: adguard/adguardhome:latest
    restart: unless-stopped
    container_name: adguard
    cap_add:
      - NET_ADMIN
      - CAP_NET_BIND_SERVICE
      - CAP_NET_RAW
    networks:
      - caddy
    ports:
      - "53:53"
      - "53:53/udp"
      - "853:853"
      - "853:853/udp"
      - "1443:443"
      - "1443:443/udp"
      - "3000:3000"
    volumes:
      - adguard_data:/opt/adguardhome/work
      - adguard_config:/opt/adguardhome/conf
      - /var/lib/docker/volumes/caddy_data/_data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/:/etc/ssl/certs/:ro

volumes:
  adguard_data:
    external: true
  adguard_config:
    external: true

networks:
  caddy:
    external: true

adguard config.yml:
bind_host: 0.0.0.0
bind_port: 3000
beta_bind_port: 0
users:
  - name: admin
    password: mypass
auth_attempts: 5
block_auth_min: 15
http_proxy: ""
language: ""
debug_pprof: false
web_session_ttl: 720
dns:
  bind_hosts:
    - 0.0.0.0
  port: 53
  statistics_interval: 10
  querylog_enabled: true
  querylog_file_enabled: true
  querylog_interval: 2160h
  querylog_size_memory: 1000
  anonymize_client_ip: false
  protection_enabled: true
  blocking_mode: default
  blocking_ipv4: ""
  blocking_ipv6: ""
  blocked_response_ttl: 10
  parental_block_host: family-block.dns.adguard.com
  safebrowsing_block_host: standard-block.dns.adguard.com
  ratelimit: 20
  ratelimit_whitelist: []
  refuse_any: true
  upstream_dns:
    - quic://family.adguard-dns.com
    - tls://family.adguard-dns.com
    - tls://family.cloudflare-dns.com
    - https://family.cloudflare-dns.com/dns-query
  upstream_dns_file: ""
  bootstrap_dns:
    - 9.9.9.10
    - 149.112.112.10
    - 2620:fe::10
    - 2620:fe::fe:10
  all_servers: false
  fastest_addr: false
  fastest_timeout: 1s
  allowed_clients: []
  disallowed_clients: []
  blocked_hosts:
    - version.bind
    - id.server
    - hostname.bind
  trusted_proxies:
    - 127.0.0.0/8
    - ::1/128
  cache_size: 0
  cache_ttl_min: 0
  cache_ttl_max: 0
  cache_optimistic: false
  bogus_nxdomain: []
  aaaa_disabled: false
  enable_dnssec: true
  edns_client_subnet: false
  max_goroutines: 300
  handle_ddr: true
  ipset: []
  ipset_file: ""
  filtering_enabled: true
  filters_update_interval: 24
  parental_enabled: true
  safesearch_enabled: false
  safebrowsing_enabled: true
  safebrowsing_cache_size: 1048576
  safesearch_cache_size: 1048576
  parental_cache_size: 1048576
  cache_time: 30
  rewrites: []
  blocked_services: []
  upstream_timeout: 10s
  private_networks: []
  use_private_ptr_resolvers: true
  local_ptr_upstreams: []
  serve_http3: true
  use_http3_upstreams: true
tls:
  enabled: true
  server_name: "example.com"
  force_https: false
  port_https: 443
  port_dns_over_tls: 853
  port_dns_over_quic: 853
  port_dnscrypt: 0
  dnscrypt_config_file: ""
  allow_unencrypted_doh: false
  strict_sni_check: false
  certificate_chain: ""
  private_key: ""
  certificate_path: "/etc/ssl/certs/example.com/example.com.crt"
  private_key_path: "/etc/ssl/certs/example.com/example.com.key"
filters:
  - enabled: true
    url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
    name: AdGuard DNS filter
    id: 1
  - enabled: true
    url: https://adaway.org/hosts.txt
    name: AdAway Default Blocklist
    id: 2
whitelist_filters: []
user_rules: []
dhcp:
  enabled: false
  interface_name: ""
  local_domain_name: lan
  dhcpv4:
    gateway_ip: ""
    subnet_mask: ""
    range_start: ""
    range_end: ""
    lease_duration: 86400
    icmp_timeout_msec: 1000
    options: []
  dhcpv6:
    range_start: ""
    lease_duration: 86400
    ra_slaac_only: false
    ra_allow_slaac: false
clients:
  runtime_sources:
    whois: true
    arp: true
    rdns: true
    dhcp: true
    hosts: true
  persistent: []
log_file: ""
log_max_backups: 0
log_max_size: 10
log_max_age: 3
log_compress: false
log_localtime: false
verbose: false
os:
  group: ""
  user: ""
  rlimit_nofile: 0
schema_version: 14

d. My complete Caddy config:

{
	log {
		level DEBUG
		output file /var/log/caddy.log {
			roll_size 10MiB
			roll_keep_for 3d
		}
	}

	# TLS global settings
	email myemail@example.com
}

example.com {
	tls {
		issuer acme
		issuer zerossl
	}
	reverse_proxy /dns-query* adguard:1443
}

3. The problem I’m having:

I’m trying to run AdGuard Home Docker container behind Caddy to serve DoH and DoT requests, it works perfectly without Caddy and when it’s set to port 443 but when I try to reverse proxy using Caddy to a custom port (1443 since Caddy is listening to 443 obviously!) I get 400 HTTP error code in my DNS client!

4. Error messages and/or full log output:

Here’s the Caddy log when a request is made:

{"level":"debug","ts":1667697363.2261975,"logger":"tls.handshake","msg":"choosing certificate","identifier":"example.com","num_choices":1}
{"level":"debug","ts":1667697363.2262568,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"example.com","subjects":["example.com"],"managed":true,"issuer_key":"acme-v02.api.letsencrypt.org-directory","hash":"2140613c047495444e3f1053b4c0da8f2188ac16d8ce1ab4a1a21b9a8bac3768"}
{"level":"debug","ts":1667697363.2262878,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"x.x.x.x","remote_port":"60232","subjects":["example.com"],"managed":true,"expiration":1675460687,"hash":"2140613c047495444e3f1053b4c0da8f2188ac16d8ce1ab4a1a21b9a8bac3768"}
{"level":"debug","ts":1667697363.3468044,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"adguard:1443","total_upstreams":1}
{"level":"debug","ts":1667697363.3499403,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"adguard:1443","duration":0.003019613,"request":{"remote_ip":"x.x.x.x","remote_port":"60232","proto":"HTTP/2.0","method":"GET","host":"example.com:443","uri":"/dns-query?dns=AAABIAABAAAAAAABCmNsb3VkZmxhcmUDY29tAAABAAEAACkE0AAAAAAADAAKAAi1YqhuIeuOJQ","headers":{"X-Forwarded-Host":["example.com:443"],"Accept":["application/dns-message"],"User-Agent":[""],"X-Forwarded-For":["x.x.x.x"],"X-Forwarded-Proto":["https"]},"tls":{"resumed":false,"version":772,"cipher_suite":4867,"proto":"h2","server_name":"example.com"}},"headers":{},"status":400}

Here’s the client log:

$ dig +short @127.0.0.1 cloudflare.com A
2022/11/06 04:46:03 279054#17 [debug] github.com/AdguardTeam/dnsproxy/upstream.lookup(): successfully finished lookup for example.com in 485.923451ms using 8.8.8.8:53. Result : [{x.x.x.x }]
2022/11/06 04:46:03 279054#17 [debug] using HTTP/2 for this upstream: HTTP3 support is not enabled
2022/11/06 04:46:03 279054#17 [debug] https://example.com:443/dns-query: sending request A cloudflare.com.
2022/11/06 04:46:03 279054#21 [debug] github.com/AdguardTeam/dnsproxy/upstream.(*bootstrapper).createDialContext.func1(): Dialing to x.x.x.x:443
2022/11/06 04:46:03 279054#21 [debug] github.com/AdguardTeam/dnsproxy/upstream.(*bootstrapper).createDialContext.func1(): dialer has successfully initialized connection to x.x.x.x:443 in 95.20894ms
2022/11/06 04:46:03 279054#17 [debug] https://example.com:443/dns-query: response: expected status 200, got 400 from https://example.com:443/dns-query
2022/11/06 04:46:03 279054#17 [debug] re-creating the http client due to expected status 200, got 400 from https://example.com:443/dns-query
2022/11/06 04:46:03 279054#17 [debug] using HTTP/2 for this upstream: HTTP3 support is not enabled
2022/11/06 04:46:03 279054#17 [debug] github.com/AdguardTeam/dnsproxy/proxy.exchangeWithUpstream(): upstream https://example.com:443/dns-query failed to exchange ;cloudflare.com.	IN	 A in 805.853048ms. Cause: expected status 200, got 400 from https://example.com:443/dns-query
2022/11/06 04:46:03 279054#17 [debug] github.com/AdguardTeam/dnsproxy/proxy.(*Proxy).replyFromUpstream(): RTT: 806.07772ms
2022/11/06 04:46:03 279054#17 [debug] github.com/AdguardTeam/dnsproxy/proxy.(*Proxy).logDNSMessage(): OUT: ;; opcode: QUERY, status: SERVFAIL, id: 60012

5. What I already tried:

I have tried substituting reverse_proxy /dns-query* adguard:1443 to reverse_proxy /dns-query* https://example.com:1443 in my Caddyfile but it gives me i/o timeout.

This is all for DoH and I have no idea what to do for DoT! Is Caddy even able to listen to port :853 and proxy it?

6. Links to relevant resources:

NGINX example here

Caddy is an HTTP server and proxy. DoT is TCP, so it cannot be proxied with the standard Caddy distribution.

But you could use GitHub - mholt/caddy-l4: Layer 4 (TCP/UDP) app for Caddy to proxy TCP or UDP. Harder to use though, no official Caddyfile support at the moment.

I couldn’t say re DoH, I don’t know much about the protocol, so I’m not sure what to suggest. Hopefully someone else can answer that.

2 Likes

For DoH, I see your Adguard Home is configured with certs and allow_unencrypted_doh: false, which means it expects clients to reach it with TLS-based connection, including Caddy. If that’s your intent, then your reverse_proxy directive in Caddy should have https:// for the upstream address.

For DoT, I’m not sure how Adguard Home works, but you need to proxy to a DNS-over-TCP server. I use CoreDNS which supports that, so check if Adguard Home has that feature as well. If it does, you need to use caddy-l4, as @francislavoie said. You have to use TCP upstream, otherwise downstream will see weird writes due to TCP-vs-UDP interaction and the middle-proxy isn’t actually in business of converting protocols.

2 Likes

Thank you @Mohammed90 and @francislavoie for your help I managed to get it working using @matt 's L4 plugin.

For future time-travellers, here’s my caddy.json and docker-compose.yml files in case you’re interested:

caddy.json:
{
  "apps": {
    "tls": {
      "certificates": {
        "automate": [
          "example.com"
        ]
      }
    },
    "layer4": {
      "servers": {
        "doh": {
          "listen": [
            ":443"
          ],
          "routes": [
            {
              "match": [
                {
                  "tls": {
                    "sni": [
                      "example.com"
                    ]
                  }
                }
              ],
              "handle": [
                {
                  "handler": "proxy",
                  "upstreams": [
                    {
                      "dial": [
                        "adguard:443"
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        "dot": {
          "listen": [
            ":853"
          ],
          "routes": [
            {
              "match": [
                {
                  "tls": {
                    "sni": [
                      "example.com"
                    ]
                  }
                }
              ],
              "handle": [
                {
                  "handler": "proxy",
                  "upstreams": [
                    {
                      "dial": [
                        "adguard:853"
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        }
      }
    }
  }
}

Caddy docker-compose.yml:
version: "3.9"
services:
  caddy:
    image: 0xlem0nade/caddy:latest  # My custom built Caddy baked with l4 and a host of other plugins, you can substitute with any image that has l4
    restart: unless-stopped
    container_name: caddy
    cap_add:
      - NET_ADMIN
      - CAP_NET_BIND_SERVICE
      - CAP_NET_RAW
    networks:
      - caddy
    ports:
      - "80:80"
      - "443:443"
      - "443:443/udp"
      - "853:853"
      - "853:853/udp"
    volumes:
      - ./etc/caddy.json:/etc/caddy/caddy.json
      - ./www:/srv
      - /var/log:/var/log
      - caddy_data:/data
      - caddy_config:/config
    command: caddy run --config /etc/caddy/caddy.json

volumes:
  caddy_data:
    external: true
  caddy_config:

networks:
  caddy:
    external: true

AdGuardHome docker-compose.yml:
version: "3.9"
services:
  adguard:
    image: adguard/adguardhome:latest
    restart: unless-stopped
    container_name: adguard
    cap_add:
      - NET_ADMIN
      - CAP_NET_BIND_SERVICE
      - CAP_NET_RAW
    networks:
      - caddy
    expose:   # 'expose' instead of 'ports' to keep adguard away from the outside world and port conflicts!
      - 53
      - "53/udp"
      - 443
      - "443/udp"
      - 853
      - "853/udp"
    dns:
      - 1.1.1.1
      - 9.9.9.9
    volumes:
      - caddy_data:/etc/letsencrypt:ro   # Access Caddy-generated certs and add them to AdGuard so that TLS passes through directly to AdGuard
      - adguard_data:/opt/adguardhome/work
      - adguard_config:/opt/adguardhome/conf

volumes:
  adguard_data:
    external: true
  adguard_config:
    external: true
  caddy_data:
    external: true

networks:
  caddy:
    external: true

AdGuardHome.yml config:
# REDACTED TO THE RELEVANT PARTS!
bind_host: 0.0.0.0
dns:
  bind_hosts:
    - 0.0.0.0
  port: 53
tls:
  enabled: true
  server_name: "example.com"
  force_https: false
  port_https: 443
  port_dns_over_tls: 853
  port_dns_over_quic: 853
  allow_unencrypted_doh: false
  certificate_path: "/etc/letsencrypt/caddy/certificates/acme-v02.api.letsencrypt.org-directory/example.com/example.com.crt"
  private_key_path: "/etc/letsencrypt/caddy/certificates/acme-v02.api.letsencrypt.org-directory/example.com/example.com.key"

There was one quirk or maybe because I didn’t know how to make it work, and that was I had to add another domain to my server for this purpose and implement SNI based routing, because I run other apps on the server and I wanted to keep the domain and filter only the /dns-query to AdGuard and other paths to my other apps (although that would break TLS passthrough), saw this comment here from matt but didn’t find any examples!

Path-based routing, DNS-over-QUIC, and HTTP3 are the only things, this setup left me wishing for, other than that pretty nifty! I hope future versions bring them along!

Cheers! :wink:

3 Likes

This topic was automatically closed after 30 days. New replies are no longer allowed.