Caddy & Caching (Souin)

1. Output of caddy version:

Latest

2. How I run Caddy:

Docker image (latest)

a. System environment:

Docker (latest)
Portainer (latest)

b. Command:

no command... just run Stack!

c. Service/unit/compose file:

version: "4.0"

#
# 2022-12-01
# caddy
#

services:

  olric:
    container_name: olric
    hostname: olric
    image: olricio/olricd:latest
    restart: always
    stdin_open: true
    tty: true
    networks:
      - proxy
    ports:
      - "3320:3320"
    expose:
      - "3320"
    environment:
      TZ: "Europe/Paris"
    volumes:
      - /etc/timezone:/etc/timezone:ro
      - /etc/localtime:/etc/localtime:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /opt/docker/standard/ssl/:/ssl/:ro
      - /opt/docker/standard/notification:/notify:ro

  caddy:
    container_name: caddy
    hostname: caddy
    image: zogg/caddy:latest
    restart: always
    stdin_open: true
    tty: true
    depends_on:
      - olric    
    networks:
      - proxy
    ports:
      - "80:80"
      - "443:443"
    expose:
      - "80"
      - "443"
    environment:
      TZ: "Europe/Paris"
      CF_API_EMAIL: [...]
      CF_DNS_API_TOKEN: "[...]"
    volumes:
      - /etc/timezone:/etc/timezone:ro
      - /etc/localtime:/etc/localtime:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /opt/docker/standard/ssl/:/ssl/:ro
      - /opt/docker/standard/notification:/notify:ro
      - /opt/docker/standard/caddy/config/Caddyfile:/etc/caddy/Caddyfile
      - /opt/docker/standard/caddy/config/entries:/etc/caddy/entries
      - /opt/docker/standard/caddy/config/json:/config
      - /opt/docker/standard/caddy/work:/data

networks:
  proxy:
    external: true

d. My complete Caddy config:

# 2022-12-01

(logs) {
	debug
	log {
		level debug
		#level error
	}
}

(badger) {
	badger {
		configuration {
			Dir "/caches/badger/"
			ValueDir "/caches/badger/"
			ValueLogFileSize 1073741824
			MemTableSize 4194304
			ValueThreshold 1
			BypassLockGuard true
		}
	}
}

(olric) {
	olric {
		url olric:3320
		configuration {
			Dir "/caches/olric/"
			EntryIdxMode 1
			RWMode 0
			SegmentSize 1024
			NodeNum 42
			SyncEnable true
			StartFileLoadingMode 1
		}
	}
}

(souin) {
	log_level debug

	allowed_http_verbs GET POST

	api {
		prometheus
		souin
	}

	#cdn {
	#	api_key {env.CF_DNS_API_TOKEN}
	#	dynamic
	#	email {env.CF_API_EMAIL}
	#	hostname zogg.fr
	#	provider cloudflare
	#	strategy soft
	#}

	headers Content-Type Authorization

	ttl 300s

	timeout {
		backend 60s
		cache 300ms
	}

  	#import badger
  	import olric

	default_cache_control "public, max-age=86400, s-maxage=86400, max-stale=3600, stale-while-revalidate=86400, stale-if-error=86400"
}

(cache) {
	order cache before rewrite
	cache {
		import souin
	}
}

{
	import logs
	import cache

	http_port 80
	https_port 443

	email {env.CF_API_EMAIL}

	servers :443 {
		protocols h1 h2 h3
		listener_wrappers {
			http_redirect
			tls
		}
	}

	servers :80 {
		protocols h1 h2 h3
	}
}

# Cloudflare
(trustedproxies) {
	trusted_proxies 10.0.0.0/8 172.16.0.0/16 192.168.0.0/16 fc00::/7 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22
}
(cloudflare) {
	tls {
		dns cloudflare {env.CF_DNS_API_TOKEN}
		resolvers 1.1.1.1 1.0.0.1
	}

	header {
		header_up Host {upstream_hostport}
		header_up X-Forwarded-For {http.request.header.CF-Connecting-IP}
		header_up X-Forwarded-Proto {scheme}
		header_up X-Forwarded-For {remote}
		header_up X-Real-IP {remote}
	}
}

(proxy) {
	import trustedproxies

	#header_up Host {upstream_hostport}
	header_up Host {host}
	
	header_up X-Forwarded-Host {remote}
	header_up X-Forwarded-Proto {scheme}
	header_up X-Real-IP {remote}
	header_up X-Forwarded-For {http.request.header.CF-Connecting-IP}
	#header_up X-Forwarded-For {remote}

	header_up Cache-Control "public, max-age=86400, s-maxage=86400, max-stale=3600, stale-while-revalidate=86400, stale-if-error=86400"
	header_down Cache-Control "public, max-age=86400, s-maxage=86400, max-stale=3600, stale-while-revalidate=86400, stale-if-error=86400"

	header_up X-Powered-By "Zogg"
	header_up Server "Caddy"
}

# Global headers
(headersGlobal) {
	Server "Caddy"
	X-Powered-By "Zogg"
	X-Server "Caddy"
}

# Security headers
(headersSecurity) {
	# Keep referrer data off of HTTP connections
	Referrer-Policy "strict-origin-when-cross-origin"

	# Enable HSTS
	Strict-Transport-Security "max-age=63072000, includeSubDomains, preload"
	X-Permitted-Cross-Domain-Policies: "none"

	# Disable clients from sniffing the media type
	X-Content-Type-Options "nosniff"

	# Clickjacking protection
	X-Frame-Options "SAMEORIGIN"

	# Disable XSS protection
	X-XSS-Protection 0

	# Permissions
	Permissions-Policy "vibrate=(self), geolocation=(self), midi=(self), notifications=(self), push=(self), microphone=(self), camera=(self), magnetometer=(self), gyroscope=(self), fullscreen=(self), payment=(self)"

	# CSP
	Content-Security-Policy "default-src 'self' *.zogg.fr data: wss: blob: https:; script-src 'self' 'unsafe-eval' 'unsafe-inline' *.zogg.fr data: blob: https:; img-src 'self' data: blob: *.zogg.Fr https:;style-src 'self' 'unsafe-inline' *.zogg.fr https:; connect-src 'self' wss: *.zogg.fr https:; frame-src 'self' https:; frame-ancestors 'self' *.zogg.fr"
}

(headersRobots) {
	# Robots
	X-Robots-Tag "none,noarchive,nosnippet,notranslate,noimageindex"
}

(headersCaching) {
	# Caching
	Cache-Control "public, max-age=86400, s-maxage=86400, max-stale=3600, stale-while-revalidate=86400, stale-if-error=86400"
	header_up Cache-Control "public, max-age=86400, s-maxage=86400, max-stale=3600, stale-while-revalidate=86400, stale-if-error=86400"
}

(headersHTTP3) {
	# HTTP/3
	Alt-Svc "h3=":443"; ma=86400, h3-29=":443"; ma=86400"
}

(headers) {
	header {
		import headersGlobal
		import headersRobots
		import headersCaching
		import headersSecurity
	}
}

(pterodadctyl) {
	import cloudflare
	header {
		import headersGlobal
		import headersRobots
		import headersCaching
		Sec-Fetch-Site "cross-site"
		X-Forwarded-Proto "https"
		Access-Control-Allow-Headers "*, Authorization"
	}
}

(common) {
	import cloudflare
	import headers
}

import /etc/caddy/entries

/etc/caddy/entries :

# 2022-12-01

error.zogg.fr {
	import common
	cache
	reverse_proxy [ip]:[port] {
		import proxy
	}
}

3. The problem I’m having:

Hello :slight_smile:

I’m new to Caddy as I switched from Traefik to Caddy yesterday…

What I use:

  • Caddy (home made Docker image with cloudflare-dns & Souin Caddy plugin)
  • Olric (as cache backend for Souin)

Almost is done except I can’t cache some content.

The files I use are:

  • docker_compose.yml
  • Caddyfile
  • entries (here is only a sample ^^)

Url to test: https://error.zogg.fr/
I always get a cache-status of: Souin; fwd=uri-miss

I don’t understand why I can’t get a hit :frowning:

4. Error messages and/or full log output:

Souin; fwd=uri-miss

5. What I already tried:

Most of my switch is ok, except this caching problem…
I don’t know if I setup the reverse_proxy correctly.

6. Links to relevant resources:

Url to test: https://error.zogg.fr/

Well… No solution found. We test many setup to debug but no way to get a hit.
I swiched back to Varnish

Sorry about that.

I’ve asked @darkweak to take a look at this a few days ago, but I don’t think he’s had a chance to look at this yet.

I know, i made a br on github but no solution found… I’d prefered souin as it act as caddy plugin so i dont need to tweak backend ip to point to varnish endpoint but this is what i finally did.

Anyway darkweak tried :+1:

1 Like

Hi! Yesterday I had a similar situation in my own setup (not your Docker example) where I was totally unable to figure out why I always just got fwd=uri-misses on one server where in contrast it worked on a staging environment.

It was also not helpful - at least without looking into the actual source code - to set the global debug level and look at the logs because there was no obvious notice about the rule that blocked caching (which could be so helpful and a time saver!!). But I could have missed something of course.

So, finally, I found it was because I had set up Basic Authorization on the production server to come, the only difference in the Souin/Caddy configuration. Without it it works. For example: when I used curl -ufoo:bar ... to the VPS without Basic Auth set up the first requests came from the cache until it TTLed out and from then on would no longer come from the cache.

Well… the problem is that for the same backends, Souin give me uri-miss and with Varnish I get Hit
So I really tried to find what’s buggy in my setup, setting logs to max and analyzing them.
I setup a basic nginx with a static html and only got miss…
Unable to find what was the problem…
I keep in touch with Souin as I prefer to lower applications handling my traffic but for now I must keep varnish :frowning:

Maybe there’s a header that Souin respects by default that Varnish does not?

Something along the lines like Module ngx_http_proxy_module
the proxy_ignore_headers directive where you can disrespect the Upstreams likes.

Hello there, if you send an Authorization HTTP request header (basic, bearer token, …) the cache will be miss because a shared http cache must not cache authenticated requests.
But if that’s the reason why it doesn’t store the cache, I will have a workaround to cache the authenticated requests (disabled by default).

1 Like

Well I understand that point.

But… with a sample ‘hello world’ html page served by a ‘classic’ nginx i didn’t got a hit…
So I don’t think there is auth on my page… unless CF setup one that I missed ?

You can now use the following hash ac03801ae86033e77e1700bb5931045ebdb1f513 to see the detail and the key directives in the Cache-Status header. If that miss you’ll know why.

xcaddy build --with github.com/darkweak/souin/plugins/caddy@ac03801ae86033e77e1700bb5931045ebdb1f513 --with github.com/darkweak/souin@ac03801ae86033e77e1700bb5931045ebdb1f513 to use the souin/plugins/caddy and souin with the given version.

1 Like

Tried to build…

#7 138.1 go: downloading github.com/antlr/antlr4 v4.11.1+incompatible
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/api: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/cache/coalescing: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/cache/providers: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/cache/surrogate/providers: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/cache/types: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/configurationtypes: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/plugins: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 142.2 github.com/darkweak/souin/plugins/caddy imports
#7 142.2        github.com/darkweak/souin/rfc: reading github.com/darkweak/souin/go.mod at revision v1.6.24: unknown revision v1.6.24
#7 143.4 2022/12/09 17:51:35 [FATAL] exit status 1
#7 ERROR: executor failed running [/bin/sh -c xcaddy build latest  --with github.com/caddy-dns/cloudflare@latest  --with github.com/darkweak/souin/plugins/caddy@ac03801ae86033e77e1700bb5931045ebdb1f513  --with github.com/darkweak/souin@ac03801ae86033e77e1700bb5931045ebdb1f513]: exit code: 1

Using this:

FROM    --platform=linux/amd64 caddy:builder AS builder

ARG     TARGETPLATFORM
ARG     TARGETOS
ARG     TARGETARCH
ARG     BUILDPLATFORM
ARG     BUILDOS
ARG     BUILDARCH
ARG     BUILDVARIANT

CMD     ["bash"]

ENV     LANG                C.UTF-8

RUN xcaddy build latest \
 --with github.com/caddy-dns/cloudflare@latest \
 --with github.com/darkweak/souin/plugins/caddy@ac03801ae86033e77e1700bb5931045ebdb1f513 \
 --with github.com/darkweak/souin@ac03801ae86033e77e1700bb5931045ebdb1f513

# --with github.com/darkweak/souin/plugins/caddy@latest \
# --with github.com/darkweak/souin@latest

FROM caddy:latest

ENV     LANG                C.UTF-8

COPY --from=builder /usr/bin/caddy /usr/bin/caddy

You can use the latest Souin version now. (v1.6.24)

Build done. Stop/Start stack.
Force refresh page:

Souin; fwd=uri-miss; key=GET-error.zogg.fr-/; detail=NOT-CACHEABLE

Console

2022/12/10 10:17:32.948 DEBUG http.handlers.cache Incoming request: &{Method:GET URL:/ Proto:HTTP/2.0 ProtoMajor:2 ProtoMinor:0 Header:map[Accept:[text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9] Accept-Encoding:[gzip] Accept-Language:[fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7] Cache-Control:[max-age=0] Cdn-Loop:[cloudflare] Cf-Connecting-Ip:[2a01:e0a:975:2b80:a1b7:b9e2:ed84:ab5c] Cf-Ipcountry:[FR] Cf-Ray:[77753b3c3af8d70e-CDG] Cf-Visitor:[{"scheme":"https"}] Cookie:[ph_phc_Jzsm6DTm6V2705zeU5dcNvQDlonOR68XvX2sh1sEOHO_posthog=%7B%22distinct_id%22%3A%22184e362e2e7173b-0c41c2ec5b4bf3-1f462c6d-1fa400-184e362e2e8180b%22%2C%22%24device_id%22%3A%22184e362e2e7173b-0c41c2ec5b4bf3-1f462c6d-1fa400-184e362e2e8180b%22%7D; _ga=GA1.1.1803529779.1670609831; _ga_GS1YNQ8H2W=GS1.1.1670609830.1.1.1670609892.0.0.0; __cf_bm=DZ0o3ihMyeYK9zVJP1OTMOTcEDvnII2TrImOK3szb6w-1670667340-0-AZIkmWmFRZ4KxJprBlAPPdYiTQO+jhj4l7xxLBbFOPtXJec/72HXZbbzbAjOt4kmprynBw3ApH1ThHm6ngZoHX4SBYXrldkQflJBIf3ctWoM6NAMhAixrnielia7xDkHOHTJ1+GWrhcXTb4ntRgsgBk=] Date:[Sat, 10 Dec 2022 10:17:32 UTC] Dnt:[1] If-Modified-Since:[Sun, 04 Dec 2022 17:39:46 GMT] Sec-Ch-Ua:["Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"] Sec-Ch-Ua-Mobile:[?0] Sec-Ch-Ua-Platform:["Linux"] Sec-Fetch-Dest:[document] Sec-Fetch-Mode:[navigate] Sec-Fetch-Site:[none] Sec-Fetch-User:[?1] Upgrade-Insecure-Requests:[1] User-Agent:[Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36] X-Forwarded-For:[2a01:e0a:975:2b80:a1b7:b9e2:ed84:ab5c] X-Forwarded-Proto:[https]] Body:0xc000aca450 GetBody:<nil> ContentLength:0 TransferEncoding:[] Close:false Host:error.zogg.fr Form:map[] PostForm:map[] MultipartForm:<nil> Trailer:map[] RemoteAddr:172.71.130.29:34012 RequestURI:/ TLS:0xc000052210 Cancel:<nil> Response:<nil> ctx:0xc000a1ad80}

Even when setting CF to pause mode.

On another subdomain, i got

Souin; fwd=uri-miss; key=; detail=CANNOT-HANDLE

Well maybe I missed something on Souin settings ?

Looks like there is a Cookie in the response. I would assume to avoid cache poisoning the reponse does not get cached.

Maybe that’s why I get cache with Varnish as I mostly kill cookies, lower host and uri

Well… The final answer is… Fu.k Chrome !
It set max-age=0 automatically and this disable caching.

Thanks to darkweak who spent many times to help me debug this !

I finally ran into hit testing on Firefox.
And for who are using Chrome, I suggest this extension: ModHeader - Modify HTTP headers
You can force something like cache-control max-age=1000 and it works too :slight_smile:

Now I can dive into Souin tuning !

3 Likes

This topic was automatically closed after 30 days. New replies are no longer allowed.