1. Output of caddy version
:
v2.5.1 h1:bAWwslD1jNeCzDa+jDCNwb8M3UJ2tPa8UZFFzPVmGKs=
2. How I run Caddy:
a. System environment:
AWS EKS Fargate
b. Command:
caddy run -config /etc/caddy/Caddyfile.json
c. Service/unit/compose file:
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
seccomp.security.alpha.kubernetes.io/pod: runtime/default
labels:
app: caddy
app.kubernetes.io/instance: caddy-dev
name: caddy
namespace: ens
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 1
selector:
matchLabels:
app: caddy
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
labels:
app: caddy
spec:
containers:
- args:
- run
- --config
- /etc/caddy/Caddyfile.json
command:
- caddy
env:
- name: AWS_CONFIG_FILE
value: /aws/config/config
- name: AWS_SHARED_CREDENTIALS_FILE
value: /aws/credentials/credentials
- name: AWS_DEFAULT_REGION
value: us-west-2
- name: AWS_DEFAULT_PROFILE
value: default
- name: AWS_SDK_LOAD_CONFIG
value: "true"
- name: AWS_DEFAULT_OUTPUT
value: json
image: caddy:v2.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 30
successThreshold: 1
tcpSocket:
port: http
timeoutSeconds: 15
name: caddy
ports:
- containerPort: 8080
name: http
protocol: TCP
- containerPort: 8443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 30
successThreshold: 1
tcpSocket:
port: http
timeoutSeconds: 15
resources:
requests:
cpu: 250m
memory: 512Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsGroup: 10001
runAsNonRoot: true
runAsUser: 10001
volumeMounts:
- mountPath: /etc/caddy/Caddyfile.json
name: caddyfile-secrets-volume
subPath: Caddyfile.json
- mountPath: /aws/credentials
name: aws-credentials-volume
readOnly: true
- mountPath: /aws/config
name: aws-config-volume
readOnly: true
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: dockerconfigjson-github-com
initContainers:
- command:
- sh
- /custom/configure-caddyfile.sh
env:
- name: APP_PROXY
value: appproxy.ens.svc.cluster.local
- name: DOH_SERVER
value: appproxy.ens.svc.cluster.local
- name: ZEROSSL_API_KEY
valueFrom:
secretKeyRef:
key: zerossl_api
name: caddy-secrets
image: alpine:3.14.1
imagePullPolicy: IfNotPresent
name: configure-caddy
resources:
requests:
cpu: 250m
memory: 512Mi
volumeMounts:
- mountPath: /etc/caddy-old/Caddyfile.json
name: caddyfile-volume
subPath: Caddyfile.json
- mountPath: /etc/caddy
name: caddyfile-secrets-volume
- mountPath: /custom
name: configure-script
readOnly: true
- mountPath: /aws/credentials
name: aws-credentials-volume
readOnly: true
- mountPath: /aws/config
name: aws-config-volume
readOnly: true
restartPolicy: Always
schedulerName: default-scheduler
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: caddyfile
name: caddyfile-volume
- emptyDir:
medium: Memory
sizeLimit: 1Mi
name: caddyfile-secrets-volume
- configMap:
defaultMode: 420
name: configure-caddyfile
name: configure-script
- name: aws-credentials-volume
secret:
defaultMode: 420
items:
- key: credentials
path: credentials
secretName: aws-profile-us-west-2
- name: aws-config-volume
secret:
defaultMode: 420
items:
- key: config
path: config
secretName: aws-profile-us-west-2
d. My complete Caddy config:
{
"admin": {
"disabled": true
},
"logging": {
"logs": {
"default": {
"writer": {
"output": "stdout"
},
"encoder": {
"fields": {
"request\u003eremote_ip": {
"filter": "delete"
},
"request\u003eheaders\u003eX-Forwarded-For": {
"filter": "delete"
}
},
"format": "filter",
"wrap": {
"format": "json"
}
},
"level": "DEBUG",
"exclude": [
"http.log.access.log0"
]
},
"log0": {
"writer": {
"output": "stdout"
},
"encoder": {
"fields": {
"request\u003eremote_ip": {
"filter": "delete"
},
"request\u003eheaders\u003eX-Forwarded-For": {
"filter": "delete"
}
},
"format": "filter",
"wrap": {
"format": "json"
}
},
"include": [
"http.log.access.log0"
]
}
}
},
"storage": {
"module": "dynamodb",
"table": "certificates",
"aws_region": "us-west-2",
"lock_timeout": "300s",
"lock_polling_interval": "10s"
},
"apps": {
"tls": {
"automation": {
"policies": [
{
"on_demand": false,
"subjects": [
"*.dev.lastmilelabs.systems",
"*.eth.dev.lastmilelabs.systems",
"eth.dev.lastmilelabs.systems",
"test.dev.lastmilelabs.systems",
"api.dev.lastmilelabs.systems",
"dns.dev.lastmilelabs.systems"
],
"issuers": [
{
"ca": "https://acme.zerossl.com/v2/DV90",
"api_key": "${ZEROSSL_API_KEY}",
"challenges": {
"dns": {
"provider": {
"name": "route53",
"aws_profile": "default",
"region": "us-west-2",
"access_key_id": "${AWS_ACCESS_KEY_ID}",
"secret_access_key": "${AWS_SECRET_ACCESS_KEY}"
},
"resolvers": [
"208.67.222.222"
],
"ttl": "300s",
"propagation_timeout": "900s"
}
},
"module": "zerossl"
}
]
},
{
"on_demand": true,
"issuers": [
{
"ca": "https://acme.zerossl.com/v2/DV90",
"api_key": "${ZEROSSL_API_KEY}",
"module": "zerossl"
}
]
}
],
"on_demand": {
"rate_limit": {
"interval": "120s",
"burst": 10
},
"ask": "http://appproxy.ens.svc.cluster.local:9999/ask"
}
}
},
"cache": {
"headers": [
"Content-Type"
],
"log_level": "info",
"etcd": {
"configuration": {
"endpoints": [
"caddy-etcd.ens.svc.cluster.local:2379"
],
"MaxCallSendMsgSize": 9500000
}
},
"ttl": "300s",
"distributed": true
},
"http": {
"http_port": 8080,
"servers": {
"srv0": {
"listener_wrappers": [
{
"wrapper": "go_proxyproto",
"timeout": "5s"
},
{
"wrapper": "tls"
}
],
"listen": [
":8080"
],
"automatic_https": {
"disable_redirects": true
},
"routes": [
{
"handle": [
{
"handler": "rate_limit",
"rate_limits": {
"remote_host": {
"key": "{http.request.remote.host}",
"window": "60s",
"max_events": 250
}
}
},
{
"handler": "static_response",
"headers": {
"Location": [
"https://{http.request.host}:443{http.request.uri}"
]
},
"status_code": 308
},
{
"handler": "headers",
"response": {
"set": {
"Server": [
"eth.dev.lastmilelabs.systems"
]
}
}
}
]
}
]
},
"srv1": {
"listener_wrappers": [
{
"wrapper": "go_proxyproto",
"timeout": "5s"
},
{
"wrapper": "tls"
}
],
"listen": [
":8443"
],
"automatic_https": {
"disable_redirects": true
},
"tls_connection_policies": [
{
"cipher_suites": [
"TLS_AES_128_GCM_SHA256",
"TLS_AES_256_GCM_SHA384",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
],
"curves": [
"x25519",
"secp256r1",
"secp384r1",
"secp521r1"
],
"protocol_min": "tls1.2",
"protocol_max": "tls1.3"
}
],
"routes": [
{
"handle": [
{
"handler": "rate_limit",
"rate_limits": {
"remote_host": {
"key": "{http.request.remote.host}",
"window": "60s",
"max_events": 250
}
}
},
{
"handler": "cache",
"ttl": "300s",
"default_cache_control": "public, max-age=86400",
"log_level": "info"
},
{
"handler": "encode",
"encodings": {
"gzip": {},
"zstd": {}
},
"prefer": [
"zstd",
"gzip"
]
},
{
"handler": "headers",
"response": {
"set": {
"Access-Control-Allow-Credentials": [
"false"
],
"Access-Control-Allow-Origin": [
"*"
],
"Cache-Control": [
"max-age=300, must-revalidate"
],
"Cross-Origin-Resource-Policy": [
"cross-origin"
],
"Permissions-Policy": [
"interest-cohort=(), geolocation=(), battery=()"
],
"Referrer-Policy": [
"strict-origin-when-cross-origin"
],
"Server": [
"eth.dev.lastmilelabs.systems"
],
"X-Content-Type-Options": [
"nosniff"
],
"X-Frame-Options": [
"SAMEORIGIN"
],
"Content-Security-Policy": [
"frame-ancestors 'self';"
],
"X-Xss-Protection": [
"1; mode=block"
],
"Strict-Transport-Security": [
"max-age=31536000; includeSubDomains; preload"
],
"X-True-Host": [
"{http.request.host}"
]
}
}
}
]
},
{
"match": [
{
"path": [
"/health"
]
}
],
"handle": [
{
"handler": "static_response",
"status_code": 200
}
]
},
{
"match": [
{
"expression": "{http.request.host} == \"dns.eth.dev.lastmilelabs.systems\" && {http.request.orig_uri.path}.contains(\"/dns-query\")"
}
],
"handle": [
{
"handler": "reverse_proxy",
"transport": {
"protocol": "http"
},
"headers": {
"request": {
"replace": {
"Host": [
{
"replace": "$1",
"search_regexp": "([-a-z0-9]+\\.eth)\\.dev.lastmilelabs.systems"
}
]
},
"delete": [
"X-Forwarded-For",
"Referer",
"User-Agent"
]
},
"response": {
"delete": [
"Access-Control-Allow-Origin",
"Transfer-Encoding",
"Cache-Control"
]
}
},
"upstreams": [
{
"dial": "${DOH_SERVER}:11000"
}
]
}
]
},
{
"handle": [
{
"handler": "reverse_proxy",
"transport": {
"protocol": "http"
},
"headers": {
"request": {
"replace": {
"Host": [
{
"replace": "$1",
"search_regexp": "([-a-z0-9]+\\.eth)\\.dev.lastmilelabs.systems"
}
]
},
"delete": [
"X-Forwarded-For",
"Referer",
"User-Agent"
]
},
"response": {
"delete": [
"Access-Control-Allow-Origin",
"Transfer-Encoding",
"Cache-Control"
]
}
},
"upstreams": [
{
"dial": "${APP_PROXY}:8888"
}
]
}
]
}
],
"logs": {
"default_logger_name": "log0"
}
}
}
}
}
}
3. The problem I’m having:
I previously issued several wildcard certificates, prior to enabling on-demand TLS:
"*.dev.lastmilelabs.systems",
"*.eth.dev.lastmilelabs.systems",
"eth.dev.lastmilelabs.systems",
"test.dev.lastmilelabs.systems",
"api.dev.lastmilelabs.systems",
"dns.dev.lastmilelabs.systems"
After enabling on-demand TLS, the previously issued wildcard certificates are no longer attempting to renew.
I even deleted the *.dev.lastmilelabs.systems
certificate from the DynamoDB storage backend but Caddy never attempts to issue a new one (simply as a test to see if Caddy would successfully fetch the new cert).
We have a large number of supported domains under *.eth.dev.lastmilelabs.systems
and our users are allowed to issue second level subdomain certs for things like app.mystore.eth.dev.lastmilelabs.systems
.
How can I configure Caddy to manage statically defined wildcard certificates (and renew them) as well as support on-demand certificates for second level subdomains?
4. Error messages and/or full log output:
{"level":"debug","ts":1662562506.9290273,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"dxvote.eth.dev.lastmilelabs.systems"}
{"level":"debug","ts":1662562506.929088,"logger":"tls.handshake","msg":"choosing certificate","identifier":"*.eth.dev.lastmilelabs.systems","num_choices":1}
{"level":"debug","ts":1662562506.929106,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"*.eth.dev.lastmilelabs.systems","subjects":["*.eth.dev.lastmilelabs.systems"],"managed":true,"issuer_key":"acme.zerossl.com-v2-DV90","hash":"8505ca9a30a8605d24b1d9f400a764098461ac302ef685a3d58ea5851d02e288"}
{"level":"debug","ts":1662562506.9295726,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["*.eth.dev.lastmilelabs.systems"],"managed":true,"expiration":1665100799,"hash":"8505ca9a30a8605d24b1d9f400a764098461ac302ef685a3d58ea5851d02e288"}
{"level":"info","ts":1662562506.9303577,"logger":"tls.on_demand","msg":"attempting certificate renewal","server_name":"dxvote.eth.dev.lastmilelabs.systems","subjects":["*.eth.dev.lastmilelabs.systems"],"expiration":1665100799,"remaining":2538292.069647882,"revoked":false}
{"level":"info","ts":1662562507.3773344,"logger":"tls.renew","msg":"acquiring lock","identifier":"dxvote.eth.dev.lastmilelabs.systems"}
{"level":"info","ts":1662562507.3849118,"logger":"tls.renew","msg":"lock acquired","identifier":"dxvote.eth.dev.lastmilelabs.systems"}
{"level":"error","ts":1662562507.388652,"logger":"tls.renew","msg":"will retry","error":"file does not exist","attempt":1,"retrying_in":60,"elapsed":0.003703527,"max_duration":2592000}
{"level":"debug","ts":1662562507.4262328,"logger":"tls.handshake","msg":"no matching certificates and no custom selection logic","identifier":"dxvote.eth.dev.lastmilelabs.systems"}
{"level":"debug","ts":1662562507.4263859,"logger":"tls.handshake","msg":"choosing certificate","identifier":"*.eth.dev.lastmilelabs.systems","num_choices":1}
{"level":"debug","ts":1662562507.4264257,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"*.eth.dev.lastmilelabs.systems","subjects":["*.eth.dev.lastmilelabs.systems"],"managed":true,"issuer_key":"acme.zerossl.com-v2-DV90","hash":"8505ca9a30a8605d24b1d9f400a764098461ac302ef685a3d58ea5851d02e288"}
{"level":"debug","ts":1662562507.4264684,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["*.eth.dev.lastmilelabs.systems"],"managed":true,"expiration":1665100799,"hash":"8505ca9a30a8605d24b1d9f400a764098461ac302ef685a3d58ea5851d02e288"}
{"level":"debug","ts":1662562507.4265227,"logger":"tls.on_demand","msg":"certificate expires soon but is already being renewed; serving current certificate","subjects":["*.eth.dev.lastmilelabs.systems"],"remaining":2538291.573479977}
{"level":"error","ts":1662562567.3930051,"logger":"tls.renew","msg":"will retry","error":"file does not exist","attempt":2,"retrying_in":120,"elapsed":60.008057567,"max_duration":2592000}
Based on the log output, it seems that Caddy does not attempt to renew the wildcard *.eth.dev.lastmilelabs.systems
but rather attempts to issue a new on-demand cert for the CN
of the requested host.
5. What I already tried:
Deleting an existing certificate from storage and restarting Caddy.
Re-ordering the JSON configuration
6. Links to relevant resources:
N/A