Hey all, thanks in advance.
1. The problem I’m having:
I’m setting up Jenkins on AWS, using an EC2 instance protected by Cloudflare. I have one container running Jenkins and another running Caddy 2.7.5. Everything works fine, except for the webhook call from Bitbucket. I can see in Caddy’s log that the webhook request from Bitbucket hits Caddy, but it is not being forwarded to Jenkins.
When I use curl to check the webhook endpoint, I see the request reaching Jenkins, so I’m very confident that the problem is related to trusted_proxies.
I tried to follow the advice in this post, but still, no luck…
2. Error messages and/or full log output:
My findings:
line 7 "ts":1699423814.408901 shows trusted_proxies being loaded
line 55 "ts":1699423841.7925751 shows a webhook request from Bitbucket and no reponse from jenkins
line 61 jenkins | 2023-11-08 06:10:55.292+0000 shows a curl request hitting jenkins.
Attaching to jenkins, caddy
caddy | {"level":"info","ts":1699423814.3935614,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
caddy | {"level":"warn","ts":1699423814.397896,"msg":"Caddyfile input is not formatted; run 'caddy fmt --overwrite' to fix inconsistencies","adapter":"caddyfile","file":"/etc/caddy/Caddyfile","line":2}
caddy | {"level":"info","ts":1699423814.4045033,"logger":"admin","msg":"admin endpoint started","address":"localhost:2019","enforce_origin":false,"origins":["//127.0.0.1:2019","//localhost:2019","//[::1]:2019"]}
caddy | {"level":"info","ts":1699423814.408102,"logger":"http.auto_https","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
caddy | {"level":"info","ts":1699423814.4085581,"logger":"http.auto_https","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
caddy | {"level":"debug","ts":1699423814.408901,"logger":"http.auto_https","msg":"adjusted config","tls":{"automation":{"policies":[{}]}},"http":{"servers":{"remaining_auto_https_redirects":{"listen":[":80"],"routes":[{},{}]},"srv0":{"listen":[":443"],"routes":[{"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","trusted_proxies":["104.192.136.0/21","185.166.140.0/22","18.205.93.0/25","18.234.32.128/25","13.52.5.0/25","173.245.48.0/20","103.21.244.0/22","103.22.200.0/22","103.31.4.0/22","141.101.64.0/18","108.162.192.0/18","190.93.240.0/20","188.114.96.0/20","197.234.240.0/22","198.41.128.0/17","162.158.0.0/15","104.16.0.0/13","104.24.0.0/14","172.64.0.0/13","131.0.72.0/22"],"upstreams":[{"dial":"jenkins:8080"}]}]}]}],"terminal":true}],"tls_connection_policies":[{}],"automatic_https":{}}}}}
caddy | {"level":"info","ts":1699423814.4096513,"logger":"http","msg":"enabling HTTP/3 listener","addr":":443"}
caddy | {"level":"info","ts":1699423814.4167016,"msg":"failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 2048 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details."}
caddy | {"level":"info","ts":1699423814.4223926,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc0004f4580"}
caddy | {"level":"debug","ts":1699423814.422566,"logger":"http","msg":"starting server loop","address":"[::]:443","tls":true,"http3":true}
caddy | {"level":"info","ts":1699423814.4235108,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2","h3"]}
caddy | {"level":"debug","ts":1699423814.4236412,"logger":"http","msg":"starting server loop","address":"[::]:80","tls":false,"http3":false}
caddy | {"level":"info","ts":1699423814.4236603,"logger":"http.log","msg":"server running","name":"remaining_auto_https_redirects","protocols":["h1","h2","h3"]}
caddy | {"level":"info","ts":1699423814.4236655,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["jenkins.psqa.net"]}
caddy | {"level":"warn","ts":1699423814.4325795,"logger":"tls","msg":"stapling OCSP","error":"no OCSP stapling for [jenkins.psqa.net]: no OCSP server specified in certificate","identifiers":["jenkins.psqa.net"]}
caddy | {"level":"debug","ts":1699423814.432609,"logger":"tls.cache","msg":"added certificate to cache","subjects":["jenkins.psqa.net"],"expiration":1699461556,"managed":true,"issuer_key":"local","hash":"88d65552a2d506682aae1472c61da5e5f97bc248bd8c43d17c93cced0fe31b83","cache_size":1,"cache_capacity":10000}
caddy | {"level":"debug","ts":1699423814.4326448,"logger":"events","msg":"event","name":"cached_managed_cert","id":"b579bec4-b449-42e0-b8f3-c766f36ab437","origin":"tls","data":{"sans":["jenkins.psqa.net"]}}
caddy | {"level":"info","ts":1699423814.7634854,"logger":"pki.ca.local","msg":"root certificate is already trusted by system","path":"storage:pki/authorities/local/root.crt"}
caddy | {"level":"info","ts":1699423814.7657878,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"}
caddy | {"level":"info","ts":1699423814.7658045,"msg":"serving initial configuration"}
caddy | {"level":"info","ts":1699423814.7658343,"logger":"tls","msg":"cleaning storage unit","description":"FileStorage:/data/caddy"}
caddy | {"level":"info","ts":1699423814.767765,"logger":"tls","msg":"finished cleaning storage units"}
jenkins | Running from: /usr/share/jenkins/jenkins.war
jenkins | webroot: /var/jenkins_home/war
jenkins | 2023-11-08 06:10:15.531+0000 [id=1] INFO winstone.Logger#logInternal: Beginning extraction from war file
jenkins | 2023-11-08 06:10:15.779+0000 [id=1] WARNING o.e.j.s.handler.ContextHandler#setContextPath: Empty contextPath
jenkins | 2023-11-08 06:10:15.954+0000 [id=1] INFO org.eclipse.jetty.server.Server#doStart: jetty-10.0.17; built: 2023-10-02T04:04:10.314Z; git: a0f5f05abaa6c3aabb7c3d35f10a6f412ab8b05f; jvm 17.0.8.1+1
jenkins | 2023-11-08 06:10:16.639+0000 [id=1] INFO o.e.j.w.StandardDescriptorProcessor#visitServlet: NO JSP Support for /, did not find org.eclipse.jetty.jsp.JettyJspServlet
jenkins | 2023-11-08 06:10:16.751+0000 [id=1] INFO o.e.j.s.s.DefaultSessionIdManager#doStart: Session workerName=node0
jenkins | 2023-11-08 06:10:18.063+0000 [id=1] INFO hudson.WebAppMain#contextInitialized: Jenkins home directory: /var/jenkins_home found at: EnvVars.masterEnvVars.get("JENKINS_HOME")
jenkins | 2023-11-08 06:10:18.342+0000 [id=1] INFO o.e.j.s.handler.ContextHandler#doStart: Started w.@28cb9120{Jenkins v2.414.3,/,file:///var/jenkins_home/war/,AVAILABLE}{/var/jenkins_home/war}
jenkins | 2023-11-08 06:10:18.381+0000 [id=1] INFO o.e.j.server.AbstractConnector#doStart: Started ServerConnector@6497b078{HTTP/1.1, (http/1.1)}{0.0.0.0:8080}
jenkins | 2023-11-08 06:10:18.408+0000 [id=1] INFO org.eclipse.jetty.server.Server#doStart: Started Server@4944252c{STARTING}[10.0.17,sto=0] @4684ms
jenkins | 2023-11-08 06:10:18.422+0000 [id=25] INFO winstone.Logger#logInternal: Winstone Servlet Engine running: controlPort=disabled
jenkins | 2023-11-08 06:10:18.988+0000 [id=31] INFO jenkins.InitReactorRunner$1#onAttained: Started initialization
jenkins | 2023-11-08 06:10:19.436+0000 [id=30] INFO jenkins.InitReactorRunner$1#onAttained: Listed all plugins
jenkins | 2023-11-08 06:10:29.670+0000 [id=30] INFO jenkins.InitReactorRunner$1#onAttained: Prepared all plugins
jenkins | 2023-11-08 06:10:29.870+0000 [id=30] INFO jenkins.InitReactorRunner$1#onAttained: Started all plugins
jenkins | 2023-11-08 06:10:31.884+0000 [id=30] INFO h.p.b.g.GlobalTimeOutConfiguration#load: global timeout not set
jenkins | 2023-11-08 06:10:32.729+0000 [id=31] INFO jenkins.InitReactorRunner$1#onAttained: Augmented all extensions
jenkins | 2023-11-08 06:10:33.413+0000 [id=31] INFO hudson.slaves.SlaveComputer#tryReconnect: Attempting to reconnect appleIntel
jenkins | 2023-11-08 06:10:33.421+0000 [id=31] INFO hudson.slaves.SlaveComputer#tryReconnect: Attempting to reconnect appleM1
jenkins | 2023-11-08 06:10:33.642+0000 [id=30] INFO jenkins.InitReactorRunner$1#onAttained: System config loaded
jenkins | 2023-11-08 06:10:34.682+0000 [id=31] INFO jenkins.InitReactorRunner$1#onAttained: System config adapted
jenkins | 2023-11-08 06:10:34.774+0000 [id=31] INFO jenkins.InitReactorRunner$1#onAttained: Loaded all jobs
jenkins | 2023-11-08 06:10:34.782+0000 [id=30] INFO jenkins.InitReactorRunner$1#onAttained: Configuration for all jobs updated
jenkins | 2023-11-08 06:10:34.892+0000 [id=31] INFO jenkins.InitReactorRunner$1#onAttained: Completed initialization
jenkins | 2023-11-08 06:10:35.051+0000 [id=24] INFO hudson.lifecycle.Lifecycle#onReady: Jenkins is fully up and running
caddy | {"level":"debug","ts":1699423840.9989338,"logger":"events","msg":"event","name":"tls_get_certificate","id":"9feeb3ca-4942-4e19-8264-7b96492ad573","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4866,4867,49195,49196,49199,49200,49171,49192,156,157,47,53,10],"ServerName":"jenkins.psqa.net","SupportedCurves":[29,23,24,25],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537,513,1539],"SupportedProtos":["http/1.1"],"SupportedVersions":[772,771,770,769],"Conn":{}}}}
caddy | {"level":"debug","ts":1699423841.0004678,"logger":"tls.handshake","msg":"choosing certificate","identifier":"jenkins.psqa.net","num_choices":1}
caddy | {"level":"debug","ts":1699423841.0008202,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"jenkins.psqa.net","subjects":["jenkins.psqa.net"],"managed":true,"issuer_key":"local","hash":"88d65552a2d506682aae1472c61da5e5f97bc248bd8c43d17c93cced0fe31b83"}
caddy | {"level":"debug","ts":1699423841.0011885,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"172.71.151.131","remote_port":"42778","subjects":["jenkins.psqa.net"],"managed":true,"expiration":1699461556,"hash":"88d65552a2d506682aae1472c61da5e5f97bc248bd8c43d17c93cced0fe31b83"}
caddy | {"level":"debug","ts":1699423841.010707,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"jenkins:8080","total_upstreams":1}
caddy | {"level":"debug","ts":1699423841.7925751,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"jenkins:8080","duration":0.781009755,"request":{"remote_ip":"172.71.151.131","remote_port":"42778","client_ip":"172.71.151.131","proto":"HTTP/1.1","method":"POST","host":"jenkins.psqa.net","uri":"/bitbucket-hook/","headers":{"X-Request-Uuid":["09e5ba04-05b5-48f2-b7ba-3b360fe389e5"],"Cf-Connecting-Ip":["2401:1d80:3224:4:9312:c577:5df:2009"],"X-Event-Time":["Wed, 08 Nov 2023 06:10:39 GMT"],"Cf-Ipcontinent":["OC"],"Cf-Ray":["822ba77dc9f3c371-SEA"],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["jenkins.psqa.net"],"X-Attempt-Number":["1"],"X-Event-Key":["pullrequest:unapproved"],"X-B3-Spanid":["0798a76b9fa49dbe"],"Content-Length":["6980"],"X-B3-Traceid":["1d26c1196bc6c89e"],"Cf-Ipcountry":["AU"],"X-B3-Sampled":["1"],"Accept":["*/*"],"Cf-Visitor":["{\"scheme\":\"https\"}"],"User-Agent":["Bitbucket-Webhooks/2.0"],"Cf-Timezone":["Australia/Sydney"],"Content-Type":["application/json"],"Cf-Iplatitude":["-33.49400"],"X-Hook-Uuid":["8ccd8a6c-6e2b-48c6-b911-dca529ba80ac"],"X-B3-Parentspanid":["1d26c1196bc6c89e"],"Cf-Iplongitude":["143.21040"],"Cdn-Loop":["cloudflare"],"Accept-Encoding":["gzip"],"X-Forwarded-For":["2401:1d80:3224:4:9312:c577:5df:2009, 172.71.151.131"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"http/1.1","server_name":"jenkins.psqa.net"}},"headers":{"Date":["Wed, 08 Nov 2023 06:10:41 GMT"],"X-Content-Type-Options":["nosniff"],"Content-Length":["0"],"Server":["Jetty(10.0.17)"]},"status":200}
caddy | {"level":"debug","ts":1699423855.1205704,"logger":"events","msg":"event","name":"tls_get_certificate","id":"19b9a194-fa70-43f7-a38e-fff41a903b5f","origin":"tls","data":{"client_hello":{"CipherSuites":[4865,4866,4867,49195,49196,49199,49200,49171,49192,156,157,47,53,10],"ServerName":"jenkins.psqa.net","SupportedCurves":[29,23,24,25],"SupportedPoints":"AA==","SignatureSchemes":[1027,2052,1025,1283,2053,1281,2054,1537,513,1539],"SupportedProtos":["http/1.1"],"SupportedVersions":[772,771,770,769],"Conn":{}}}}
caddy | {"level":"debug","ts":1699423855.1206605,"logger":"tls.handshake","msg":"choosing certificate","identifier":"jenkins.psqa.net","num_choices":1}
caddy | {"level":"debug","ts":1699423855.1206784,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"jenkins.psqa.net","subjects":["jenkins.psqa.net"],"managed":true,"issuer_key":"local","hash":"88d65552a2d506682aae1472c61da5e5f97bc248bd8c43d17c93cced0fe31b83"}
caddy | {"level":"debug","ts":1699423855.1206865,"logger":"tls.handshake","msg":"matched certificate in cache","remote_ip":"172.69.214.74","remote_port":"29766","subjects":["jenkins.psqa.net"],"managed":true,"expiration":1699461556,"hash":"88d65552a2d506682aae1472c61da5e5f97bc248bd8c43d17c93cced0fe31b83"}
caddy | {"level":"debug","ts":1699423855.1831908,"logger":"http.handlers.reverse_proxy","msg":"selected upstream","dial":"jenkins:8080","total_upstreams":1}
jenkins | 2023-11-08 06:10:55.292+0000 [id=13] WARNING c.c.j.p.BitbucketHookReceiver#doIndex: The Jenkins job cannot be triggered. You might not have configured correctly the WebHook on BitBucket with the last slash `http://<JENKINS-URL>/bitbucket-hook/` or a 'Test connection' invocation of the hook was triggered.
caddy | {"level":"debug","ts":1699423855.294687,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"jenkins:8080","duration":0.111409728,"request":{"remote_ip":"172.69.214.74","remote_port":"29766","client_ip":"172.69.214.74","proto":"HTTP/1.1","method":"GET","host":"jenkins.psqa.net","uri":"/bitbucket-hook/","headers":{"Cf-Ray":["822ba7d5bace3a0b-YYZ"],"Cf-Ipcontinent":["NA"],"Cf-Ipcity":["Burlington"],"Cf-Visitor":["{\"scheme\":\"https\"}"],"User-Agent":["curl/8.1.2"],"Cf-Region":["Ontario"],"Cf-Region-Code":["ON"],"Accept":["*/*"],"Cf-Iplongitude":["-79.76260"],"X-Forwarded-Proto":["https"],"X-Forwarded-For":["24.226.125.171, 172.69.214.74"],"Cf-Connecting-Ip":["24.226.125.171"],"X-Forwarded-Host":["jenkins.psqa.net"],"Accept-Encoding":["gzip"],"Cf-Postal-Code":["L7L"],"Cdn-Loop":["cloudflare"],"Cf-Timezone":["America/Toronto"],"Cf-Ipcountry":["CA"],"Cf-Iplatitude":["43.38020"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"http/1.1","server_name":"jenkins.psqa.net"}},"headers":{"Date":["Wed, 08 Nov 2023 06:10:55 GMT"],"X-Content-Type-Options":["nosniff"],"Content-Length":["0"],"Server":["Jetty(10.0.17)"]},"status":200}
3. Caddy version:
2.7.5
4. How I installed and ran Caddy:
services:
jenkins:
container_name: jenkins
hostname: jenkins.local
image: jenkins/jenkins:lts
volumes:
- /jenkins_server/jenkins_home:/var/jenkins_home
restart: unless-stopped
# No ports exposed; Caddy will proxy the necessary ones
caddy:
container_name: caddy
hostname: caddy.local
image: caddy
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile
ports:
- "443:443"
- "5000:5000"
restart: unless-stopped
depends_on:
- jenkins
a. System environment:
EC2 instance running Docker as in the docker-compose file above
b. Command:
docker compose up
d. My complete Caddy config:
{
debug
#admin off
local_certs
#skip_install_trust
#ocsp_stapling off
#renew_interval 60m
}
(bitbucketcloudflare) {
trusted_proxies 104.192.136.0/21 185.166.140.0/22 18.205.93.0/25 18.234.32.128/25 13.52.5.0/25 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22
}
jenkins.psqa.net {
reverse_proxy jenkins:8080 {
import bitbucketcloudflare
}
}