Strange "invalid memory address or nil pointer dereference" with custom build

1. Caddy version (caddy version):

2.4.5 built with Better fix for #142; patches 335551df · caddyserver/certmagic@88b8609 · GitHub

2. How I run Caddy:

a. System environment:

b. Command:

systemctl start caddy

c. Service/unit/compose file:

# caddy.service
#
# For using Caddy with a config file.
#
# Make sure the ExecStart and ExecReload commands are correct
# for your installation.
#
# See https://caddyserver.com/docs/install for instructions.
#
# WARNING: This service does not use the --resume flag, so if you
# use the API to make changes, they will be overwritten by the
# Caddyfile next time the service is restarted. If you intend to
# use Caddy's API to configure it, add the --resume flag to the
# `caddy run` command or use the caddy-api.service file instead.

# https://www.freedesktop.org/software/systemd/man/systemd.unit.html
[Unit]
Description=Caddy
Documentation=https://caddyserver.com/docs/
After=network.target network-online.target
Requires=network-online.target

# No more than StartLimitBurst service start during StartLimitIntervalSec period
StartLimitIntervalSec=500
StartLimitBurst=5

# https://www.freedesktop.org/software/systemd/man/systemd.service.html
[Service]
# Wait RestartSec before an auto restart on-failure
Restart=on-failure
RestartSec=5s
Type=notify
User=caddy
Group=caddy
ExecStart=/usr/bin/caddy run --environ --config /etc/caddy/Caddyfile
ExecReload=/usr/bin/caddy reload --config /etc/caddy/Caddyfile
TimeoutStopSec=5s
LimitNOFILE=1048576
LimitNPROC=512
PrivateTmp=true
ProtectSystem=full
AmbientCapabilities=CAP_NET_BIND_SERVICE

[Install]
WantedBy=multi-user.target

d. My complete Caddyfile or JSON config:

(log) {
    log {
        output file {args.0} {
            roll_size     100MiB
        }
        level DEBUG
        format json
        format filter {
            wrap json
            fields {
                common_log delete
                resp_headers>Content-Security-Policy delete
                request>remote_addr ip_mask {
                    ipv4 24
                    ipv6 32
                }
            }
        }
    }
}

{
    debug

    #Bind admin endpoint on all interfaces
    admin 0.0.0.0:2019

    on_demand_tls {
        ask http://localhost:8000/api/cname/validate
    }

    # Workaround to prevent on_demand being enabled on all domains
    # see https://github.com/caddyserver/caddy/pull/4128
    email security+caddy@lempire.co

    import log /var/log/caddy/default_req.log

    # Reduce caddy reload grace_period to 3 min (default was 20 min) to improve reload
    # See https://caddy.community/t/hangs-on-reload/12010/35
    # Because of websockets long cnx, reload always took grace_period duration
    grace_period 3m
}

# Redirect any accesses using an IP instead of a domain name
# hackerone report: https://hackerone.com/reports/723126
51.91.128.49:80,
91.121.37.140:80,
147.135.229.201:80,
51.254.243.65:80,
54.36.18.249:80,
46.105.54.36:80
{
    redir https://app.lemlist.com{uri}
}

(cloudflare_matchers) {
    remote_ip 103.21.244.0/22
    remote_ip 103.22.200.0/22
    remote_ip 103.31.4.0/22
    remote_ip 104.16.0.0/13
    remote_ip 104.24.0.0/14
    remote_ip 108.162.192.0/18
    remote_ip 131.0.72.0/22
    remote_ip 141.101.64.0/18
    remote_ip 162.158.0.0/15
    remote_ip 172.64.0.0/13
    remote_ip 173.245.48.0/20
    remote_ip 188.114.96.0/20
    remote_ip 190.93.240.0/20
    remote_ip 197.234.240.0/22
    remote_ip 198.41.128.0/17
    remote_ip 2400:cb00::/32
    remote_ip 2606:4700::/32
    remote_ip 2803:f800::/32
    remote_ip 2405:b500::/32
    remote_ip 2405:8100::/32
    remote_ip 2a06:98c0::/29
    remote_ip 2c0f:f248::/32
}

(front_servers) {
    to localhost:42001
    to localhost:42002
    to localhost:42003
    to localhost:42004
    to localhost:42005
    to localhost:42006
    to localhost:42007
    to localhost:42008

    health_uri /api/ping
    health_interval 1s
    health_timeout 5s
}

(badbots_filter) {
    @badbots {
        header_regexp User-Agent "(?i)Fuzz Faster U Fool.*"
    }
    header @badbots {
        X-Cache-Debug "bbf"
        Server ""
    }
    respond @badbots "Access denied" 403
}

#
# Main application
#

app.lemlist.com,
app2.lemlist.com
{
    import log /var/log/caddy/front_req.log

    # route keyword is mandatory to force Caddy to keep our
    #   directives order instead of using the Caddy builtin order
    route {

        request_body {
            max_size 20MB
        }

        @from_cloudflare {
            import cloudflare_matchers
        }

        @not_from_cloudflare {
            not {
                import cloudflare_matchers
            }
        }

        ##############################################
        # pass all api lempire (e, i) requests to one
        #   dedicated Meteor app process
        ##############################################
        @api_e_i {
            path /api/lempire/*
        }

        header @api_e_i {
            X-Cache-Debug "app.api_e_i"
            Server ""
        }

        reverse_proxy @api_e_i {
            to localhost:8000
        }

        ##############################################
        # pass all api requests to Meteor api process
        ##############################################
        @api {
            path /api/*
        }

        header @api {
            X-Cache-Debug "app.api"
            Server ""
        }

        reverse_proxy @api {
            to localhost:42100
        }

        ##############################################
        # Caches
        ##############################################
        # must be after /api routes or https://app.lemlist.com/api/froala/files/*.jpg will not be served by meteor
        @cache_app {
            path *.svg *.woff2 *.eot *.ttf *.woff *.jpg *.jpeg *.png *.gif *.ico *.html
        }

        header @cache_app {
            # 1d expiration
            Cache-Control max-age=86400
            X-Cache-Debug "app.cache"
            Server ""
        }

        file_server @cache_app {
            root /opt/lemlist/app/programs/web.browser/app
        }

        @cache_app2 {
            path_regexp ^/lib/.*\.(map|js|css)$
        }

        header @cache_app2 {
            # 1d expiration
            Cache-Control max-age=86400
            X-Cache-Debug "app.cache2"
            Server ""
        }

        file_server @cache_app2 {
            root /opt/lemlist/app/programs/web.browser/app
        }


        @cache_version {
            path_regexp ^/[0-9a-z]+\.(js|css|map).*
        }

        header @cache_version {
            # 1d expiration
            Cache-Control max-age=86400
            X-Cache-Debug "app.cache_version"
            Server ""
        }

        file_server @cache_version {
            root /opt/lemlist/web.browser
        }


        ##############################################
        # Anything else goes to Meteor fronts
        ##############################################
        header @from_cloudflare {
            X-Cache-Debug "app.main.cf"
            Server ""
        }

        reverse_proxy @from_cloudflare {
            header_up X-Real-IP {http.request.header.CF-Connecting-IP}
            header_up X-Forwarded-For {http.request.header.CF-Connecting-IP}
            header_up Lb-Hash {http.request.header.CF-Connecting-IP}:{remote_port}

            import front_servers

            lb_policy header CF-Connecting-IP
        }

        header @not_from_cloudflare {
            X-Cache-Debug "app.main"
            Server ""
        }

        reverse_proxy @not_from_cloudflare {
            header_up Lb-Hash {remote_host}:{remote_port}

            import front_servers

            lb_policy ip_hash
        }

        handle * {
            respond 404
        }
    }

    handle_errors {
        @errors {
            expression {http.error.status_code} == '502'
        }
        rewrite @errors /{http.error.status_code}.html
        file_server {
            root /var/www
        }
    }

    #dev only : to work on laptop localhost
    #tls internal
}

#
# Custom domains
#
(custom_domains) {
    import log /var/log/caddy/custom_{args.0}_req.log

    route {
        import badbots_filter

        #######################################
        # render a special page when on root so the user can see if
        #   its custom domain is correctly setted up
        #######################################
        header / {
            X-Cache-Debug "custom.cname_ok"
            Server ""
        }

        file_server / {
            root /var/www/lemlist_cname_ok
        }

        #######################################
        # pass all requests "track open/click" to track
        #######################################
        @track {
            path_regexp ^/api/(t/c|track/open|reply)/
        }

        header @track {
            X-Cache-Debug "custom.track"
            Server ""
        }

        reverse_proxy @track {
            to localhost:42160
        }

        #######################################
        # pass all requests "images + pages" to images
        #######################################

        @images {
            path /*
        }

        header @images {
            X-Cache-Debug "custom.images"
            Server ""
        }

        reverse_proxy @images {
            to localhost:42140
        }

        handle * {
            respond 404
        }

    }
}

# WARNING : DO NOT add any other site address than :443 here
:443
{
    import custom_domains on_demand

    tls {
        on_demand

        issuer acme {
            email security+caddy@lempire.co
            dir https://acme.zerossl.com/v2/DV90
            eab ...
            ...
        }

        issuer acme {
            email security+caddy@lempire.co
            dir https://acme-v02.api.letsencrypt.org/directory
        }
    }
}

pages.lemlist.com
{
    import custom_domains predefined
}

# WARNING : DO NOT add any other site address than :80 here
:80
{
    import custom_domains port80
}

#
# public direct access to service
#
(public_passthrough) {
    {args.0}
    {
        import log /var/log/caddy/public_passthrough_{args.0}_req.log

        route {
            request_body {
                max_size 20MB
            }

            header * {
                X-Cache-Debug "{args.0}.main"
                Server ""
            }

            reverse_proxy * {
                to localhost:{args.1}
            }
        }
    }
}
import public_passthrough api.lemlist.com 42100
import public_passthrough img.lemlist.com 42150
# https://front.lempire.co -> admin
import public_passthrough front.lem.ovh 8000


#
# private direct access to services
#
(private_passthrough) {
    front.lem.ovh:{args.0}
    {
        import log /var/log/caddy/private_passthrough_{args.0}_req.log

        route {
            @cache_version {
                path_regexp ^/[0-9a-z]+\.(js|css|map).*
            }

            header @cache_version {
                # 1d expiration
                Cache-Control max-age=86400
                X-Cache-Debug "direct.cache_version"
                Server ""
            }

            file_server @cache_version {
                root /opt/lemlist/web.browser
            }

            header * {
                X-Cache-Debug "direct.main"
                Server ""
            }

            reverse_proxy * {
                to localhost:{args.1}
            }
        }

        basicauth * {
            zzz ....
        }
    }
}

# Fronts
import private_passthrough 43001 42001
import private_passthrough 43002 42002
import private_passthrough 43003 42003
import private_passthrough 43004 42004
import private_passthrough 43005 42005
import private_passthrough 43006 42006
import private_passthrough 43007 42007
import private_passthrough 43008 42008

# API
import private_passthrough 43100 42100
# Cron
import private_passthrough 43130 42130
# Images Front
import private_passthrough 43150 42150
# Track
import private_passthrough 43160 42160
# Images
import private_passthrough 43140 42140

3. The problem I’m having:

Got 2 crashes in one week, not sure both are related… But I never got crashes with previous versions, using caddy for 6 months.

For the 1st crash, the service stopped.
Strangely the 2nd one was “only” a panic, the process was running, but sites were 408

May be let focus on 2nd one, as logs are interesting:

4. Error messages and/or full log output:

There is this very strange log line where removing_subjects is empty, same for removing_hash

"msg":"cache full; evicting random certificate","removing_subjects":[],"removing_hash":"","inserting_subjects":["pages.csmagency.ie"],"inserting_hash":"185532f43cf6f805e14cb7a035742456aadb0d6e400d578c8ecde2cfe1c090f4"

a bit more logs until the panic occured in removeCertificate():

{"level":"debug","ts":1633549994.4163284,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["custom.lemlist.com"],"managed":true,"expiration":1640822399,"hash":"a2fcedd5f125596efefa7c1021fa5d074e41d028876e084c4636ecd1da512122"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.421294,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4213588,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4213848,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4240758,"logger":"tls.handshake","msg":"choosing certificate","identifier":"trail.lemlist.com","num_choices":1}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4241347,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"trail.lemlist.com","subjects":["trail.lemlist.com"],"managed":true,"issuer_key":"acme.zerossl.com-v2-DV90","hash":"26c05f9b2e8d127a6fe949a1145302a843f44089c7567ae9e2c1ce1c8a854013"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4241555,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["trail.lemlist.com"],"managed":true,"expiration":1640822399,"hash":"26c05f9b2e8d127a6fe949a1145302a843f44089c7567ae9e2c1ce1c8a854013"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4276536,"logger":"tls.cache","msg":"cache full; evicting random certificate","removing_subjects":[],"removing_hash":"","inserting_subjects":["pages.csmagency.ie"],"inserting_hash":"185532f43cf6f805e14cb7a035742456aadb0d6e400d578c8ecde2cfe1c090f4"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4280682,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4281065,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4281197,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
2021-10-06 21:53:14	
{"level":"debug","ts":1633549994.4286802,"logger":"http.stdlib","msg":"http: panic serving 212.102.49.131:44993: runtime error: invalid memory address or nil pointer dereference\ngoroutine 71579125 [running]:\nnet/http.(*conn).serve.func1(0xc04fa35540)\n\t/usr/local/go/src/net/http/server.go:1824 +0x153\npanic(0x16d7ac0, 0x25baad0)\n\t/usr/local/go/src/runtime/panic.go:971 +0x499\ngithub.com/caddyserver/certmagic.(*Cache).removeCertificate(0xc0004808c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0359e7e00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:281 +0x419\ngithub.com/caddyserver/certmagic.(*Cache).unsyncedCacheCertificate(0xc0004808c0, 0xc04e1d6180, 0x3, 0x4, 0x17d61c0, 0xc03321f800, 0x0, 0x0, 0x0, 0xc01f65ee00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:227 +0xbfd\ngithub.com/caddyserver/certmagic.(*Cache).cacheCertificate(0xc0004808c0, 0xc04e1d6180, 0x3, 0x4, 0x17d61c0, 0xc03321f800, 0x0, 0x0, 0x0, 0xc01f65ee00, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/cache.go:184 +0x98\ngithub.com/caddyserver/certmagic.(*Config).CacheManagedCertificate(0xc01570c820, 0xc047fa69a8, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/certificates.go:114 +0x1a5\ngithub.com/caddyserver/certmagic.(*Config).getCertDuringHandshake(0xc01570c820, 0xc008bea780, 0x300000101, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:271 +0x17a6\ngithub.com/caddyserver/certmagic.(*Config).GetCertificate(0xc01570c820, 0xc008bea780, 0x12, 0xc000205600, 0x7f959a6d2878)\n\t/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:66 +0x118\ngithub.com/caddyserver/caddy/v2/modules/caddytls.(*ConnectionPolicy).buildStandardTLSConfig.func1(0xc008bea780, 0x410018, 0xb8, 0x1834fe0)\n\t/home/vagrant/caddy/modules/caddytls/connpolicy.go:185 +0x10f\ncrypto/tls.(*Config).getCertificate(0xc000701380, 0xc008bea780, 0xc008bea780, 0x18, 0x20)\n\t/usr/local/go/src/crypto/tls/common.go:1017 +0x3cb\ncrypto/tls.(*serverHandshakeState).processClientHello(0xc00e981968, 0xc000269380, 0x59a605)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:223 +0x2c8\ncrypto/tls.(*serverHandshakeState).handshake(0xc00e981968, 0xc000b55400, 0x0)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:64 +0x37\ncrypto/tls.(*Conn).serverHandshake(0xc00ab4f880, 0xc00299b6e0, 0x14)\n\t/usr/local/go/src/crypto/tls/handshake_server.go:58 +0x14b\ncrypto/tls.(*Conn).Handshake(0xc00ab4f880, 0x0, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:1391 +0xc9\nnet/http.(*conn).serve(0xc04fa35540, 0x1bcc958, 0xc0008f5020)\n\t/usr/local/go/src/net/http/server.go:1840 +0x1a5\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:3013 +0x39b"}

5. What I already tried:

I read carefully the certmagic/cache.go code, I cannot explain how is it possible to have an empty cert.Names
I grep “subject:”[]" in logs from start to the panic, the only occurence is the “evicting random certificate” line (I was expecting an occurence from a “added certificate to cache” log). So it seems no badly formed certificate have been pushed to the cache in 1st place.

May be it’s related to the fact that I built caddy myself …

6. Links to relevant resources:

How I built caddy on a fresh ubuntu 18.04 VM

sudo apt-get update
sudo apt install -y gcc

# Install go
wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go1.16.2.linux-amd64.tar.gz
export PATH=$PATH:/usr/local/go/bin

git clone https://github.com/caddyserver/caddy.git
cd caddy
go get -d -v github.com/caddyserver/certmagic@88b8609b4d8fb8c7b1865322602606b85240c80d
cd cmd/caddy/
go build

For records I also put the 1st crash logs

syslog:


goroutine 74613350 [running]:

panic: interface conversion: interface {} is nil, not acme.Authorization
github.com/mholt/acmez.(*Client).ObtainCertificateUsingCSR(0xc024ed9788, 0x1bcc958, 0xc0156e44b0, 0xc036e2fd18, 0x5, 0xc01d01f0c0, 0x1, 0x4, 0x1, 0x0, ...)
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/acmemanager.go:315 +0x1b3
github.com/caddyserver/certmagic.(*ACMEManager).doIssue(0xc028225600, 0x1bcc958, 0xc0156e44b0, 0xc008e18fc0, 0xc021a53601, 0xc005b046a0, 0x410018, 0x100, 0x188eb80)
/home/vagrant/go/pkg/mod/github.com/mholt/acmez@v1.0.0/client.go:137 +0x1d88
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/acmemanager.go:244 +0xd6
github.com/caddyserver/certmagic.(*ACMEManager).Issue(0xc028225600, 0x1bcc958, 0xc0156e44b0, 0xc008e18fc0, 0x0, 0xc00013d4c0, 0x19)
github.com/caddyserver/caddy/v2/modules/caddytls.(*ACMEIssuer).Issue(0xc000638ea0, 0x1bcc958, 0xc0156e44b0, 0xc008e18fc0, 0x1, 0x1, 0xc04a271900)
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/async.go:106 +0x239
github.com/caddyserver/certmagic.doWithRetry(0x1bcc920, 0xc0156e44b0, 0xc025de0d20, 0xc005b056a0, 0x1, 0x1)
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/config.go:739 +0x373
github.com/caddyserver/certmagic.(*Config).renewCert.func2(0x1bcc958, 0xc0156e44b0, 0x0, 0x0)
/home/vagrant/caddy/modules/caddytls/acmeissuer.go:234 +0xd4
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:614 +0xd1c
created by github.com/caddyserver/certmagic.(*Config).renewDynamicCertificate
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/handshake.go:582 +0x12b
github.com/caddyserver/certmagic.(*Config).renewDynamicCertificate.func3(0x1bcc920, 0xc027acfec0, 0xc03b71a150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/config.go:643
github.com/caddyserver/certmagic.(*Config).RenewCertAsync(...)
/home/vagrant/go/pkg/mod/github.com/caddyserver/certmagic@v0.14.6-0.20210922202800-88b8609b4d8f/config.go:788 +0x445
github.com/caddyserver/certmagic.(*Config).renewCert(0xc0044d4320, 0x1bcc920, 0xc027acfec0, 0xc013822bd0, 0x14, 0x0, 0x0, 0x0)

caddy logs:

{"level":"debug","ts":1632957714.5587342,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
{"level":"debug","ts":1632957714.558773,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.558793,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.5610003,"logger":"tls.handshake","msg":"choosing certificate","identifier":"custom.lemlist.com","num_choices":1}
{"level":"debug","ts":1632957714.5610418,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"custom.lemlist.com","subjects":["custom.lemlist.com"],"managed":true,"issuer_key":"acme-v02.api.letsencrypt.org-directory","hash":"bbd009380caa15dcd0d5b4916d04f4ea597646e09dbb338b5c7ae3f4e664f1e8"}
{"level":"debug","ts":1632957714.5610566,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["custom.lemlist.com"],"managed":true,"expiration":1635596077,"hash":"bbd009380caa15dcd0d5b4916d04f4ea597646e09dbb338b5c7ae3f4e664f1e8"}
{"level":"debug","ts":1632957714.562655,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
{"level":"debug","ts":1632957714.5627108,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.5627244,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.564746,"logger":"tls.handshake","msg":"choosing certificate","identifier":"trail.lemlist.com","num_choices":1}
{"level":"debug","ts":1632957714.5647938,"logger":"tls.handshake","msg":"default certificate selection results","identifier":"trail.lemlist.com","subjects":["trail.lemlist.com"],"managed":true,"issuer_key":"acme-v02.api.letsencrypt.org-directory","hash":"8184623789b794200d766df624c4b3a1b94ce92057f0ba2c47da846111bfb332"}
{"level":"debug","ts":1632957714.5648077,"logger":"tls.handshake","msg":"matched certificate in cache","subjects":["trail.lemlist.com"],"managed":true,"expiration":1635596077,"hash":"8184623789b794200d766df624c4b3a1b94ce92057f0ba2c47da846111bfb332"}
{"level":"debug","ts":1632957714.5663855,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/var/www/lemlist_cname_ok","request_path":"/","result":"/var/www/lemlist_cname_ok"}
{"level":"debug","ts":1632957714.5664387,"logger":"http.handlers.file_server","msg":"located index file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.5664504,"logger":"http.handlers.file_server","msg":"opening file","filename":"/var/www/lemlist_cname_ok/index.html"}
{"level":"debug","ts":1632957714.6416004,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42100","request":{"remote_addr":"141.101.68.0:11840","proto":"HTTP/2.0","method":"OPTIONS","host":"app.lemlist.com","uri":"/api/linkedin/updateUser","headers":{"Sec-Fetch-Mode":["cors"],"Sec-Fetch-Site":["cross-site"],"Sec-Fetch-Dest":["empty"],"X-Forwarded-Proto":["https"],"Origin":["chrome-extension://khnbclggeggefodgimdekejhipkeobnc"],"Accept-Language":["fr,en;q=0.9,fr-FR;q=0.8,en-US;q=0.7"],"Cdn-Loop":["cloudflare"],"Accept-Encoding":["gzip"],"Cf-Ipcountry":["FR"],"Cf-Visitor":["{\"scheme\":\"https\"}"],"Access-Control-Request-Headers":["content-type"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36"],"Cf-Connecting-Ip":["2a01:cb00:3e8:7500:74d4:179:5a51:8372"],"X-Forwarded-For":["2a01:cb00:3e8:7500:74d4:179:5a51:8372, 141.101.68.135"],"Cf-Ray":["6968f3546c533b13-CDG"],"Accept":["*/*"],"Access-Control-Request-Method":["POST"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","proto_mutual":true,"server_name":"app.lemlist.com"}},"headers":{"Content-Type":["application/json"],"Access-Control-Allow-Headers":["*"],"Access-Control-Allow-Origin":["*"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["2"]},"status":200}
{"level":"debug","ts":1632957714.6615615,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42100","request":{"remote_addr":"3.88.135.0:57638","proto":"HTTP/1.1","method":"POST","host":"api.lemlist.com","uri":"/api/unsubscribes/msturdivant%40needld.org","headers":{"User-Agent":["Zapier"],"Authorization":["Basic dGVhX2FOeEM2YzdTZGhjbWJaRkw4OmM1MmJkNjQwZDc0ZGJjNzk4MjNlMTNmZjgwM2E4ZDAw"],"Accept":["*/*"],"Content-Length":["0"],"Accept-Encoding":["gzip,deflate"],"X-Forwarded-For":["3.88.135.41"],"X-Forwarded-Proto":["https"]},"tls":{"resumed":true,"version":772,"cipher_suite":4865,"proto":"","proto_mutual":true,"server_name":"api.lemlist.com"}},"headers":{"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["97"],"Retry-After":["2"],"X-Ratelimit-Limit":["20"],"X-Ratelimit-Remaining":["19"],"X-Ratelimit-Reset":["Thu Sep 30 2021 01:21:56 GMT+0200 (Central European Summer Time)"],"Content-Type":["application/json"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"]},"status":200}
{"level":"debug","ts":1632957714.666391,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42160","request":{"remote_addr":"104.28.94.0:12822","proto":"HTTP/1.1","method":"GET","host":"trail.subkit.co","uri":"/api/track/open/usr_jEKd39rHekHWyg9xk/tsk_aYRWtcNjiMyAAY69d","headers":{"X-Forwarded-For":["104.28.94.168"],"Accept":["image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"],"Accept-Language":["en-US,en;q=0.9"],"X-Forwarded-Proto":["http"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["Mozilla/5.0"]}},"headers":{"Keep-Alive":["timeout=5"],"Content-Length":["42"],"Content-Type":["image/gif"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"],"Connection":["keep-alive"]},"status":200}
{"level":"debug","ts":1632957714.7614925,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42160","request":{"remote_addr":"66.102.7.0:52132","proto":"HTTP/1.1","method":"GET","host":"trail.summercollabs.com","uri":"/api/reply/data-ll-tsk_vdTBHbLBuSmzrcytY","headers":{"X-Forwarded-For":["66.102.7.82"],"X-Forwarded-Proto":["http"],"User-Agent":["Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)"],"Accept-Encoding":["gzip, deflate, br"]}},"headers":{"Date":["Wed, 29 Sep 2021 23:21:54 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["42"],"Content-Type":["image/gif"]},"status":200}
{"level":"debug","ts":1632957714.7891092,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42160","request":{"remote_addr":"146.112.163.0:31964","proto":"HTTP/1.1","method":"GET","host":"track.betterthanaletter.co","uri":"/api/reply/data-ll-tsk_R5W3z42rwibrSNnY6","headers":{"X-Forwarded-For":["146.112.163.34"],"X-Forwarded-Proto":["http"],"Pragma":["no-cache"],"Cache-Control":["no-cache"],"Upgrade-Insecure-Requests":["1"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.212 Safari/537.36"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"],"Accept-Encoding":["gzip, deflate"]}},"headers":{"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["42"],"Content-Type":["image/gif"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"]},"status":200}
{"level":"debug","ts":1632957714.9001927,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42100","request":{"remote_addr":"141.101.68.0:11840","proto":"HTTP/2.0","method":"POST","host":"app.lemlist.com","uri":"/api/linkedin/updateUser","headers":{"Cdn-Loop":["cloudflare"],"Cf-Ipcountry":["FR"],"X-Forwarded-For":["2a01:cb00:3e8:7500:74d4:179:5a51:8372, 141.101.68.135"],"Cf-Ray":["6968f3549c6e3b13-CDG"],"Content-Length":["5080"],"Sec-Ch-Ua":["\"Chromium\";v=\"94\", \"Google Chrome\";v=\"94\", \";Not A Brand\";v=\"99\""],"Origin":["chrome-extension://khnbclggeggefodgimdekejhipkeobnc"],"Accept-Encoding":["gzip"],"X-Forwarded-Proto":["https"],"Accept":["application/json"],"Sec-Ch-Ua-Platform":["\"macOS\""],"Sec-Fetch-Site":["cross-site"],"Cf-Visitor":["{\"scheme\":\"https\"}"],"Sec-Ch-Ua-Mobile":["?0"],"User-Agent":["Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36"],"Sec-Fetch-Dest":["empty"],"Accept-Language":["fr,en;q=0.9,fr-FR;q=0.8,en-US;q=0.7"],"Cf-Connecting-Ip":["2a01:cb00:3e8:7500:74d4:179:5a51:8372"],"Content-Type":["application/json"],"Sec-Fetch-Mode":["cors"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","proto_mutual":true,"server_name":"app.lemlist.com"}},"headers":{"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["2"],"Content-Type":["application/json"],"Access-Control-Allow-Headers":["*"],"Access-Control-Allow-Origin":["*"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"]},"status":200}
{"level":"debug","ts":1632957714.9680083,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42140","request":{"remote_addr":"66.102.7.0:39262","proto":"HTTP/1.1","method":"GET","host":"trail.summercollabs.com","uri":"/api/files/Files/fil_LAAbat2tpdxyz6Lg9.png","headers":{"X-Forwarded-For":["66.102.7.82"],"X-Forwarded-Proto":["http"],"User-Agent":["Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)"],"Accept-Encoding":["gzip, deflate, br"]}},"headers":{"Content-Length":["55482"],"X-Content-Type-Options":["nosniff"],"Access-Control-Allow-Origin":["*"],"Content-Type":["image/png"],"Date":["Wed, 29 Sep 2021 23:21:54 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"]},"status":200}
{"level":"debug","ts":1632957715.1272316,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42140","request":{"remote_addr":"20.53.104.0:13378","proto":"HTTP/1.1","method":"GET","host":"track.luckemasks.com","uri":"/favicon.ico","headers":{"User-Agent":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"],"Accept":["*/*"],"Referer":["http://track.luckemasks.com/api/t/c/usr_tv6BfosizmjxEKRcb/tsk_kuxuosEgnNnAt5gLo/ZW5jX1UyRn"],"Accept-Encoding":["gzip, deflate"],"Accept-Language":["en-US,en;q=0.9"],"X-Forwarded-For":["20.53.104.64"],"X-Forwarded-Proto":["http"]}},"headers":{"X-Content-Type-Options":["nosniff"],"X-Frame-Options":["DENY"],"Content-Security-Policy":["default-src 'self' blob: https://zapier.com https://*.zapier.com https://*.linkedin.com https://chefsimon.co https://*.hotjar.com;"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"]},"status":404}
{"level":"debug","ts":1632957715.216518,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42140","request":{"remote_addr":"67.247.235.0:52520","proto":"HTTP/1.1","method":"GET","host":"trail.macproductsusa.co","uri":"/api/image-templates/itp_2ndXvWPAdGhmQLnfG/lea_vAXa3xm5fAr69f9Pv?t=1632941597056&r=9ay5QYHiscdobc3AC","headers":{"X-Forwarded-For":["67.247.235.149"],"Accept":["image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"],"Accept-Language":["en-us"],"X-Forwarded-Proto":["http"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"]}},"headers":{"Access-Control-Allow-Origin":["*"],"Content-Type":["image/jpeg"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["56466"]},"status":200}
{"level":"debug","ts":1632957715.2320747,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42160","request":{"remote_addr":"67.247.235.0:52521","proto":"HTTP/1.1","method":"GET","host":"trail.macproductsusa.co","uri":"/api/reply/data-ll-tsk_jzpsRmPgMxBQDJSSB","headers":{"Accept":["image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"],"Accept-Language":["en-us"],"X-Forwarded-Proto":["http"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"],"X-Forwarded-For":["67.247.235.149"]}},"headers":{"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["42"],"Content-Type":["image/gif"]},"status":200}
{"level":"debug","ts":1632957715.2842534,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42100","request":{"remote_addr":"35.175.221.0:41316","proto":"HTTP/1.1","method":"GET","host":"api.lemlist.com","uri":"/api/unsubscribes/msturdivant%40needld.org","headers":{"Authorization":["Basic dGVhX2FOeEM2YzdTZGhjbWJaRkw4OmM1MmJkNjQwZDc0ZGJjNzk4MjNlMTNmZjgwM2E4ZDAw"],"Accept":["*/*"],"Accept-Encoding":["gzip,deflate"],"X-Forwarded-For":["35.175.221.227"],"X-Forwarded-Proto":["https"],"User-Agent":["Zapier"]},"tls":{"resumed":true,"version":772,"cipher_suite":4865,"proto":"","proto_mutual":true,"server_name":"api.lemlist.com"}},"headers":{"X-Ratelimit-Reset":["Thu Sep 30 2021 01:21:56 GMT+0200 (Central European Summer Time)"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Keep-Alive":["timeout=5"],"Content-Length":["64"],"Retry-After":["1.353"],"X-Ratelimit-Limit":["20"],"X-Ratelimit-Remaining":["18"],"Content-Type":["application/json"],"Connection":["keep-alive"]},"status":200}
{"level":"debug","ts":1632957715.288194,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42160","request":{"remote_addr":"67.247.235.0:52522","proto":"HTTP/1.1","method":"GET","host":"trail.macproductsusa.co","uri":"/api/track/open/usr_nD5DBZ8bKN4TzR8xi/tsk_jzpsRmPgMxBQDJSSB","headers":{"X-Forwarded-Proto":["http"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"],"X-Forwarded-For":["67.247.235.149"],"Accept":["image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"],"Accept-Language":["en-us"]}},"headers":{"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["42"],"Content-Type":["image/gif"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"]},"status":200}
{"level":"debug","ts":1632957715.2888951,"logger":"tls.issuance.acme.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/new-order","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/(devel) CertMagic acmez (linux; amd64)"]},"response_headers":{"Boulder-Requester":["19998790"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["354"],"Content-Type":["application/json"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Location":["https://acme-staging-v02.api.letsencrypt.org/acme/order/19998790/649314478"],"Replay-Nonce":["0001llc0uGe6j3tJc_MV8emEdRZl1gOoBdu0LfSdPpURGeA"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]},"status_code":201}
{"level":"debug","ts":1632957715.3359973,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42140","request":{"remote_addr":"67.247.235.0:52523","proto":"HTTP/1.1","method":"GET","host":"trail.macproductsusa.co","uri":"/api/froala/files/tea_omWQhMPnbLhiiadDb-63c0af79455b937d7734f85cc21e03d8","headers":{"User-Agent":["Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"],"X-Forwarded-For":["67.247.235.149"],"Accept":["image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"],"Accept-Language":["en-us"],"X-Forwarded-Proto":["http"],"Accept-Encoding":["gzip, deflate"]}},"headers":{"Content-Type":["image/png"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"Content-Length":["1442"],"X-Content-Type-Options":["nosniff"],"Access-Control-Allow-Origin":["*"]},"status":200}
{"level":"debug","ts":1632957715.3950064,"logger":"http.handlers.reverse_proxy","msg":"upstream roundtrip","upstream":"localhost:42140","request":{"remote_addr":"20.53.104.0:13379","proto":"HTTP/1.1","method":"GET","host":"track.luckemasks.com","uri":"/favicon.ico","headers":{"X-Forwarded-For":["20.53.104.64"],"X-Forwarded-Proto":["http"],"User-Agent":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"],"Accept":["*/*"],"Referer":["http://track.luckemasks.com/api/t/c/usr_tv6BfosizmjxEKRcb/tsk_kuxuosEgnNnAt5gLo/ZW5jX1UyRn"],"Accept-Encoding":["gzip, deflate"],"Accept-Language":["en-US,en;q=0.9"]}},"headers":{"Connection":["keep-alive"],"Keep-Alive":["timeout=5"],"X-Content-Type-Options":["nosniff"],"X-Frame-Options":["DENY"],"Content-Security-Policy":["default-src 'self' blob: https://cdn.jsdelivr.net/gh/froala/design-blocks@master/screenshots http://*.google-analytics.com https://*.google-analytics.com https://zapier.com https://*.zapier.com https://*.linkedin.com https://chefsimon.co https://*.hotjar.com;"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"]},"status":404}
{"level":"debug","ts":1632957715.4222934,"logger":"tls.issuance.acme.acme_client","msg":"http request","method":"POST","url":"https://acme-staging-v02.api.letsencrypt.org/acme/authz-v3/596702288","headers":{"Content-Type":["application/jose+json"],"User-Agent":["Caddy/(devel) CertMagic acmez (linux; amd64)"]},"response_headers":{"Boulder-Requester":["19998790"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["106"],"Content-Type":["application/problem+json"],"Date":["Wed, 29 Sep 2021 23:21:55 GMT"],"Link":["<https://acme-staging-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["0001O5Ia0UYSzC-MsEbCSCI-g4IM5UrzB98keBeLXYOGFCs"],"Server":["nginx"]},"status_code":404}
{"level":"info","ts":1632957715.4223874,"logger":"tls.renew","msg":"releasing lock","identifier":"lemtrail.statrys.com"}

The panic seems to be coming from here:

and that should always be set as an Authorization type if the authorization is Invalid. But it’s nil. Hmm.

Are there any logs that say “added certificate to cache” where the “subjects” field is an empty array []? That could be helpful.

Hello,

nop.
I was also expecting this, but nothing, and I grep all logs from start to panic.
only occurence of *subjects":[] is the line I already pasted.

1 Like

Odd. I will look into this more after the weekend.