Reverse_proxy abort error log spam

Help
1

1. The problem I’m having:

My logs (and disks by extension) are getting hammered by http.handlers.reverse_proxy. Three of these per second. Enabling debug doesn’t output anything more.

2. Error messages and/or full log output:

{"level":"error","ts":1678879726.6404388,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879727.5201046,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879728.2465327,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879728.4619973,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879728.8975136,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879729.4509084,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879729.5613203,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879729.995524,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879730.1294448,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879730.819954,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879730.9830859,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879731.4404242,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879731.5852535,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879732.096478,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"context canceled"}
{"level":"error","ts":1678879732.196998,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879732.7978745,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879732.9191403,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}
{"level":"error","ts":1678879733.3612697,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","error":"client disconnected"}

3. Caddy version:

2.6.4

4. How I installed and ran Caddy:

I am using the hotio image, custom binary with cloudflare and replace-response.

a. System environment:

Docker, 64-bit intel, unRAID

b. Command:

c. Service/unit/compose file:

d. My complete Caddy config:

{
	http_port 8080
	https_port 8443
	order replace after encode
}
# replace-response + cloudflare 
(cf) {
	tls {
		resolvers 1.1.1.1
		dns cloudflare redacted-token
	}
}

(safe) {
	log {
		output file /config/logs/access.log
	}

	header {
		# disable FLoC tracking
		Permissions-Policy interest-cohort=()
		# enable HSTS
		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
		# disable clients from sniffing the media type
		X-Content-Type-Options nosniff
		# clickjacking protection
		X-Frame-Options DENY

		# keep referrer data off of HTTP connections
		Referrer-Policy no-referrer-when-downgrade
	}
}

(theme) {
	# 0 = app, 1 = theme, 2 = target, 3 = tag
	replace {
	  "</{args.3}>" "<link rel='stylesheet' type='text/css' href='https://theme-park.dev/css/base/{args.0}/plex.css'></{args.3}>"
	}

	reverse_proxy {args.2} {
		header_up Accept-Encoding identity
		header_down -x-webkit-csp
		header_down -content-security-policy
	}
}

nicholai.dev {
	import safe
	import cf
	reverse_proxy nicholai_dev:1312
}

home.nicholai.dev {
	import safe
	import cf
	root * /config/home
	file_server
	respond / "Welcome home." 418
}

booksonic.nicholai.dev {
	import safe
	import cf
	reverse_proxy booksonic:4040
}

status.nicholai.dev {
	import safe
	import cf
	import theme "uptime-kuma" "plex" status:3001 "head"
}

maria.nicholai.dev {
	import safe
	import cf
	reverse_proxy phpmyadmin
}

paste.nicholai.dev {
	import safe
	import cf
	handle /logo.png {
		root * /config/haste
		file_server
	}

	reverse_proxy haste:7777
}

tv.nicholai.dev {
	import safe
	import cf
	import theme "sonarr" "plex" 192.168.1.42:8989 "head"
}

movies.nicholai.dev {
	import safe
	import cf
	import theme "radarr" "plex" 192.168.1.42:7878 "head"
}

qb.nicholai.dev {
	import safe
	import cf
	import theme "vuetorrent" "plex" 192.168.1.42:8081 "body"
}

os.nicholai.dev {
	import cf
	log {
		output file /config/logs/access.log
		level WARN
	}

	reverse_proxy 192.168.1.42:5055

	header {
		Strict-Transport-Security "max-age=31536000; includeSubDomains;"
		Permissions-Policy interest-cohort=()
		X-Frame-Options "SAMEORIGIN"
		X-Content-Type-Options "nosniff"
		Referrer-Policy "strict-origin"
		X-Robots-Tag "noindex, nofollow, nosnippet, noarchive"
		Content-Security-Policy "default-src 'self'; connect-src 'self' https://plex.tv https://api.github.com; style-src 'self' 'unsafe-inline' https://rsms.me/inter/inter.css; script-src 'self' 'unsafe-inline'; img-src 'self' data: https://plex.tv https://assets.plex.tv https://gravatar.com https://secure.gravatar.com https://i2.wp.com https://image.tmdb.org; font-src 'self' https://rsms.me/inter/font-files/ https://fonts.gstatic.com;"
	}
}

li.nicholai.dev {
	import safe
	import cf
	import theme "lidarr" "plex" 192.168.1.42:8686 "head"
}

pr.nicholai.dev {
	import safe
	import cf
	import theme "prowlarr" "plex" 192.168.1.42:9696 "head"
}

ad.nicholai.dev {
	import safe
	import cf
	import theme "adguard" "plex" AdGuard-Home:3000 "head"
}

lyd.nicholai.dev {
	import safe
	import cf
	reverse_proxy audiobookshelf:80
}

calibre.nicholai.dev {
	import safe
	import cf
	reverse_proxy calibre-web:8083
}

sky.nicholai.dev {
	import safe
	import cf
	rewrite /.well-known/carddav /remote.php/dav
	rewrite /.well-known/caldav /remote.php/dav

	reverse_proxy https://nextcloud {
		transport http {
			tls
			tls_insecure_skip_verify
		}
	}
}

5. Links to relevant resources:

2

They’re debug-level logs because they’re not errors. It’s simply that the client is connecting and then disconnecting before the proxying is complete. (3 per second isn’t too bad.) If you want you could block the clients that are misbehaving, but it’s otherwise no harm to your server.

3

But they’re not debug-level, they’re error-level logs for debug messages, and so I can’t filter them out by removing debug.
It’s just a very high level of (log) noise for what’s not errors or critical messages, for my non-enterprise use-case. This server is mostly idle but I can hear the literal noise of the hdd tick every second because of this :smiley:

1 Like
4

Oh, I misunderstood “Enabling debug doesn’t output anything more.” – you’re right (I was mobile when reading your message); the log level is error. My apologies.

The Go standard library actually panics when this happens, so we chose to emit an error instead.

We could maybe clean these up a bit, but I think context canceled is from the Go standard lib.

Log emissions can be controlled more granularly using JSON – would you be interested in that?

5

Oh yeah! I’m comfortable with JSON, I just haven’t looked into how the Caddy schema looks yet. Caddyfile has been too easy to use to even consider it so far. I can give it a shot with exclude. Is there a way to output the current Caddyfile as JSON for a starting point?

6

JSON isn’t necessary to do that.

You can configure logging via Caddyfile global options to exclude loggers by their name:

Yes, using the caddy adapt command.

1 Like
7

This topic was automatically closed 30 days after the last reply. New replies are no longer allowed.