Transition 2.5.2 > 2.6.0 getting responds 200 but 0 bytes?

1. Output of caddy version:

v2.6.0 h1:lHDynvM+sTOi9Aq4Y15b4FtkqzPB36WbUrZvVdwzTCA=

With Plugins
caddy-dns/cloudflare
caddyserver/transform-encoder

2. How I run Caddy:

as a service to check if caddy is running if not start it

a. System environment:

Windows 11

b. Command:

caddy run

d. My complete Caddy config:

(theheaders) {
        header X-Content-Type-Options "nosniff"
        header X-Xss-Protection "1; mode=block"
        header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
        header Cache-Control "no-store, no-cache, must-revalidate, max-age=0"
        header Pragma "no-cache"
        header X-Frame-Options "SAMEORIGIN"
        header Permissions-Policy "accelerometer=(), ambient-light-sensor=(), autoplay=(self), camera=(), encrypted-media=(), fullscreen=(self), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(*), speaker=(), sync-xhr=(), usb=(), vr=()"

        header X-Robots-Tag "none"
        header Access-Control-Max-Age "86400"

        header /* {
                -Server
        }
}

:80 {
        log {
                format transform [{ts}] - User={user_id} - X-Forwarded-For={request>headers>X-Forwarded-For} - remote_ip={request>remote_ip} Country={request>headers>Cf-Ipcountry} {request>method} {request>headers>X-Forwarded-Proto} {request>host} {request>uri} {request>headers>Referer>[0]} {request>headers>User-Agent>[0]} - {request>proto} {status} {size} - {
                        #{request>headers} <--- add this if you want full log
                        time_format "02/Jan/2006 15:04:05 -0700"
                }

                output file C:\Users\wazer\Desktop\caddy\newnotworks\logs\domain.log {
                        roll true # Rotate logs, enabled by default
                        roll_size_mb 5 # Set max size 5 MB
                        roll_gzip true # Whether to compress rolled files
                        #roll_local_time true # Use localhost time
                        roll_keep 2 # Keep at most 2 log files
                        roll_keep_days 7 # Keep log files for 7 days
                }
        }

        #SSL Settings
        tls testdummy@nonodummy.org {
                resolvers 1.1.1.1
        }
        import theheaders
        #header X-Real-IP {http.request.header.CF-Connecting-IP}
        #header X-Forwarded-For {http.request.header.CF-Connecting-IP}
        #header X-Forwarded-Host {http.request.hostport}
        encode gzip

        @blocked {
                #Only allow user-agents specified below and block everything else
                not header_regexp User-Agent (?i:UserAgent1*|UserAgent2*)

                #Allow lan ip's to skip user-agents block
                not remote_ip private_ranges

                #Allow wan ip's to skip user-agents block
                #not remote_ip forwarded WAN_IP
        }
        handle @blocked {
                abort
        }

        @UserAgent1 {
		        not remote_ip private_ranges
                #not remote_ip forwarded WAN_IP
                #Check if useragent is valid below then handle them correct file
                header_regexp User-Agent (?i:UserAgent1*)
        }

        @UserAgent1LAN {
                remote_ip private_ranges
                #remote_ip forwarded WAN_IP
                header_regexp User-Agent (?i:UserAgent1*)
        }

        @UserAgent2 {
		        not remote_ip private_ranges
                #not remote_ip forwarded WAN_IP
                #Check if useragent is valid below then handle them correct file
                header_regexp User-Agent (?i:UserAgent2*)
        }

        @UserAgent2LAN {
                remote_ip private_ranges
                #remote_ip forwarded WAN_IP
                header_regexp User-Agent (?i:UserAgent2*)
        }

        handle {
                @public {
                        #Allow lan ip's to skip basicAuth
                        #not remote_ip private_ranges
                        #Allow wan ip found in forwarded header skip basicAuth
                        #not remote_ip forwarded WAN_IP
                }
                basicauth @public {
                        #Works with 2.5.5 caddy
                        #dummy JDJhJDE0JDJ0REF2N2IyWUZ4cE96U1llbFdaVWVBdlZqbzlFb1I3MWVxMXRwdzM5QTNhOGdXRVZCbGVH # username and password is dummy

                        #Works with caddy 2.6.0 and so on
                        dummy $2a$14$xunbzZrcuN3HS0y.oBQzr.YuCtRG.aC9xNyKRvOZa2p41R3Py7n7W # username and password is dummy
                }
				
				
                handle @UserAgent1 {
                        # reads from @UserAgent1
                        handle_path /testout
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\testout.xml
                        file_server browse
                        rewrite * /testout
                }

                handle @UserAgent1LAN {
                        # reads from @UserAgent1LAN
                        handle_path /testin
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\testin.xml
                        file_server browse
                        rewrite * /testin
                }

                handle @UserAgent2 {
                        # reads from @UserAgent2
                        handle_path /testoutrec
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\testoutrec.xml
                        file_server browse
                        rewrite * /testoutrec
                }

                handle @UserAgent2LAN {
                        # reads from @UserAgent2LAN
                        handle_path /testinrec
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\testinrec.xml
                        file_server browse
                        rewrite * testinrec
                }
				
		handle_path /test2 {
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\test2.xml
                        file_server browse
                }


        }
}

3. The problem I’m having:

Getting responds 200 on files but 0 bytes ?

5. What I already tried:

I thought it was first because of the new basic auth syntax with hash-password being changed so I changed that, I also tried removing checks for ip and user-agents, with same result.

Rolling back to version 2.5.2 and it works again?
v2.5.2 h1:eCJdLyEyAGzuQTa5Mh3gETnYWDClo1LjtQm2q9RNZrs=
With Plugins
caddy-dns/cloudflare
caddyserver/transform-encoder

What am I missing here in the changes to migrate this over to latest version?

Can you provide more information? (See the steps in the help template you filled out.) What is a curl -v command that produces that kind of response (and what is its output)?

I updated the caddy template ConfigFile in OP to not use cloudflare and just use internal test since its easier for me do to it this way until I hope we can get this solved?

Okay something happen in the 2.6.1 hotfix instead of 2.6.0

I just tested its moving in progress and I actually get bytes now so something changed in the file_server? but its still not working as expected for me.

You will notice both XML text is not the same…

This is version 2.5.2 caddy respecting the url to /test2 ignoring all the other user-agent flags and goes straight to “handle_path /test2” and reads file test2.xml

curl -v -H "User-Agent:UserAgent1" http://dummy:dummy@192.168.1.3/test2

Output

*   Trying 192.168.1.3:80...
* Connected to 192.168.1.3 (192.168.1.3) port 80 (#0)
* Server auth using Basic with user 'dummy'
> GET /test2 HTTP/1.1
> Host: 192.168.1.3
> Authorization: Basic ZHVtbXk6ZHVtbXk=
> Accept: */*
> User-Agent:UserAgent1
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Accept-Ranges: bytes
< Access-Control-Max-Age: 86400
< Cache-Control: no-store, no-cache, must-revalidate, max-age=0
< Content-Length: 173
< Content-Type: text/xml; charset=utf-8
< Etag: "ril4o34t"
< Last-Modified: Thu, 22 Sep 2022 00:14:27 GMT
< Permissions-Policy: accelerometer=(), ambient-light-sensor=(), autoplay=(self), camera=(), encrypted-media=(), fullscreen=(self), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(*), speaker=(), sync-xhr=(), usb=(), vr=()
< Pragma: no-cache
< Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
< X-Content-Type-Options: nosniff
< X-Frame-Options: SAMEORIGIN
< X-Robots-Tag: none
< X-Xss-Protection: 1; mode=block
< Date: Thu, 22 Sep 2022 01:22:53 GMT
<
<?xml version="1.0" encoding="utf-8"?>
<s generator-info-data="DomainObjects/4.3.7998.25875">
  <data id="2">
    <display-data>data test2</display-data>
  </data>
</s>* Connection #0 to host 192.168.1.3 left intact

This is version 2.6.1 caddy not respecting url to /test2 and goes straight to another file where user-agent is pointed to “handle @UserAgent1” and reads file *testin.xml

curl -v -H "User-Agent:UserAgent1" http://dummy:dummy@192.168.1.3/test2

Output

*   Trying 192.168.1.3:80...
* Connected to 192.168.1.3 (192.168.1.3) port 80 (#0)
* Server auth using Basic with user 'dummy'
> GET /test2 HTTP/1.1
> Host: 192.168.1.3
> Authorization: Basic ZHVtbXk6ZHVtbXk=
> Accept: */*
> User-Agent:UserAgent1
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Accept-Ranges: bytes
< Access-Control-Max-Age: 86400
< Cache-Control: no-store, no-cache, must-revalidate, max-age=0
< Content-Length: 176
< Content-Type: text/xml; charset=utf-8
< Etag: "ril4t24w"
< Last-Modified: Thu, 22 Sep 2022 00:17:26 GMT
< Permissions-Policy: accelerometer=(), ambient-light-sensor=(), autoplay=(self), camera=(), encrypted-media=(), fullscreen=(self), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(*), speaker=(), sync-xhr=(), usb=(), vr=()
< Pragma: no-cache
< Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
< X-Content-Type-Options: nosniff
< X-Frame-Options: SAMEORIGIN
< X-Robots-Tag: none
< X-Xss-Protection: 1; mode=block
< Date: Thu, 22 Sep 2022 01:20:54 GMT
<
<?xml version="1.0" encoding="utf-8"?>
<s generator-info-data="DomainObjects/4.3.7998.25875">
  <data id="in">
    <display-data>data test in</display-data>
  </data>
</s>* Connection #0 to host 192.168.1.3 left intact

Would the fix be just to move

		handle_path /test2 {
                        root * C:\Users\wazer\Desktop\caddy\newnotworks\files\test2.xml
                        file_server browse
                }

Below this section

                @public {
                        #Allow lan ip's to skip basicAuth
                        #not remote_ip private_ranges
                        #Allow wan ip found in forwarded header skip basicAuth
                        #not remote_ip forwarded WAN_IP
                }
                basicauth @public {
                        #Works with 2.5.5 caddy
                        #dummy JDJhJDE0JDJ0REF2N2IyWUZ4cE96U1llbFdaVWVBdlZqbzlFb1I3MWVxMXRwdzM5QTNhOGdXRVZCbGVH # username and password is dummy

                        #Works with caddy 2.6.0 and so on
                        dummy $2a$14$xunbzZrcuN3HS0y.oBQzr.YuCtRG.aC9xNyKRvOZa2p41R3Py7n7W # username and password is dummy
                }

And always be above all handles with path in it? so all handle_path needs to be first priority before handle @?

		handle @? {
		}

It kinda seems weird if it would work, but also my config could have been wrong in 2.5.2 but simply just worked, being lucky it would take handle_path below any of the @handles when going to the specified url

There was a change to how sorting of directives happen, and it may have affected you in this situation.

You can run caddy adapt --pretty --config Caddyfile to see the adapted JSON output, and you’ll see the difference in how the routes get sorted in the final JSON output.

That does not make sense to me, in the json the pretty print still shows handle_path /test2 at the bottom?

Is there a function to do this in pure caddyfile format? or could it be done for us that likes caddyfile in order to fix such issues later on? :slight_smile:

{
	"logging": {
		"logs": {
			"default": {
				"exclude": [
					"http.log.access.log0"
				]
			},
			"log0": {
				"writer": {
					"filename": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\logs\\domain.log",
					"output": "file",
					"roll_keep": 2
				},
				"encoder": {
					"format": "transform",
					"template": "[{ts}] - User={user_id} - X-Forwarded-For={request>headers>X-Forwarded-For} - remote_ip={request>remote_ip} Country={request>headers>Cf-Ipcountry} {request>method} {request>headers>X-Forwarded-Proto} {request>host} {request>uri} {request>headers>Referer>[0]} {request>headers>User-Agent>[0]} - {request>proto} {status} {size} -",
					"time_format": "02/Jan/2006 15:04:05 -0700"
				},
				"include": [
					"http.log.access.log0"
				]
			}
		}
	},
	"apps": {
		"http": {
			"servers": {
				"srv0": {
					"listen": [
						":80"
					],
					"routes": [
						{
							"match": [
								{
									"path": [
										"/*"
									]
								}
							],
							"handle": [
								{
									"handler": "headers",
									"response": {
										"deferred": true,
										"delete": [
											"Server"
										]
									}
								}
							]
						},
						{
							"handle": [
								{
									"handler": "headers",
									"response": {
										"set": {
											"X-Content-Type-Options": [
												"nosniff"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"X-Xss-Protection": [
												"1; mode=block"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"Strict-Transport-Security": [
												"max-age=31536000; includeSubDomains; preload"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"Cache-Control": [
												"no-store, no-cache, must-revalidate, max-age=0"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"Pragma": [
												"no-cache"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"X-Frame-Options": [
												"SAMEORIGIN"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"Permissions-Policy": [
												"accelerometer=(), ambient-light-sensor=(), autoplay=(self), camera=(), encrypted-media=(), fullscreen=(self), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(*), speaker=(), sync-xhr=(), usb=(), vr=()"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"X-Robots-Tag": [
												"none"
											]
										}
									}
								},
								{
									"handler": "headers",
									"response": {
										"set": {
											"Access-Control-Max-Age": [
												"86400"
											]
										}
									}
								},
								{
									"encodings": {
										"gzip": {}
									},
									"handler": "encode",
									"prefer": [
										"gzip"
									]
								}
							]
						},
						{
							"group": "group12",
							"match": [
								{
									"not": [
										{
											"header_regexp": {
												"User-Agent": {
													"pattern": "(?i:UserAgent1*|UserAgent2*)"
												}
											}
										},
										{
											"remote_ip": {
												"ranges": [
													"192.168.0.0/16",
													"172.16.0.0/12",
													"10.0.0.0/8",
													"127.0.0.1/8",
													"fd00::/8",
													"::1"
												]
											}
										}
									]
								}
							],
							"handle": [
								{
									"handler": "subroute",
									"routes": [
										{
											"handle": [
												{
													"abort": true,
													"handler": "static_response"
												}
											]
										}
									]
								}
							]
						},
						{
							"group": "group12",
							"handle": [
								{
									"handler": "subroute",
									"routes": [
										{
											"handle": [
												{
													"handler": "authentication",
													"providers": {
														"http_basic": {
															"accounts": [
																{
																	"password": "$2a$14$xunbzZrcuN3HS0y.oBQzr.YuCtRG.aC9xNyKRvOZa2p41R3Py7n7W",
																	"username": "dummy"
																}
															],
															"hash": {
																"algorithm": "bcrypt"
															},
															"hash_cache": {}
														}
													}
												}
											],
											"match": [
												{}
											]
										},
										{
											"group": "group10",
											"handle": [
												{
													"handler": "subroute",
													"routes": [
														{
															"handle": [
																{
																	"handler": "vars",
																	"root": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\files\\testout.xml"
																}
															]
														},
														{
															"group": "group2",
															"handle": [
																{
																	"handler": "rewrite",
																	"uri": "/testout"
																}
															]
														},
														{
															"handle": [
																{
																	"handler": "subroute",
																	"routes": [
																		{
																			"handle": [
																				{
																					"handler": "rewrite",
																					"strip_path_prefix": "/testout"
																				}
																			]
																		}
																	]
																}
															],
															"match": [
																{
																	"path": [
																		"/testout"
																	]
																}
															]
														},
														{
															"handle": [
																{
																	"browse": {},
																	"handler": "file_server",
																	"hide": [
																		".\\Caddyfile"
																	]
																}
															]
														}
													]
												}
											],
											"match": [
												{
													"header_regexp": {
														"User-Agent": {
															"pattern": "(?i:UserAgent1*)"
														}
													},
													"not": [
														{
															"remote_ip": {
																"ranges": [
																	"192.168.0.0/16",
																	"172.16.0.0/12",
																	"10.0.0.0/8",
																	"127.0.0.1/8",
																	"fd00::/8",
																	"::1"
																]
															}
														}
													]
												}
											]
										},
										{
											"group": "group10",
											"handle": [
												{
													"handler": "subroute",
													"routes": [
														{
															"handle": [
																{
																	"handler": "vars",
																	"root": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\files\\testin.xml"
																}
															]
														},
														{
															"group": "group4",
															"handle": [
																{
																	"handler": "rewrite",
																	"uri": "/testin"
																}
															]
														},
														{
															"handle": [
																{
																	"handler": "subroute",
																	"routes": [
																		{
																			"handle": [
																				{
																					"handler": "rewrite",
																					"strip_path_prefix": "/testin"
																				}
																			]
																		}
																	]
																}
															],
															"match": [
																{
																	"path": [
																		"/testin"
																	]
																}
															]
														},
														{
															"handle": [
																{
																	"browse": {},
																	"handler": "file_server",
																	"hide": [
																		".\\Caddyfile"
																	]
																}
															]
														}
													]
												}
											],
											"match": [
												{
													"header_regexp": {
														"User-Agent": {
															"pattern": "(?i:UserAgent1*)"
														}
													},
													"remote_ip": {
														"ranges": [
															"192.168.0.0/16",
															"172.16.0.0/12",
															"10.0.0.0/8",
															"127.0.0.1/8",
															"fd00::/8",
															"::1"
														]
													}
												}
											]
										},
										{
											"group": "group10",
											"handle": [
												{
													"handler": "subroute",
													"routes": [
														{
															"handle": [
																{
																	"handler": "vars",
																	"root": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\files\\testoutrec.xml"
																}
															]
														},
														{
															"group": "group6",
															"handle": [
																{
																	"handler": "rewrite",
																	"uri": "/testoutrec"
																}
															]
														},
														{
															"handle": [
																{
																	"handler": "subroute",
																	"routes": [
																		{
																			"handle": [
																				{
																					"handler": "rewrite",
																					"strip_path_prefix": "/testoutrec"
																				}
																			]
																		}
																	]
																}
															],
															"match": [
																{
																	"path": [
																		"/testoutrec"
																	]
																}
															]
														},
														{
															"handle": [
																{
																	"browse": {},
																	"handler": "file_server",
																	"hide": [
																		".\\Caddyfile"
																	]
																}
															]
														}
													]
												}
											],
											"match": [
												{
													"header_regexp": {
														"User-Agent": {
															"pattern": "(?i:UserAgent2*)"
														}
													},
													"not": [
														{
															"remote_ip": {
																"ranges": [
																	"192.168.0.0/16",
																	"172.16.0.0/12",
																	"10.0.0.0/8",
																	"127.0.0.1/8",
																	"fd00::/8",
																	"::1"
																]
															}
														}
													]
												}
											]
										},
										{
											"group": "group10",
											"handle": [
												{
													"handler": "subroute",
													"routes": [
														{
															"handle": [
																{
																	"handler": "vars",
																	"root": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\files\\testinrec.xml"
																}
															]
														},
														{
															"group": "group8",
															"handle": [
																{
																	"handler": "rewrite",
																	"uri": "testinrec"
																}
															]
														},
														{
															"handle": [
																{
																	"handler": "subroute",
																	"routes": [
																		{
																			"handle": [
																				{
																					"handler": "rewrite",
																					"strip_path_prefix": "/testinrec"
																				}
																			]
																		}
																	]
																}
															],
															"match": [
																{
																	"path": [
																		"/testinrec"
																	]
																}
															]
														},
														{
															"handle": [
																{
																	"browse": {},
																	"handler": "file_server",
																	"hide": [
																		".\\Caddyfile"
																	]
																}
															]
														}
													]
												}
											],
											"match": [
												{
													"header_regexp": {
														"User-Agent": {
															"pattern": "(?i:UserAgent2*)"
														}
													},
													"remote_ip": {
														"ranges": [
															"192.168.0.0/16",
															"172.16.0.0/12",
															"10.0.0.0/8",
															"127.0.0.1/8",
															"fd00::/8",
															"::1"
														]
													}
												}
											]
										},
										{
											"group": "group10",
											"handle": [
												{
													"handler": "subroute",
													"routes": [
														{
															"handle": [
																{
																	"handler": "rewrite",
																	"strip_path_prefix": "/test2"
																}
															]
														},
														{
															"handle": [
																{
																	"handler": "vars",
																	"root": "C:\\Users\\wazer\\Desktop\\caddy\\newnotworks\\files\\test2.xml"
																},
																{
																	"browse": {},
																	"handler": "file_server",
																	"hide": [
																		".\\Caddyfile"
																	]
																}
															]
														}
													]
												}
											],
											"match": [
												{
													"path": [
														"/test2"
													]
												}
											]
										}
									]
								}
							]
						}
					],
					"logs": {
						"default_logger_name": "log0"
					}
				}
			}
		}
	}
}