OPNSense - High availability - Websockets issue

Help
1

1. The problem I’m having:

I have a highly available OPNSense setup running in VMs on Proxmox. The issue I am having is regarding Websockets:

  1. Proxmox host WebUI shell drops connection, does not migrate
  2. Jellyfin video playback stops when connection is dropped due to Websockets issue.

2. Error messages and/or full log output:

I couldn't produce an error

3. Caddy version:

| **

2.0.4_3**

4. How I installed and ran Caddy:

as a Plugin in OPNSense

a. System environment:

OPNSense

b. Command:

N/A

c. Service/unit/compose file:

N/A

d. My complete Caddy config:

# DO NOT EDIT THIS FILE -- OPNsense auto-generated file


# caddy_user=root

# Global Options
{
	log {
		output net unixgram//var/run/caddy/log.sock {
		}
		format json {
			time_format rfc3339
		}
	}

	servers {
		protocols h1 h2
		trusted_proxies static 172.17.0.0/24 173.245.48.0/20 103.21.244.0/22 103.31.4.0/22 103.22.200.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22
		client_ip_headers Connection
		client_ip_headers Host
		client_ip_headers Upgrade
		client_ip_headers X-Real-IP
	}

	email internal@mstack.dev
	grace_period 10s
	skip_install_trust
	import /usr/local/etc/caddy/caddy.d/*.global
}

# Reverse Proxy Configuration


*.casalab.cc {
	tls /usr/local/etc/caddy/certificates/68a508597add7.pem /usr/local/etc/caddy/certificates/68a508597add7.key {
	}
}

*.int.casalab.cc {
	tls /usr/local/etc/caddy/certificates/68a5115deb631.pem /usr/local/etc/caddy/certificates/68a5115deb631.key {
	}
}

opn-cerberus.int.casalab.cc {
	handle {
		reverse_proxy https://10.0.0.1:4443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

aegis.int.casalab.cc {
	handle {
		reverse_proxy https://10.0.2.102:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

adguard-cerberus.int.casalab.cc {
	handle {
		reverse_proxy 10.0.0.1:3000 {
			transport http {
			}
		}
	}
}

ai.casalab.cc {
	handle {
		reverse_proxy 10.2.1.2:3000 {
			transport http {
			}
		}
	}
}

kasm.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.3:443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

opn-aegis.int.casalab.cc {
	handle {
		reverse_proxy https://10.0.0.2:4443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

cerberus.int.casalab.cc {
	handle {
		reverse_proxy https://10.0.2.101:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

zeus.int.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.101:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

vaultwarden.casalab.cc {
	handle {
		reverse_proxy https://10.1.1.6:8000 {
			lb_policy first
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

odin-portainer.int.casalab.cc {
	handle {
		reverse_proxy https://10.1.1.2:9443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

jellyfin.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:8096 {
			header_down Connection "keep-alive"
			header_down X-Accel-Buffering "no"
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 300s
			}
		}
	}
}

jellyseer.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:5055 {
		}
	}
}

asus.int.casalab.cc {
	handle {
		reverse_proxy https://10.254.254.1:8443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

cloud.casalab.cc {
	handle {
		reverse_proxy 10.1.1.2:8080 {
			transport http {
			}
		}
	}
}

adguard-aegis.int.casalab.cc {
	handle {
		reverse_proxy 10.0.0.2:3000 {
		}
	}
}

horaco-attic.int.casalab.cc {
	handle {
		reverse_proxy 10.0.2.3 {
			transport http {
			}
		}
	}
}

horaco-lounge.int.casalab.cc {
	handle {
		reverse_proxy 10.0.2.4 {
		}
	}
}

omv.int.casalab.cc {
	handle {
		reverse_proxy 10.1.1.1 {
		}
	}
}

tplink-attic.int.casalab.cc {
	handle {
		reverse_proxy 10.0.2.5 {
		}
	}
}

kvm-cerberus.int.casalab.cc {
	handle {
		reverse_proxy 10.5.1.11 {
		}
	}
}

kvm-aegis.int.casalab.cc {
	handle {
		reverse_proxy 10.5.1.22 {
		}
	}
}

sentry.int.casalab.cc {
	handle {
		reverse_proxy https://10.5.1.102:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

zeus-portainer.int.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.1:9443 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

dashy.casalab.cc {
	handle {
		reverse_proxy 10.5.1.1:4000 {
		}
	}
}

prometheus-alert.int.casalab.cc {
	handle {
		reverse_proxy 10.5.1.3:9093 {
		}
	}
}

image.casalab.cc {
	handle {
		reverse_proxy 10.2.1.1:9090 {
		}
	}
}

pbs.int.casalab.cc {
	handle {
		reverse_proxy https://10.1.1.4:8007 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
			}
		}
	}
}

sunshine.int.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.1:47990 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

catalyst.int.casalab.cc {
	handle {
		reverse_proxy 10.0.2.10 {
			transport http {
			}
		}
	}
}

ollama.casalab.cc {
	handle {
		reverse_proxy 10.2.1.1:1234 {
			transport http {
			}
		}
	}
}

blender.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.1:3004 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

orca.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.1:3002 {
			transport http {
				tls_insecure_skip_verify
			}
		}
	}
}

cloud.int.casalab.cc {
}

sabnzbd.int.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:8282 {
		}
	}
}

pdf.casalab.cc {
	handle {
		reverse_proxy /outpost.goauthentik.io/* http://10.1.1.5:8022 {
		}
		forward_auth http://10.1.1.5:8022 {
			uri /outpost.goauthentik.io/auth/caddy
			copy_headers X-Authentik-Username
			copy_headers X-Authentik-Groups
			copy_headers X-Authentik-Email
			copy_headers X-Authentik-Name
			copy_headers X-Authentik-Uid
			copy_headers X-Authentik-Jwt
			copy_headers X-Authentik-Meta-Jwks
			copy_headers X-Authentik-Meta-Outpost
			copy_headers X-Authentik-Meta-Provider
			copy_headers X-Authentik-Meta-App
			copy_headers X-Authentik-Meta-Version
		}
		reverse_proxy 10.1.1.5:8083 {
		}
	}
}

authentik.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:8022 {
			transport http {
			}
		}
	}
}

auth.int.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:8022 {
		}
	}
}

ha.casalab.cc {
	handle {
		reverse_proxy 10.5.1.40:8123 {
		}
	}
}

argus.int.casalab.cc {
	handle {
		reverse_proxy https://10.2.1.102:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

apollo.int.casalab.cc {
	handle {
		reverse_proxy https://10.1.1.103:8006 {
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

ntfy.casalab.cc {
	handle {
		reverse_proxy 10.5.1.7 {
		}
	}
}

odin.int.casalab.cc {
	handle {
		reverse_proxy https://10.1.1.101:8006 {
			header_up Connection {http.request.header.Connection}
			header_up Host {http.reverse_proxy.upstream.host}
			header_up Upgrade {http.request.header.Upgrade}
			header_up X-Real-IP {remote_host}

			transport http {
				versions 1.1
				keepalive 500s
				tls_insecure_skip_verify
			}
		}
	}
}

immich.casalab.cc {
	handle {
		reverse_proxy 10.1.1.3:2283 {
			header_up X-Real-IP {remote_host}
		}
	}
}

fastpoe.int.casalab.cc {
	handle {
		reverse_proxy 10.0.2.9 {
		}
	}
}

frigate.casalab.cc {
	handle {
		reverse_proxy 10.1.1.4:5000 {
		}
	}
}

qbit.int.casalab.cc {
	handle {
		reverse_proxy 10.1.1.5:8181 {
		}
	}
}

game {
	handle {
		reverse_proxy https://10.2.1.1:47990 {
		}
	}
}

import /usr/local/etc/caddy/caddy.d/*.conf

5. Links to relevant resources:

2

Hello,

I do not know what your expectation is, but there is no sync between multiple caddy instances when running them on opnsense. There is also no shared storage between them.

They’re just separate instances that run at the same time, and whoever is the current CARP master and holds the primary IP addresses will get the connections.

1 Like