Need help converting NGINX config

1. My Caddy version (caddy version):

v2.0.0-beta.20 h1:oUNG1uh0UV8LWLlAVDZolFzk112++V/pxY+fF0HLmlY=

2. How I run Caddy:

Just testing so sudo caddy run for now or when using caddyfile, sudo caddy run --config ~/caddy/Caddyfile

a. System environment:

Ubuntu 18.04

d. My complete Caddyfile or JSON config:

I have been trying both:
Caddyfile

localhost {
  reverse_proxy /weather/* localhost:3010 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /ptz/* localhost:3006 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /liveview/* localhost:3004 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /archive/* localhost:3003 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /alarms/* localhost:3002 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /web_app_socket/* localhost:3001 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
  reverse_proxy /* localhost:3001 {
    header_up Host {host}
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-For {remote_host}
    header_up X-Forwarded-Proto {scheme}
  }
}

JSON Config file

{
  "apps": {
    "http": {
      "servers": {
        "intelliview": {
          "listen": [":443"],
          "routes": [
            {
              "handle": [
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/web_app_socket/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3001"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/console_socket/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3001"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/console_socket/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3001"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/alarms/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3002"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/analytics/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3005"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/archive/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3003"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/liveview/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3004"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/ptz/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3006"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/weather/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3010"
                    }
                  ]
                },
                {
                  "handler": "reverse_proxy",
                  "health_checks": {
                    "active": {
                      "expect_status": 2,
                      "path": "/"
                    }
                  },
                  "transport": {
                    "protocol": "http",
                    "read_buffer_size": 4096
                  },
                  "upstreams": [
                    {
                      "dial": "localhost:3001"
                    }
                  ]
                }
              ],
              "match": [{
                "host": [
                  "localhost"
                ]
              }]
            }
          ]
        }
      }
    }
  }
}

3. The problem I’m having:

Trying to convert NGINX config to something that works with Caddy. I was tasked 2 days to research. I have been playing around. I got to point where website (SPA) uploads to browser, but api calls and websockets (socket.io) are not working.

Here is the NGINX config:

map $http_upgrade $connection_upgrade {
    default upgrade;
    ''      close;
}

########### Upstreams ###########

upstream web_app {
    ip_hash;
    server localhost:3001;
}

upstream alarms {
    ip_hash;
    server localhost:3002;
}

upstream analytics {
    ip_hash;
    server localhost:3005;
}

upstream archive {
     ip_hash;
     server localhost:3003;
}

upstream liveview {
    ip_hash;
    server localhost:3004;
}

upstream ptz {
    ip_hash;
    server localhost:3006;
}

upstream weather {
    ip_hash;
    server localhost:3010;
}

# upstreams
include sites-available/*.upstream;

server {

############# s0 on port 80 (HTTP) #############
    listen       80;
    listen  [::]:80;

    server_name _;

    charset utf-8;

    gzip            on;
    gzip_types      text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
    gzip_min_length 1000;
    gzip_proxied    any;

    ############ include all dynamically created configs ##############
    include sites-available/*.conf;

    location / {
        proxy_pass http://web_app/;
        #HTTP version 1.1 is needed for sockets
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

############## socket for web server communications ###############		
    location /web_app_socket/ { ### route the websockets of the web app
        #Configure proxy to pass data to upstream node1
        proxy_pass http://web_app/web/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /console_socket/ { ### route the websockets of the web app
        #Configure proxy to pass data to upstream node1
        proxy_pass http://web_app/console/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /alarms/ { 
        #Configure proxy to pass data to upstream node1
        proxy_pass http://alarms/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /analytics/ {
        #Configure proxy to pass data to upstream node1
        proxy_pass http://analytics/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /archive/ {
        #Configure proxy to pass data to upstream node1
        proxy_pass http://archive/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /liveview/ {
        #Configure proxy to pass data to upstream node1
        proxy_pass http://liveview/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /ptz/ {
        #Configure proxy to pass data to upstream node1
        proxy_pass http://ptz/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }

    location /weather/ {
        #Configure proxy to pass data to upstream node1
        proxy_pass http://weather/socket.io/;
        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header Referer $http_referer;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto  $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_read_timeout 600s;
        proxy_connect_timeout 600s;
    }
}

The hope is to get something that works with the JSON. We want to be able to reverse_proxy dynamically to upstream servers. Currently, we write NGINX files and restart the server to do this. We also want HTTPS as our default, which we have had issues with NGINX and our websocket connections (they stop without error notification).

Any help, or getting me on the right track, would be appreciated…

Thanks,
Jeff

I think, maybe, I need to set http 1.1 for the websockets, but I had difficulty finding it in the v2 docs.

Welcome Jeff!

FYI, you can remove:

header_up Host {host}
header_up X-Forwarded-For {remote_host}

As those are the default anyway.

Have you seen/tried this? GitHub - caddyserver/nginx-adapter: Run Caddy with your NGINX config

but api calls and websockets (socket.io) are not working.

Saying “not working” isn’t going to help us help you – what isn’t working? What’s the error message? How can it be reproduced?

The socket request with GET is getting the main website (index,html)

https://localhost/web_app_socket/?EIO=3&transport=polling&t=N4tDy7D

A socket with POST fails

I heard about the NGINX adapter, I will look into it

The nginx_adapter failed to build, so I created an issue: Fails to build · Issue #11 · caddyserver/nginx-adapter · GitHub

That may work for me, if I can get it resolved.

In the meantime, any ideas why my sockets are not connecting properly?

image

I don’t see an Upgrade header in your websocket request. Should you not be using wss:// instead of https:// in the browser?

If I put this in browser, it redirects incorrectly to site: https://localhost/web_app_socket

The sockets start with polling (http/https) and upgrade automatically (to ws/wss). These are embedded devices and sometimes get on really bad networks (thing mobile edge). If they don’t start with polling, then a connection cannot be made to downgrade, so the reverse is better.

What is the Upgrade?
and do you know if I can for http1.1 on the socket?

The Upgrade header is what’s used by the browser to tell the server to change wss:// from an HTTP connection into a persistent TCP connection.

Caddy will make any websocket connection use HTTP 1.1 automatically, that’s unnecessary in this situation.

I’m not sure I understand the issue here. If you use https://, it’ll make an HTTPS connection, not websocket.

Maybe you meant to do:

reverse_proxy /web_app_socket/* localhost:3001/web/socket.io/ {
    header_up X-Real-IP {remote_host}
    header_up X-Forwarded-Proto {scheme}
}

And you might possibly want to add this to force websockets if that’s what you need:

    header_up Connection Upgrade
    header_up Upgrade websocket

Caddy forwards any headers by default so you don’t need the bit you have in your nginx config, i.e.

        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;

Thank you for the information.

ERROR	http.log.error
making dial info: upstream localhost:3001/web/socket.io/: 
invalid dial address localhost:3001/web/socket.io/: 
address web/socket.io/: missing port in address

In our NGINX config, there is this comment:

        #HTTP version 1.1 is needed for sockets
        proxy_http_version 1.1;

Any idea how to force HTTP/1.1 iwith Caddy v2?

I feel like I am close. Just missing a few things. I do appreciate all the help. :heart:

Hmm…

Maybe something like this:

route /web_app_socket/* {
	uri * strip_prefix /web_app_socket
	rewrite * /web/socket.io{uri}
	reverse_proxy * http://localhost:3001 {
		header_up X-Real-IP {remote_host}
		header_up X-Forwarded-Proto {scheme}
	}
}

To explain: we want to match requests for /web_app_socket/*, but send them to the backend as /web/socket.io/* (as per your nginx config). Currently, it seems like Caddy v2 doesn’t support having a path at the end of a proxy dial address (it did in v1, but for implementation reasons, doesn’t in v2 yet).

To do this, I use a route block which allows fine-grained control over the order in which things are run. In this case, I want to rewrite to strip the /web_app_socket prefix from the URI, then rewrite it to add the /web/socket.io prefix instead.

@matt I’m surprised my previous example adapted without error if it was later going to complain about being an invalid dial address.

Adapt is different from validate. Adapt just takes tokens and marshals them into JSON. If you want stronger validation use the --validate flag.

1 Like

@francislavoie That worked!
image

Now to convert the rest of my sockets. Thanks so much!

I’ll keep this conversatiion going so I can place my completed Caddy file here in the hope of helping others (and possibly a docs update). :heart:

1 Like

Sorry, one more question. We use port :3020 during dev and I need to set this up for CORS. Suggestions?

Sorry, I’m not too experienced with CORS stuff.

Maybe @Whitestrake can help?

I also handle CORS in the express server. When using “localhost” it hits a breakpoint there. When I use “localhost:3020”, then it never hits the breakpoint, so it must be getting blocked by Caddy. Just need to tell Caddy to allow that port to pass through.

I may have solved the dev issue by using webpack_dev_server’s proxy system and rewrite rules to access the services directly instead of using the reverse proxy that Caddy is providing (we use the reverse proxy for external upstream servers, as well as the browser)

The final result:

localhost {
  route /weather/* {
    uri * strip_prefix /weather
    rewrite * /socket.io{uri}
    reverse_proxy * http://localhost:3010 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /ptz/* {
    uri * strip_prefix /ptz
    rewrite * /socket.io{uri}
    reverse_proxy * http://localhost:3006 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /liveview/* {
    uri * strip_prefix /liveview
    rewrite * /socket.io{uri}
    reverse_proxy * http://localhost:3004 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /archive/* {
    uri * strip_prefix /archive
    rewrite * /socket.io{uri}
    reverse_proxy * http://localhost:3003 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /alarms/* {
    uri * strip_prefix /alarms
    rewrite * /socket.io{uri}
    reverse_proxy * http://localhost:3002 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /console_socket/* {
    uri * strip_prefix /console_socket
    rewrite * /console/socket.io{uri}
    reverse_proxy * http://localhost:3001 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /web_app_socket/* {
    uri * strip_prefix /web_app_socket
    rewrite * /web/socket.io{uri}
    reverse_proxy * http://localhost:3001 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
  route /* {
    reverse_proxy * http://localhost:3001 {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
    }
  }
}
3 Likes

Cool!

If you want to, you can also drop the /* from route /* and the * from reverse_proxy * and uri *. But that is just a matter of preference and simplicity at this point.

You might even be able to consolidate some of your uri strip_prefix + rewrite combos with a single uri replace line.

Thanks for posting your solution!

Since you seem to reuse this same block everywhere:

      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type

You can make a snippet at the top of the file (outside your site block) like this:

(proxy_options) {
      header_up X-Real-IP {remote_host}
      header_up X-Forwarded-Proto {scheme}
      header_up Access-Control-Allow-Origin *
      header_up Access-Control-Allow-Credentials true
      header_up Access-Control-Allow-Headers Cache-Control,Content-Type
}

Then use it later:

import proxy_options

Also as @matt was suggesting, I think you can simplify

    uri * strip_prefix /web_app_socket
    rewrite * /web/socket.io{uri}

to

    uri replace /web_app_socket /web/socket.io

Same thing, but in one step instead of two! :smile: