What I'm trying to do is to deploy a dockerized monorepo project (using NX as the monorepo framework) with the Nestjs + React + MySQL + Nginx stack on a VPS. I want the nginx proxy to listen on the host's port 88 (because another stack uses port 80, it's an old stack I do not dare touch). The OS of the VPS is CentOS 7.
I'll try to spare most of the details of the builds (Dockerfile) but know that the builds work, it is all working in my local environment (mostly due to the fact that I dont use nginx-proxy for local development) and I know it's either a matter of my Docker configs (I use docker-compose) or the host's networking that comes into play.
Here's a 'bird's eye view' of the stack:
- React-frontend container is running a react app (using nx serve react-frontend) on port 4200 in the container, exposing port 4200 to the host
- backend-api container is running a nodejs app (using nodejs entrypoint) on port 3333 of the container, exposing the port to the host
- a MySQL container running a mysql server running on port 3306 of the container, exposed on port 3307 of the Host
- A nginx-proxy using the jwilder/nginx-proxy docker image (I also tried with nginxproxy/nginx-proxy docker image) listening on port 88 of the host and redirecting the request to react-frontend container through proxy pass (this is the part that I'm failing at).
So here's my "compose-prod.yml" docker-compose file:
version: "3.7"
networks:
corp:
driver: bridge
nginx-proxy:
external:
name: nginx-proxy
volumes:
backend-db-volume:
driver: local
services:
nginx-proxy:
image: jwilder/nginx-proxy # also tried nginxproxy/nginx-proxy image
container_name: nginx-proxy
networks:
- corp
- nginx-proxy
environment:
HTTP_PORT: 88
ports:
- "88:88" # also tried "88:80" but that gives me "connection refused" in the browser
volumes:
- /var/run/docker.sock:/tmp/docker.sock:ro
backend-db:
image: backend-db
hostname: backend-db
restart: unless-stopped
volumes:
- backend-db-volume:/var/lib/mysql
networks:
- corp
build:
context: ./apps/backend-db
dockerfile: ./Dockerfile
ports:
- 3307:3306
expose:
- 3306
backend-api:
container_name: backend-api
depends_on:
- backend-db
build:
context: ./
cache_from:
- base-image:nx-base
dockerfile: ./apps/backend-api/Dockerfile
args:
NODE_ENV: "production"
BUILD_FLAG: ""
image: backend-api:nx-dev
ports:
- "3333:3333"
environment:
NODE_ENV: "production"
PORT: 3333
[... other env configs ommitted, like DB variables, etc.]
networks:
- corp
restart: on-failure
react-frontend:
container_name: react-frontend
build:
context: ./
dockerfile: ./apps/react-frontend/Dockerfile
args:
NODE_ENV: "production"
BUILD_FLAG: ""
image: react-frontend:nx-dev
environment:
VIRTUAL_HOST: react-frontend # note that my domain is react-frontend.com, obfuscated ofc ... which I also tried using in VIRTUAL_HOST config
VIRTUAL_PORT: 4200
NGINX_PROXY_CONTAINER: nginx-proxy
NODE_ENV: "production"
[...other env configs ommitted]
ports:
- "4200:4200"
expose:
- 4200
networks:
- nginx-proxy
- corp
restart: on-failure
The nginx-proxy container automatically detects containers running with VIRTUAL_HOST env. variable enabled, generates configs for those from the compose-prod.yml file. Right now, the configuration generated, that I get using the "docker exec nginx-proxy cat /etc/nginx/conf.d/default.conf" command is this:
# nginx-proxy version : 1.0.1-6-gc4ad18f
# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
# scheme used to connect to this server
map $http_x_forwarded_proto $proxy_x_forwarded_proto {
default $http_x_forwarded_proto;
'' $scheme;
}
# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
# server port the client connected to
map $http_x_forwarded_port $proxy_x_forwarded_port {
default $http_x_forwarded_port;
'' $server_port;
}
# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
# Connection header that may have been passed to this server
map $http_upgrade $proxy_connection {
default upgrade;
'' close;
}
# Apply fix for very long server names
server_names_hash_bucket_size 128;
# Default dhparam
ssl_dhparam /etc/nginx/dhparam/dhparam.pem;
# Set appropriate X-Forwarded-Ssl header based on $proxy_x_forwarded_proto
map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl {
default off;
https on;
}
gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
log_format vhost '$host $remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'"$upstream_addr"';
access_log off;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
ssl_prefer_server_ciphers off;
error_log /dev/stderr;
resolver 127.0.0.11;
# HTTP 1.1 support
proxy_http_version 1.1;
proxy_buffering off;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $proxy_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
proxy_set_header X-Original-URI $request_uri;
# Mitigate httpoxy attack (see README for details)
proxy_set_header Proxy "";
server {
server_name _; # This is just an invalid value which will never trigger on a real hostname.
server_tokens off;
listen 88;
access_log /var/log/nginx/access.log vhost;
return 503;
}
# react-frontend
upstream react-frontend {
## Can be connected with "react-frontend_corp" network
# react-frontend
server <IP of react-frontend container on Docker network>:4200;
# Cannot connect to network 'nginx-proxy' of this container
# Cannot connect to network 'react-frontend_corp' of this container
## Can be connected with "nginx-proxy" network
# react-frontend
server <IP of react-frontend container on Docker network>:4200;
}
server {
server_name react-frontend;
listen 88 ;
access_log /var/log/nginx/access.log vhost;
location / {
proxy_pass http://react-frontend;
}
}
When I access "example.com:88" I get a "503 Service Temporarily Unavailable" page in my browser returned from nginx and I see this in nginx's access logs:
nginx-proxy | nginx.1 | example.com xx.yy.zz.ip - - [06/Jul/2022:16:48:12 +0000] "GET / HTTP/1.1" 503 592 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
nginx-proxy | nginx.1 | example.com xx.yy.zz.ip - - [06/Jul/2022:16:48:12 +0000] "GET /favicon.ico HTTP/1.1" 503 592 "http://example.com:88/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
I omit the Dockerfiles since the nginx-proxy container is not built, it's taken as is from the image and all the builds work ... it's the deployment which gives me trouble.
Anyone who has any pointer on what I'm missing ? What I should check ? This is for a personal project and even though I can get around as a devop, Docker networking/deployment still baffles me sometimes.
EDIT: I'm adding the VPS (host) vhost config (nginx) here ... so maybe I can proxy-pass using this configuration ... how would I go about modifying this config so I can proxy pass requests to the "example" docker container exposing port 4200 (instead of a root directory on the VPS) ?
# configuration file /etc/nginx/conf.d/users/example.conf:
proxy_cache_path /var/cache/ea-nginx/proxy/example levels=1:2 keys_zone=example:10m inactive=60m;
#### main domain for example ##
server {
server_name example.com www.example.com mail.example.com;
listen 80;
listen [::]:80;
include conf.d/includes-optional/cloudflare.conf;
set $CPANEL_APACHE_PROXY_PASS $scheme://apache_backend_${scheme}_51_222_24_216;
# For includes:
set $CPANEL_APACHE_PROXY_IP 51.222.24.216;
set $CPANEL_APACHE_PROXY_SSL_IP 51.222.24.216;
set $CPANEL_PROXY_CACHE example;
set $CPANEL_SKIP_PROXY_CACHING 0;
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /var/cpanel/ssl/apache_tls/example.com/combined;
ssl_certificate_key /var/cpanel/ssl/apache_tls/example.com/combined;
ssl_protocols TLSv1.2 TLSv1.3;
proxy_ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256;
proxy_ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256;
root /home/example/public_html;
location /cpanelwebcall {
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass http://127.0.0.1:2082/cpanelwebcall;
}
location /Microsoft-Server-ActiveSync {
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass http://127.0.0.1:2090/Microsoft-Server-ActiveSync;
}
location = /favicon.ico {
allow all;
log_not_found off;
access_log off;
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass $CPANEL_APACHE_PROXY_PASS;
}
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass $CPANEL_APACHE_PROXY_PASS;
}
location / {
proxy_cache $CPANEL_PROXY_CACHE;
proxy_no_cache $CPANEL_SKIP_PROXY_CACHING;
proxy_cache_bypass $CPANEL_SKIP_PROXY_CACHING;
proxy_cache_valid 200 301 302 60m;
proxy_cache_valid 404 1m;
proxy_cache_use_stale error timeout http_429 http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
proxy_cache_revalidate on;
proxy_cache_min_uses 1;
proxy_cache_lock on;
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass $CPANEL_APACHE_PROXY_PASS;
}
include conf.d/server-includes/*.conf;
include conf.d/users/example/*.conf;
include conf.d/users/example/example.com/*.conf;
}
server {
listen 80;
listen [::]:80;
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /var/cpanel/ssl/apache_tls/example.com/combined;
ssl_certificate_key /var/cpanel/ssl/apache_tls/example.com/combined;
server_name cpanel.example.com cpcalendars.example.com cpcontacts.example.com webdisk.example.com webmail.example.com;
include conf.d/includes-optional/cloudflare.conf;
set $CPANEL_APACHE_PROXY_PASS $scheme://apache_backend_${scheme}_51_222_24_216;
# For includes:
set $CPANEL_APACHE_PROXY_IP 51.222.24.216;
set $CPANEL_APACHE_PROXY_SSL_IP 51.222.24.216;
location /.well-known/cpanel-dcv {
root /home/example/public_html;
disable_symlinks if_not_owner;
}
location /.well-known/pki-validation {
root /home/example/public_html;
disable_symlinks if_not_owner;
}
location /.well-known/acme-challenge {
root /home/example/public_html;
disable_symlinks if_not_owner;
}
location / {
# Force https for service subdomains
if ($scheme = http) {
return 301 https://$host$request_uri;
}
# no cache
proxy_cache off;
proxy_no_cache 1;
proxy_cache_bypass 1;
# pass to Apache
include conf.d/includes-optional/cpanel-proxy.conf;
proxy_pass $CPANEL_APACHE_PROXY_PASS;
}
}
