Nginx server Configure & Compile

We can use ppa:nginx that is even mentioned in official install documentation, and then compile our extra modules in it as in procedure explained here.

I prefer building it from source, so we will do it in that way.

Also, make sure we have all the packages needed for compilation:

# needed packages
sudo apt install -y build-essential zlib1g-dev libpcre3 libpcre3-dev unzip uuid-dev git
# also required
sudo apt install -y libxslt-dev libgd-dev libgeoip-dev checkinstall

# needed for Google BoringSSL (HTTP/3 patch)
sudo apt install -y curl git cmake ninja-build golang libpcre3-dev zlib1g-dev
# Also need latest Rust (and Cargo), not the ones from Ubuntu repos
curl https://sh.rustup.rs -sSf | sh

Download nginx modules

I usually update everything before continuing:

apt update; apt -y upgrade; apt dist-upgrade; apt -y autoremove; apt clean

Copy-pasting all this will probably work:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170


# Specify version numbers
#
BUILD_FOLDER=/usr/local/src
#
NGINX_VERSION=1.23.1          # @see: http://nginx.org/en/download.html
NPS_VERSION=1.13.35.2-stable  # @see: https://developers.google.com/speed/pagespeed/module/release_notes
HDRSMORE_VERSION=0.34         # @see: https://github.com/openresty/headers-more-nginx-module/tags

cd ${BUILD_FOLDER}

# Download ngx_pagespeed module sources 
#
wget https://github.com/pagespeed/ngx_pagespeed/archive/v${NPS_VERSION}.zip -O ngx_pagespeed.zip
unzip ngx_pagespeed.zip && rm ngx_pagespeed.zip
cd *pagespeed-ngx-${NPS_VERSION}*
NPS_FOLDER=$(basename ${PWD})

psol_url=https://dl.google.com/dl/page-speed/psol/${NPS_VERSION}.tar.gz
[ -e scripts/format_binary_url.sh ] && psol_url=$(scripts/format_binary_url.sh PSOL_BINARY_URL) # add x64 or x86 suffixes
wget ${psol_url}
tar -xzvf $(basename ${psol_url}) && rm $(basename ${psol_url})
cd ..

read # pause

# Download `headers-more-nginx-module` sources.
# Now we can not only add, but also change or remove headers. Use it to remove nginx server signature.  
#
wget https://github.com/openresty/headers-more-nginx-module/archive/v${HDRSMORE_VERSION}.zip -O ngx_headers-more.zip
unzip ngx_headers-more.zip && rm ngx_headers-more.zip

read # pause

# Also include brotli compression
# @see: https://github.com/google/ngx_brotli

git clone --recursive https://github.com/google/ngx_brotli.git

# Not using anymore; download `ngx_cache_purge` module:
# download ngx_cache_purge source
#
# cd /usr/local/src/
# _TMP=2.3
# wget http://labs.frickle.com/files/ngx_cache_purge-${_TMP}.tar.gz
# tar -xvf ngx_cache_purge-*.tar.gz
# rm ngx_cache_purge-*.tar.gz

## Compile nginx from source

# Compile nginx with additional modules:

# download nginx source; 
wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz
tar -xvf nginx-${NGINX_VERSION}.tar.gz
rm nginx-${NGINX_VERSION}.tar.gz

read # pause

# HTTP/3 and QUIC patch only works with the 1.16.x release branch 
git clone --recursive https://github.com/cloudflare/quiche

read # pause

cd nginx-${NGINX_VERSION}/
patch -p01 < ../quiche/nginx/nginx-1.16.patch

# I have used 'nginx -V' to get the last configure directives
#
# @see: https://www.nginx.com/resources/wiki/start/topics/tutorials/installoptions/
#
# New configure now looks like this:
#
# --with-file-aio both with --with-threads must be specified
# --with-ipv6 is not needed as it is configured automatically
# --sbin-path was mandatory
# --with-debug: not really needed
#
# Also not using anymore.
# @see https://gcc.gnu.org/onlinedocs/gcc/Option-Summary.html
#   --with-cc-opt='-g -O2 -fPIE -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2'
#   --with-ld-opt='-Wl,-Bsymbolic-functions -fPIE -pie -Wl,-z,relro -Wl,-z,now'
#
read -r -d '' PS_NGX_EXTRA_FLAGS <<'EOF'
    --prefix=/usr/share/nginx
    --sbin-path=/usr/sbin/nginx
    --conf-path=/etc/nginx/nginx.conf
    --modules-path=/usr/lib/nginx/modules

    --http-log-path=/var/log/nginx/access.log
    --error-log-path=/var/log/nginx/error.log

    --lock-path=/var/lock/nginx.lock
    --pid-path=/run/nginx.pid
    --http-client-body-temp-path=/var/lib/nginx/body
    --http-fastcgi-temp-path=/var/lib/nginx/fastcgi
    --http-proxy-temp-path=/var/lib/nginx/proxy
    --http-scgi-temp-path=/var/lib/nginx/scgi
    --http-uwsgi-temp-path=/var/lib/nginx/uwsgi

    --with-file-aio
    --with-threads
    --with-pcre-jit

    --with-http_ssl_module
    --with-http_stub_status_module
    --with-http_realip_module
    --with-http_auth_request_module
    --with-http_addition_module
    --with-http_dav_module
    --with-http_gunzip_module
    --with-http_gzip_static_module
    --with-http_image_filter_module=dynamic
    --with-http_sub_module
    --with-http_xslt_module=dynamic
    --with-stream=dynamic
    --with-stream_ssl_module
    --with-mail=dynamic
    --with-mail_ssl_module
    --with-http_geoip_module=dynamic
    --with-http_v2_module

    --with-debug
    --with-http_slice_module
EOF


# Configure
#
sudo apt install -y libssl-dev
./configure \
    --with-http_v2_module \
    \
    --add-module=${BUILD_FOLDER}/ngx_brotli \
    --add-dynamic-module=${BUILD_FOLDER}/${NPS_FOLDER} \
    --add-dynamic-module=${BUILD_FOLDER}/headers-more-nginx-module-${HDRSMORE_VERSION} \
    \
    ${PS_NGX_EXTRA_FLAGS}

# Version for HTTP/3 and BoringSSL
# Without OCSP stapling so I don't use it
#
./configure \
    --with-http_v2_module \
    --with-http_v3_module \
    --with-openssl=../quiche/deps/boringssl \
    --with-quiche=../quiche \
    \
    --add-module=${BUILD_FOLDER}/ngx_brotli \
    --add-dynamic-module=${BUILD_FOLDER}/${NPS_FOLDER} \
    --add-dynamic-module=${BUILD_FOLDER}/headers-more-nginx-module-${HDRSMORE_VERSION} \
    \
    ${PS_NGX_EXTRA_FLAGS}


read # pause

# Install
#
make && make install

# The following replacement for `make install` doesn't work
# as it won't overwrite original file
# make && \
# checkinstall --install=no -y && \
# dpkg -i nginx_${NGINX_VERSION}-1_amd64.deb && \



# systemctl unmask nginx
# systemctl enable nginx

Done. The new package has been installed and saved to /usr/local/src/nginx-1.11.5/nginx-1.11.5-from-source_1.11.5-1_amd64.deb You can remove it from your system anytime using: nginx-1.11.5-from-source

If you didn’t previously have a version of nginx installed from source, maybe you’ll need to set up init scripts. Or not?.

Enable dynamic modules

echo ’load_module modules/ngx_pagespeed.so;’ > /usr/share/nginx/modules-available/mod-pagespeed.conf echo ’load_module modules/ngx_http_headers_more_filter_module.so;’ > /usr/share/nginx/modules-available/mod-http-headers-more-filter.conf sudo ln -s /usr/share/nginx/modules-available/mod-pagespeed.conf /etc/nginx/modules-enabled/ sudo ln -s /usr/share/nginx/modules-available/mod-http-headers-more-filter.conf /etc/nginx/modules-enabled/

compile-nginx.sh · GitHub

A MUST! h5bp/server-configs-nginx

Nginx settings.

FuelPHP-Nginx/nginx at master · rajibmp/FuelPHP-Nginx

server { listen 80 default_server; listen 443 default_server ssl;

client_max_body_size        10M;

root                        /usr/share/nginx/html;
index                       index.php;

# Handle images directly
location ~* ^.+\.(jpg|jpeg|gif|png|ico|css|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|js|flv|swf)$ {
    root /usr/share/nginx/html;
}

location /
    index index.html index.php;
    try_files $uri $uri/ @handler;
    expires 30d;
}

location  /. {
    return 404;
}

location @handler {
    rewrite / /index.php;
}

location ~ \.php/ {
    rewrite ^(.*\.php)/ $1 last;
}

# All Requests
location ~ \.php$ {

    ## Catch 404s that try_files miss
    if (!-e $request_filename) {
        rewrite / /index.php last;
    }

    fastcgi_split_path_info         ^(.+\.php)(/.+)$;

    fastcgi_pass                    <strong>php:9000</strong>;
    fastcgi_index                   index.php;

    include fastcgi_params;
    fastcgi_param                   SCRIPT_FILENAME $document_root$fastcgi_script_name;

    fastcgi_param                   REMOTE_ADDR $remote_addr;

    fastcgi_buffer_size             1024k;
    fastcgi_buffers                 500 512k;
    fastcgi_connect_timeout         1200;
    fastcgi_send_timeout            1200;
    fastcgi_read_timeout            1200;

}

gzip                on;
gzip_min_length     1000;
gzip_proxied        any;

}

Configuring Fast CGI Params for Multi-Site

Default fastcgi_params file that is installed on a fresh nginx server has this line:

fastcgi_param SERVER_NAME $server_name;

Update the fastcgi_param to:

fastcgi_param SERVER_NAME $host;

Nginx uses $_SERVER['server_name'] as the first server listed in the virtual server config. Way to get around this is to setup the FastCGI params to use host instead of server_name.

Why don’t we use `$http_host` variable?

Let’s explain these nginx variables a little:

$host equals $http_host, lowercase and without the port number, and when there is no HTTP_HOST header then it equals to $server_name.
$http_host equals always the HTTP_HOST request header.
$server_name is the first server_name that is defined in the current server block. If you have multiple server_names, only the first one will appear. If you happen to have a regex in the first one, that becomes the $server_name and all sorts of ugly stuff can happen.

Sources & Articles

Configuring Fast CGI Params for Multi-Site

Difference of $host and $http_host in Nginx

nginx “server” directive with multiple “server_name” entries: always first one is passed to PHP’s $_SERVER[‘SERVER_NAME’] - Stack Overflow

Change SERVER_NAME from $server_name to $host in global fastcgi_param

sed -i -r 's/^(.*SERVER_NAME\s+).*$/\n# Multi-site fix\n\1$host;/gi' /etc/nginx/fastcgi_params

502 Bad Gateway: Upstream sent big headers

Started noticing when you try to logout from system, nginx spits out the dreaded 502 Bad Gateway message.

I have discovered a problem in error.log, and it is about communication between nginx and PHP.

upstream sent too big header while reading response header from upstream

Bug happened on all the servers (thetube, hotmess & ganzohr), except Plesk where they already have a little increased buffer values.

Solution?

How to determine optimal values?

Setting Nginx FastCGI response buffer sizes.

Everything explained:

Read and you will be enlightened:

The maximum response size:

awk '($9 ~ /200/)' access.log  | awk '{print $10}' | sort -nr | head -n 1

Average response size:

echo $(( `awk '($9 ~ /200/)' access.log | awk '{print $10}' | awk '{s+=$1} END {print s}'` / `awk '($9 ~ /200/)' access.log  | wc -l` ))

I’ve got on save-up.ch: max: 396k, avg: 46k

Put inside http { } of nginx.conf section as we want a global solution.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26



cat <<'EOF' > /etc/nginx/conf.d/fastcgi.conf

# Increase buffers
#
# Detailed explanation of all the values
# @see: http://nginx.org/en/docs/http/ngx_http_fastcgi_module.html

# Increase maximum execution time to 20 minutes
#
fastcgi_connect_timeout  30s; # 60s, timeout for establishing a connection with a FastCGI server
fastcgi_send_timeout   1200s; # 60s, timeout for transmitting a request to the FastCGI server 
fastcgi_read_timeout   1200s; # 60s, timeout for reading a response from the FastCGI server

# Increase buffers to insane values
#
fastcgi_buffers        500 512k; # total of 256M (8 4k|8k), number and size of buffers for reading a response. Size is equal to one memory page
fastcgi_buffer_size       1024k;
fastcgi_busy_buffers_size 1024k;

# Disable swapping to disk
#
fastcgi_max_temp_file_size       0; # 1024m, on response buffering, if does not fit into the fastcgi buffers, part can be saved to file (zero disables this)
fastcgi_temp_file_write_size 1024k; # 8k|16k, limits the temp-file size, only when buffering of responses is enabled

EOF

Woow: what a resource: https://www.scalescale.com/tips/nginx/

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44



cat << EOF > /etc/nginx/conf.d/ssl.conf

# Speed up SSL handshake
#
# SSL Session Cache; using only shared cache without the built-in cache should be more efficient
# @see: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
#
# Parameters generated from Mozilla:
# @see: https://mozilla.github.io/server-side-tls/ssl-config-generator/
#
ssl_session_cache shared:SSL:50m;
ssl_session_timeout 1d;
ssl_session_tickets on; # 

# Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
#
ssl_dhparam /etc/ssl/private/dhparams.pem;

# HTTP/2 need stronger ciphers
#
# Recommended Mozilla ciphers (Modern):
# @see: https://mozilla.github.io/server-side-tls/ssl-config-generator/
#
ssl_prefer_server_ciphers on;
ssl_ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GC;
# ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK';

# Working ciphers
# @see: https://bjornjohansen.no/optimizing-https-nginx
#
# ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5;
# ssl_ciphers EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!MEDIUM;
#
## Not working
# @see: https://sethvargo.com/getting-an-a-plus-on-qualys-ssl-labs-tester/
#
# ssl_ciphers AES256+EECDH:AES256+EDH:!aNULL;
# ssl_ciphers AES256+EECDH:AES256+EDH;
#
# These were fastest negotiation ciphers in the past, but not working:
# ssl_ciphers ALL:!ADH:!EXP:!LOW:!RC2:!3DES:!SEED:!RC4:+HIGH:+MEDIUM;

EOF

wget https://raw.githubusercontent.com/h5bp/server-configs-nginx/master/h5bp/web_performance/cache_expiration.conf -O /etc/nginx/conf.d/cache_expiration.conf

Force www

if ($host !~* ^(www)) { return 301 $scheme://www.$host$request_uri; }

Force non-www

if ($host ~* ^www.(.*)$) { return 301 $scheme://$1$request_uri; }

Force SSL

if ($scheme = http) { return 301 https://$host$request_uri$is_args$args; }

Force non-SSL

if ($scheme = https) { return 301 http://$host$request_uri$is_args$args; }

Understanting nginx

Understanding Nginx Server and Location Block Selection Algorithms | DigitalOcean How it works: nginx and error pages

Plesk nginx configure

Mine configuration:

First part are original compile options on Plesk, and after that - I added mine.


./configure \
    --prefix=/usr/share \
    --sbin-path=/usr/sbin/nginx \
    --conf-path=/etc/nginx/nginx.conf \
    --error-log-path=/var/log/nginx/error.log \
    --http-log-path=/var/log/nginx/access.log \
    --lock-path=/var/lock/nginx.lock \
    --pid-path=/var/run/nginx.pid \
    --http-client-body-temp-path=/var/lib/nginx/body \
    --http-fastcgi-temp-path=/var/lib/nginx/fastcgi \
    --http-proxy-temp-path=/var/lib/nginx/proxy \
    --http-scgi-temp-path=/var/lib/nginx/scgi \
    --http-uwsgi-temp-path=/var/lib/nginx/uwsgi \
    --user=nginx \
    --group=nginx \
    --with-ipv6 \
    --with-file-aio \
    --with-http_ssl_module \
    --with-http_realip_module \
    --with-http_sub_module \
    --with-http_dav_module \
    --with-http_gzip_static_module \
    --with-http_stub_status_module \
    \
    --with-http_auth_request_module \
    --with-http_addition_module \
    --with-http_spdy_module \
    --with-http_xslt_module \
    --with-mail \
    --with-mail_ssl_module \
    --add-module=/usr/local/src/ngx_pagespeed-release-1.9.32.6-beta \
    --add-module=/usr/local/src/ngx_cache_purge-2.3

# I removed the passenger module that was in Plesk original config
    --add-module=/usr/share/passenger/ngx_http_passenger_module \

# Not using some of modules
    --with-http_image_filter_module (requires GD library)
    --with-http_geoip_module  (requires GeoIP library)

Install Nginx, Passenger, PageSpeed with spdy on Ubuntu 14.04 « Jethro’s logs

How to server different PHP per folder

# Use HHVM
#
location / {
    try_files $uri $uri/ /index.php$is_args$args;
}

# For this location old PHP
location ^~ /hoertest-js/ {

    try_files $uri $uri/ /hoertest-js/index.php$is_args$args;

    location ~ \.php$ {
        try_files /dummy_nonexistant_file @phpfpm;
    }
}

location ~ \.php$ {
    try_files /dummy_nonexistant_file @hhvm;
}


# HHVM with failover to php-fpm
#
location @hhvm {
    fastcgi_pass ...
}

location @phpfpm {
    fastcgi_pass ...
}

WordPress specific Nginx conf tweaks: pothi/WordPress-Nginx: WordPress specific Nginx conf tweaks - Please see the updated repo at Also here: nginx settings for WordPress.

Nginx Security

Top 20 Nginx WebServer Best Security Practices

Proverio sam i ovo:

–with-file-aio

Boosting NGINX Performance 9x with Thread Pools Boosting Nginx Performance with Thread Pools | Hacker News

Just add in the ‘http’, ‘server’, or ’location’ context