NGINX: Everyday server stops working at same time

Question

I've got an NGINX server on a Debian OS(10) that is hosting a webpage. Every day at a specific time the server stops working for around 1-2minutes. This happens every day from 19.11 until 22.11(11 minutes every full hour). I've checked the logs and can't find anything. The process isn't restarted, there are no CRONs running at that time or any other scripts. I've written a script to track some metrics(RAM, CPU, network interface, sockets) and everything is ok:

    Thu May 25 10:11:37 PM AST 2023
PING 192.168.245.165 (192.168.245.165) 56(84) bytes of data.
64 bytes from 192.168.245.165: icmp_seq=1 ttl=64 time=0.191 ms

--- 192.168.245.165 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.191/0.191/0.191/0.000 ms
Total: 1881
TCP:   1825 (estab 1494, closed 290, orphaned 0, timewait 280)

Transport Total     IP        IPv6
RAW       1         1         0
UDP       34        16        18
TCP       1535      1515      20
INET      1570      1532      38
FRAG      0         0         0

              total        used        free      shared  buff/cache   available
Mem:          251Gi       2.8Gi       1.3Gi        17Gi       247Gi       229Gi
Swap:         3.7Gi       3.6Gi       111Mi
Linux 4.19.0-6-amd64 (origin02)         05/25/2023      _x86_64_        (56 CPU)

10:11:37 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
10:11:37 PM  all    0.16    0.00    0.71    0.61    0.00    0.14    0.00    0.00    0.00   98.37

My NGINX conf:

# /etc/nginx/nginx.conf
user www-data;
pid /run/nginx.pid;

# Set number of worker processes automatically based on number of CPU cores.
worker_processes auto;
worker_rlimit_nofile 65535;
# Enables the use of JIT for regular expressions to speed-up their processing.
pcre_jit on;

# Configures default error logger.
error_log /var/log/nginx/error.log error;

# Includes files with directives to load dynamic modules.
include /etc/nginx/modules/*.conf;

# Uncomment to include files with config snippets into the root context.
# NOTE: This will be enabled by default in Alpine 3.15.
#include /etc/nginx/conf.d/*.conf;

events {
        # The maximum number of simultaneous connections that can be opened by
        # a worker process.
        worker_connections 65535;
}

http {
        # Cache off
        open_file_cache off;
        # Includes virtual hosts configs.
        include /etc/nginx/sites-available/*;
        include /etc/nginx/conf.d/*.conf;
        # Includes mapping of file name extensions to MIME types of responses
        # and defines the default type.
        include /etc/nginx/mime.types;
        default_type application/octet-stream;

        # Name servers used to resolve names of upstream servers into addresses.
        # It's also needed when using tcpsocket and udpsocket in Lua modules.
        #resolver 1.1.1.1 1.0.0.1 2606:4700:4700::1111 2606:4700:4700::1001;

        # Don't tell nginx version to the clients. Default is 'on'.
        server_tokens off;

        # Specifies the maximum accepted body size of a client request, as
        # indicated by the request header Content-Length. If the stated content
        # length is greater than this size, then the client receives the HTTP
        # error code 413. Set to 0 to disable. Default is '1m'.
        client_max_body_size 1m;

        # Sendfile copies data between one FD and other from within the kernel,
        # which is more efficient than read() + write(). Default is off.
        sendfile on;

        # Causes nginx to attempt to send its HTTP response head in one packet,
        # instead of using partial frames. Default is 'off'.
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;

        # Enables the specified protocols. Default is TLSv1 TLSv1.1 TLSv1.2.
        # TIP: If you're not obligated to support ancient clients, remove TLSv1.1.
        ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;

        # Path of the file with Diffie-Hellman parameters for EDH ciphers.
        # TIP: Generate with: `openssl dhparam -out /etc/ssl/nginx/dh2048.pem 2048`
        ssl_dhparam /etc/ssl/nginx/dh2048.pem;

        # Specifies that our cipher suits should be preferred over client ciphers.
        # Default is 'off'.
        ssl_prefer_server_ciphers on;

        # Enables a shared SSL cache with size that can hold around 8000 sessions.
        # Default is 'none'.
        ssl_session_cache shared:SSL:2m;

        # Specifies a time during which a client may reuse the session parameters.
        # Default is '5m'.
        ssl_session_timeout 1h;

        # Disable TLS session tickets (they are insecure). Default is 'on'.
        ssl_session_tickets off;

        # Enable gzipping of responses.
        gzip  on;
        gzip_vary on;
        gzip_proxied any;
        gzip_comp_level 6;
        gzip_buffers 16 8k;
        gzip_http_version 1.1;
        gzip_disable "msie6";
        gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript application/vnd.apple.mpegurl;

        # Helper variable for proxying websockets.
        map $http_upgrade $connection_upgrade {
                default upgrade;
                '' close;
        }

        # Specifies the main log format.
#        log_format main '$remote_addr - $remote_user [$time_local] "$request" '
#                        '$status $body_bytes_sent "$http_referer" '
#                        '"$http_user_agent" "$http_x_forwarded_for"';

        log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for" '
                      '"$host" sn="$server_name" '
                      'rt=$request_time '
                      'ua="$upstream_addr" us="$upstream_status" '
                      'ut="$upstream_response_time" ul="$upstream_response_length" '
                      'cs=$upstream_cache_status' ;
        # Sets the path, format, and configuration for a buffered log write.
        #access_log /var/log/nginx/access.log main;


        # Includes virtual hosts configs.
        #include /etc/nginx/http.d/*.conf;
}

# TIP: Uncomment if you use stream module.
#include /etc/nginx/stream.conf;

...and server conf:

server {
        listen 80 default_server;
        listen [::]:80 default_server;

        root /var/www/html;

        # Add index.php to the list if you are using PHP
        index index.html;

        access_log /var/log/nginx/origin02.access.log;

        set_real_ip_from 192.168.245.128/26;
        real_ip_header X-Forwarded-For;

        server_name _;

        location / {
                access_log  /var/log/nginx/blocked.log;
                try_files $uri $uri/ =404;
        }

        location = /basic_status {
                stub_status;
                allow 127.0.0.1;
                allow ::1;
                allow 192.168.200.246;
                deny all;
        }
        location ~ ^/ts/[\w]+/[\d-]+/[\w.-]+\.m3u8\?initial_seek_value=\d+/keepalive {
                return 204;
                access_log /var/log/nginx/keepalive.log;
        }
        
        location ~ ^/[0-9a-zA-Z]+/keepalive {
                return 204;
                access_log /var/log/nginx/keepalive.log;
        }
        
        location ~ ^/*.*/keepalive.* {
                access_log /var/log/nginx/keepalive.log;
                return 204;
        }

        location ~ ^/loc01/[0-9a-zA-Z]+/teardown {
                return 204;
                access_log /var/log/nginx/teardown.log;
        }

        location /web/NFS {
                alias /mnt/ramdisk/NFS;
                autoindex off;
                add_header 'Access-Control-Allow-Origin' '*' always;
                add_header 'Access-Control-Expose-Headers' 'Content-Length';
                add_header 'Access-Control-Allow-Headers' 'Range';
                add_header Cache-Control no-cache;
        }

        location /live {
                alias /mnt/ramdisk/content/;
                autoindex off;
                add_header 'Access-Control-Allow-Origin' '*';
                add_header Cache-Control no-cache;
        }


        location /LIVE {
                alias /mnt/ramdisk/LIVE/;
                autoindex off;
                add_header 'Access-Control-Allow-Origin' '*';
                add_header Cache-Control no-cache;
        }

        location /ts/scripts {
                alias /mnt/storage/scripts;
                autoindex off;
                add_header 'Access-Control-Allow-Origin' '*';
                add_header Cache-Control no-cache;
        }

        location /seg {
                alias /var/www/data;
                autoindex off;
                add_header 'Access-Control-Allow-Origin' '*';
                add_header Cache-Control no-cache;
        }

        error_page 404 /404.html;
        location = /404.html {
                root /var/www/html/;
                internal;
        }

        error_page 403 /403.html;
        location = /403.html {
                root /var/www/html/;
                internal;
        }

        error_page 500 /500.html;
        location = /500.html {
                root /var/www/html/;
                internal;
        }
}

Answer 1

FYI. The issue was the location of files on the external mount. The mount couldn't handle the amount of connections.