NGINX Ingress Controller UDP Load Balancing Packet Drops

Question

I am performing PT or Load testing activity on our application pods for UDP traffic in Kubernetes cluster. The load first comes to NGINX Ingress Controller pod and then it gets forwarded to application pods.

If I send the traffic to directly my pods, it is able to handle traffic of 70k tps. But If I send the same traffic via NGINX, the packet drops don't happen at application. But it happen at NGINX level.

I tried to take the reference from below links, but nothing helped. https://gist.github.com/v0lkan/90fcb83c86918732b894 https://gist.github.com/denji/8359866

Config used, removed the HTTP Block

# Configuration checksum: 13343735496899811162

# setup custom paths that do not require root access
pid /tmp/nginx/nginx.pid;

daemon off;

worker_processes 5;

worker_rlimit_nofile 1047552;

worker_shutdown_timeout 240s ;

events {
        multi_accept        on;
        worker_connections  16384;
        use                 epoll;

}

stream {
        lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;";

        lua_shared_dict tcp_udp_configuration_data 5M;

        init_by_lua_block {
                collectgarbage("collect")

                -- init modules
                local ok, res

                ok, res = pcall(require, "configuration")
                if not ok then
                error("require failed: " .. tostring(res))
                else
                configuration = res
                end

                ok, res = pcall(require, "tcp_udp_configuration")
                if not ok then
                error("require failed: " .. tostring(res))
                else
                tcp_udp_configuration = res
                tcp_udp_configuration.prohibited_localhost_port = '10246'

                end

                ok, res = pcall(require, "tcp_udp_balancer")
                if not ok then
                error("require failed: " .. tostring(res))
                else
                tcp_udp_balancer = res
                end
        }

        init_worker_by_lua_block {
                tcp_udp_balancer.init_worker()
        }

        lua_add_variable $proxy_upstream_name;

        log_format log_stream '[$remote_addr] [$time_local] $protocol $status $bytes_sent $bytes_received $session_time';

        access_log /var/log/nginx/access.log log_stream ;

        error_log  /var/log/nginx/error.log crit;

        upstream upstream_balancer {
                server 0.0.0.1:1234; # placeholder

                balancer_by_lua_block {
                        tcp_udp_balancer.balance()
                }
        }

        server {
                listen 127.0.0.1:10247;

                access_log off;

                content_by_lua_block {
                        tcp_udp_configuration.call()
                }
        }

        # TCP services

        # UDP services

        server {
                preread_by_lua_block {
                        ngx.var.proxy_upstream_name="udp-cgnat-cgnat-app-2232";
                }

                listen                  2200 udp;

                listen                  [::]:2200 udp;

                proxy_responses         1;
                proxy_timeout           600s;
                proxy_next_upstream     on;
                proxy_next_upstream_timeout 600s;
                proxy_next_upstream_tries   3;
                proxy_pass              upstream_balancer;
        }

        server {
                preread_by_lua_block {
                        ngx.var.proxy_upstream_name="udp-cgnat-cgnat-app-2233";
                }

                listen                  2201 udp;

                listen                  [::]:2201 udp;

                proxy_responses         1;
                proxy_timeout           600s;
                proxy_next_upstream     on;
                proxy_next_upstream_timeout 600s;
                proxy_next_upstream_tries   3;
                proxy_pass              upstream_balancer;
        }

        server {
                preread_by_lua_block {
                        ngx.var.proxy_upstream_name="udp-cgnat-cgnat-app-2234";
                }

                listen                  2202 udp;

                listen                  [::]:2202 udp;

                proxy_responses         1;
                proxy_timeout           600s;
                proxy_next_upstream     on;
                proxy_next_upstream_timeout 600s;
                proxy_next_upstream_tries   3;
                proxy_pass              upstream_balancer;
        }

        server {
                preread_by_lua_block {
                        ngx.var.proxy_upstream_name="udp-cgnat-cgnat-app-2235";
                }

                listen                  2203 udp;

                listen                  [::]:2203 udp;

                proxy_responses         1;
                proxy_timeout           600s;
                proxy_next_upstream     on;
                proxy_next_upstream_timeout 600s;
                proxy_next_upstream_tries   3;
                proxy_pass              upstream_balancer;
        }

        # Stream Snippets

}

vCPU: 5

Memory: 20GB

This is the generated nginx conf inside nginx pod.

Are there any parameters that I should change for better performance.

The testing is being done for UDP traffic only.

Following is the NGINX ingress controller that we are using, https://kubernetes.github.io/ingress-nginx/examples/

Also let me know if there is any other UDP load balancer that I should try which has better performance.