diff --git a/ci/setup-new-buildkite-agent/setup-buildkite.sh b/ci/setup-new-buildkite-agent/setup-buildkite.sh index 5387762ebd..499d1f023b 100755 --- a/ci/setup-new-buildkite-agent/setup-buildkite.sh +++ b/ci/setup-new-buildkite-agent/setup-buildkite.sh @@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE TimeoutStartSec=10 TimeoutStopSec=0 KillMode=process -LimitNOFILE=65536 +LimitNOFILE=500000 [Install] WantedBy=multi-user.target diff --git a/ci/setup-new-buildkite-agent/setup-limits.sh b/ci/setup-new-buildkite-agent/setup-limits.sh index 841ab84108..140c26a3ab 100755 --- a/ci/setup-new-buildkite-agent/setup-limits.sh +++ b/ci/setup-new-buildkite-agent/setup-limits.sh @@ -8,5 +8,5 @@ source "$HERE"/utils.sh ensure_env || exit 1 # Allow more files to be opened by a user -sed -i 's/^\(# End of file\)/* soft nofile 65535\n\n\1/' /etc/security/limits.conf +echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf diff --git a/docs/src/running-validator/validator-start.md b/docs/src/running-validator/validator-start.md index 94c69ce8ec..9f6eae9374 100644 --- a/docs/src/running-validator/validator-start.md +++ b/docs/src/running-validator/validator-start.md @@ -46,10 +46,14 @@ that CUDA is enabled: `"[ solana::validator] CUDA is enabled"` ## System Tuning -For Linux validators, the solana repo includes a daemon to adjust system settings to optimize -performance (namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy). +### Linux +#### Automatic +The solana repo includes a daemon to adjust system settings to optimize performance +(namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy). -The daemon (`solana-sys-tuner`) is included in the solana binary release. +The daemon (`solana-sys-tuner`) is included in the solana binary release. Restart +it, *before* restarting your validator, after each software upgrade to ensure that +the latest recommended settings are applied. To run it: @@ -57,6 +61,53 @@ To run it: sudo solana-sys-tuner --user $(whoami) > sys-tuner.log 2>&1 & ``` +#### Manual +If you would prefer to manage system settings on your own, you may do so with +the following commands. + +##### **Increase UDP buffers** +```bash +sudo bash -c "cat >/etc/sysctl.d/20-solana-udp-buffers.conf </etc/sysctl.d/20-solana-mmaps.conf </etc/security/limits.d/90-solana-nofiles.conf < Result<()> { fn adjust_ulimit_nofile() -> Result<()> { // Rocks DB likes to have many open files. The default open file descriptor limit is // usually not enough - let desired_nofile = 65000; + let desired_nofile = 500000; fn get_nofile() -> libc::rlimit { let mut nofile = libc::rlimit { diff --git a/sys-tuner/src/main.rs b/sys-tuner/src/main.rs index aa00bd7cdb..45f6f59afa 100644 --- a/sys-tuner/src/main.rs +++ b/sys-tuner/src/main.rs @@ -93,7 +93,7 @@ fn tune_kernel_udp_buffers_and_vmmap() { sysctl_write("net.core.wmem_default", "134217728"); // increase mmap counts for many append_vecs - sysctl_write("vm.max_map_count", "1000000"); + sysctl_write("vm.max_map_count", "500000"); } #[cfg(unix)]