From 62e3c084d384fea0a4803bf39e8513e9557b89b2 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2020 17:13:27 +0000
Subject: [PATCH] Update system tuning and docs (bp #11680) (#11830)

* Sync FD limit and max maps to 500k

(cherry picked from commit 11951eb0099a7a1b4727e5204ae9d443c19484e7)

* Expand system tuning docs

(cherry picked from commit 5354df8c1cff0597e735379d2bb35ef3c2c181e8)

Co-authored-by: Trent Nelson <trent@solana.com>
---
 .../setup-buildkite.sh                        |  2 +-
 ci/setup-new-buildkite-agent/setup-limits.sh  |  2 +-
 docs/src/running-validator/validator-start.md | 57 ++++++++++++++++++-
 ledger/src/blockstore.rs                      |  2 +-
 sys-tuner/src/main.rs                         |  2 +-
 5 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/ci/setup-new-buildkite-agent/setup-buildkite.sh b/ci/setup-new-buildkite-agent/setup-buildkite.sh
index 5387762ebd..499d1f023b 100755
--- a/ci/setup-new-buildkite-agent/setup-buildkite.sh
+++ b/ci/setup-new-buildkite-agent/setup-buildkite.sh
@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=65536
+LimitNOFILE=500000
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ci/setup-new-buildkite-agent/setup-limits.sh b/ci/setup-new-buildkite-agent/setup-limits.sh
index 841ab84108..140c26a3ab 100755
--- a/ci/setup-new-buildkite-agent/setup-limits.sh
+++ b/ci/setup-new-buildkite-agent/setup-limits.sh
@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-sed -i 's/^\(# End of file\)/* soft nofile 65535\n\n\1/' /etc/security/limits.conf
+echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf
 
diff --git a/docs/src/running-validator/validator-start.md b/docs/src/running-validator/validator-start.md
index 94c69ce8ec..9f6eae9374 100644
--- a/docs/src/running-validator/validator-start.md
+++ b/docs/src/running-validator/validator-start.md
@@ -46,10 +46,14 @@ that CUDA is enabled: `"[<timestamp> solana::validator] CUDA is enabled"`
 
 ## System Tuning
 
-For Linux validators, the solana repo includes a daemon to adjust system settings to optimize
-performance (namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy).
+### Linux
+#### Automatic
+The solana repo includes a daemon to adjust system settings to optimize performance
+(namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy).
 
-The daemon (`solana-sys-tuner`) is included in the solana binary release.
+The daemon (`solana-sys-tuner`) is included in the solana binary release. Restart
+it, *before* restarting your validator, after each software upgrade to ensure that
+the latest recommended settings are applied.
 
 To run it:
 
@@ -57,6 +61,53 @@ To run it:
 sudo solana-sys-tuner --user $(whoami) > sys-tuner.log 2>&1 &
 ```
 
+#### Manual
+If you would prefer to manage system settings on your own, you may do so with
+the following commands.
+
+##### **Increase UDP buffers**
+```bash
+sudo bash -c "cat >/etc/sysctl.d/20-solana-udp-buffers.conf <<EOF
+# Increase UDP buffer size
+net.core.rmem_default = 134217728
+net.core.rmem_max = 134217728
+net.core.wmem_default = 134217728
+net.core.wmem_max = 134217728
+EOF"
+```
+```bash
+sudo sysctl -p /etc/sysctl.d/20-solana-udp-buffers.conf
+```
+
+##### **Increased memory mapped files limit**
+```bash
+sudo bash -c "cat >/etc/sysctl.d/20-solana-mmaps.conf <<EOF
+# Increase memory mapped files limit
+vm.max_map_count = 500000
+EOF"
+```
+```bash
+sudo sysctl -p /etc/sysctl.d/20-solana-mmaps.conf
+```
+Add
+```
+LimitNOFILE=500000
+```
+to the `[Service]` section of your systemd service file, if you use one,
+otherwise add it to `/etc/systemd/system.conf`.
+```bash
+sudo systemctl daemon-reload
+```
+```bash
+sudo bash -c "cat >/etc/security/limits.d/90-solana-nofiles.conf <<EOF
+# Increase process file descriptor count limit
+* - nofile 500000
+EOF"
+```
+```bash
+### Close all open sessions (log out then, in again) ###
+```
+
 ## Generate identity
 
 Create an identity keypair for your validator by running:
diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs
index a7b76ea892..213f66e9bb 100644
--- a/ledger/src/blockstore.rs
+++ b/ledger/src/blockstore.rs
@@ -3288,7 +3288,7 @@ fn adjust_ulimit_nofile() -> Result<()> {
 fn adjust_ulimit_nofile() -> Result<()> {
     // Rocks DB likes to have many open files.  The default open file descriptor limit is
     // usually not enough
-    let desired_nofile = 65000;
+    let desired_nofile = 500000;
 
     fn get_nofile() -> libc::rlimit {
         let mut nofile = libc::rlimit {
diff --git a/sys-tuner/src/main.rs b/sys-tuner/src/main.rs
index aa00bd7cdb..45f6f59afa 100644
--- a/sys-tuner/src/main.rs
+++ b/sys-tuner/src/main.rs
@@ -93,7 +93,7 @@ fn tune_kernel_udp_buffers_and_vmmap() {
     sysctl_write("net.core.wmem_default", "134217728");
 
     // increase mmap counts for many append_vecs
-    sysctl_write("vm.max_map_count", "1000000");
+    sysctl_write("vm.max_map_count", "500000");
 }
 
 #[cfg(unix)]