Move some common scripts from multinode-demo/ to scripts/

scripts/configure-metrics.sh

@@ -0,0 +1,45 @@
+# |source| this file
+#
+# The SOLANA_METRICS_CONFIG environment variable is formatted as a
+# comma-delimited list of parameters. All parameters are optional.
+#
+# Example:
+#   export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
+#
+configure_metrics() {
+  [[ -n $SOLANA_METRICS_CONFIG ]] || return 0
+
+  declare metrics_params
+  IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG"
+  for param in "${metrics_params[@]}"; do
+    IFS='=' read -r -a pair <<< "$param"
+    if [[ ${#pair[@]} != 2 ]]; then
+      echo Error: invalid metrics parameter: "$param" >&2
+    else
+      declare name="${pair[0]}"
+      declare value="${pair[1]}"
+      case "$name" in
+      host)
+        export INFLUX_HOST="$value"
+        echo INFLUX_HOST="$INFLUX_HOST" >&2
+        ;;
+      db)
+        export INFLUX_DATABASE="$value"
+        echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2
+        ;;
+      u)
+        export INFLUX_USERNAME="$value"
+        echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2
+        ;;
+      p)
+        export INFLUX_PASSWORD="$value"
+        echo INFLUX_PASSWORD="********" >&2
+        ;;
+      *)
+        echo Error: Unknown metrics parameter name: "$name" >&2
+        ;;
+      esac
+    fi
+  done
+}
+configure_metrics

scripts/metrics-write-datapoint.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Send a metrics datapoint
+#
+
+point=$1
+if [[ -z $point ]]; then
+  echo "Data point not specified"
+  exit 1
+fi
+
+echo "Influx data point: $point"
+if [[ -z $INFLUX_DATABASE || -z $INFLUX_USERNAME || -z $INFLUX_PASSWORD ]]; then
+  echo Influx user credentials not found
+  exit 0
+fi
+
+echo "https://metrics.solana.com:8086/write?db=${INFLUX_DATABASE}&u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}" \
+  | xargs curl --max-time 5 -XPOST --data-binary "$point"

scripts/oom-monitor.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+#
+# Reports Linux OOM Killer activity
+#
+
+here=$(dirname "$0")
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/oom-score-adj.sh
+
+if [[ $(uname) != Linux ]]; then
+  exit 0
+fi
+
+syslog=/var/log/syslog
+if [[ ! -r $syslog ]]; then
+  echo Unable to read $syslog
+  exit 0
+fi
+
+# Adjust OOM score to reduce the chance that this script will be killed
+# during an Out of Memory event since the purpose of this script is to
+# report such events
+oom_score_adj "self" -500
+
+while read -r victim; do
+  echo "Out of memory event detected, $victim killed"
+  "$here"/metrics-write-datapoint.sh "oom-killer,victim=$victim killed=1"
+done < <( \
+  tail --follow=name --retry -n0 $syslog \
+  | sed --unbuffered -n 's/^.* Out of memory: Kill process [1-9][0-9]* (\([^)]*\)) .*/\1/p' \
+)
+exit 1

scripts/oom-score-adj.sh

@@ -0,0 +1,20 @@
+# |source| this file
+#
+# Adjusts the OOM score for the specified process.  Linux only
+#
+# usage: oom_score_adj [pid] [score]
+#
+oom_score_adj() {
+  declare pid=$1
+  declare score=$2
+  if [[ $(uname) != Linux ]]; then
+    return
+  fi
+
+  echo "$score" > "/proc/$pid/oom_score_adj" || true
+  declare currentScore
+  currentScore=$(cat "/proc/$pid/oom_score_adj" || true)
+  if [[ $score != "$currentScore" ]]; then
+    echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)"
+  fi
+}