Move some common scripts from multinode-demo/ to scripts/

ci/localnet-sanity.sh

@@ -6,9 +6,9 @@
 
 cd "$(dirname "$0")"/..
 source ci/upload_ci_artifact.sh
-source multinode-demo/common.sh
+source scripts/configure-metrics.sh
 
-./multinode-demo/setup.sh
+multinode-demo/setup.sh
 
 backgroundCommands="drone leader validator validator-x"
 pids=()
@@ -16,7 +16,7 @@ pids=()
 for cmd in $backgroundCommands; do
   echo "--- Start $cmd"
   rm -f log-"$cmd".txt
-  ./multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 &
+  multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 &
   declare pid=$!
   pids+=("$pid")
   echo "pid: $pid"
@@ -67,11 +67,11 @@ flag_error() {
 #   set -x
 #   multinode-demo/test/wallet-sanity.sh
 # ) || flag_error
-# 
+#
 # echo "--- Node count"
 # (
 #   set -x
-#   ./multinode-demo/client.sh "$PWD" 3 -c --addr 127.0.0.1
+#   multinode-demo/client.sh "$PWD" 3 -c --addr 127.0.0.1
 # ) || flag_error
 
 killBackgroundCommands
@@ -79,6 +79,7 @@ killBackgroundCommands
 echo "--- Ledger verification"
 (
   set -x
+  source multinode-demo/common.sh
   $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/ledger verify
 ) || flag_error
 

ci/testnet-deploy.sh

@@ -6,7 +6,7 @@
 # with GCP and has sufficient permission.
 #
 here=$(dirname "$0")
-metrics_write_datapoint="$here"/../multinode-demo/metrics_write_datapoint.sh
+metrics_write_datapoint="$here"/../scripts/metrics-write-datapoint.sh
 
 # TODO: Switch over to rolling updates
 ROLLING_UPDATE=false

ci/testnet-sanity.sh

@@ -4,7 +4,7 @@
 #
 
 cd "$(dirname "$0")/.."
-source multinode-demo/common.sh
+source scripts/metrics-write-datapoint.sh
 
 NET_URL=$1
 if [[ -z $NET_URL ]]; then
@@ -45,7 +45,7 @@ echo "--- $NET_URL: node count"
 if [[ -n "$USE_SNAP" ]]; then
   # TODO: Merge client.sh functionality into solana-bench-tps proper and
   #       remove this USE_SNAP case
-  cmd=$solana_bench_tps
+  cmd=solana.bench-tps
 else
   cmd=multinode-demo/client.sh
 fi

multinode-demo/common.sh

@@ -102,50 +102,8 @@ solana_ledger_tool=$(solana_program ledger-tool)
 export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
 export RUST_BACKTRACE=1
 
-
+# shellcheck source=scripts/configure-metrics.sh
-# The SOLANA_METRICS_CONFIG environment variable is formatted as a
+source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
-# comma-delimited list of parameters. All parameters are optional.
-#
-# Example:
-#   export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
-#
-configure_metrics() {
-  [[ -n $SOLANA_METRICS_CONFIG ]] || return 0
-
-  declare metrics_params
-  IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG"
-  for param in "${metrics_params[@]}"; do
-    IFS='=' read -r -a pair <<< "$param"
-    if [[ ${#pair[@]} != 2 ]]; then
-      echo Error: invalid metrics parameter: "$param" >&2
-    else
-      declare name="${pair[0]}"
-      declare value="${pair[1]}"
-      case "$name" in
-      host)
-        export INFLUX_HOST="$value"
-        echo INFLUX_HOST="$INFLUX_HOST" >&2
-        ;;
-      db)
-        export INFLUX_DATABASE="$value"
-        echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2
-        ;;
-      u)
-        export INFLUX_USERNAME="$value"
-        echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2
-        ;;
-      p)
-        export INFLUX_PASSWORD="$value"
-        echo INFLUX_PASSWORD="********" >&2
-        ;;
-      *)
-        echo Error: Unknown metrics parameter name: "$name" >&2
-        ;;
-      esac
-    fi
-  done
-}
-configure_metrics
 
 tune_networking() {
   # Skip in CI
@@ -177,20 +135,6 @@ tune_networking() {
   fi
 }
 
-oom_score_adj() {
-  declare pid=$1
-  declare score=$2
-  if [[ $(uname) != Linux ]]; then
-    return
-  fi
-
-  echo "$score" > "/proc/$pid/oom_score_adj" || true
-  declare currentScore
-  currentScore=$(cat "/proc/$pid/oom_score_adj" || true)
-  if [[ $score != "$currentScore" ]]; then
-    echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)"
-  fi
-}
 
 SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config
 SOLANA_CONFIG_PRIVATE_DIR=${SNAP_DATA:-$PWD}/config-private

multinode-demo/drone.sh

@@ -10,6 +10,9 @@ here=$(dirname "$0")
 source "$here"/common.sh
 SOLANA_CONFIG_DIR="$SOLANA_CONFIG_DIR"-drone
 
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
 if [[ -d "$SNAP" ]]; then
   # Exit if mode is not yet configured
   # (typically the case after the Snap is first installed)

multinode-demo/leader.sh

@@ -7,6 +7,9 @@ here=$(dirname "$0")
 # shellcheck source=multinode-demo/common.sh
 source "$here"/common.sh
 
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
 if [[ -d "$SNAP" ]]; then
   # Exit if mode is not yet configured
   # (typically the case after the Snap is first installed)

multinode-demo/wallet.sh

@@ -9,6 +9,9 @@ here=$(dirname "$0")
 # shellcheck source=multinode-demo/common.sh
 source "$here"/common.sh
 
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
 # if $1 isn't host:path, something.com, or a valid local path
 if [[ ${1%:} != "$1" || "$1" =~ [^.]\.[^.] || -d $1 ]]; then
   leader=$1 # interpret

scripts/configure-metrics.sh

@@ -0,0 +1,45 @@
+# |source| this file
+#
+# The SOLANA_METRICS_CONFIG environment variable is formatted as a
+# comma-delimited list of parameters. All parameters are optional.
+#
+# Example:
+#   export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
+#
+configure_metrics() {
+  [[ -n $SOLANA_METRICS_CONFIG ]] || return 0
+
+  declare metrics_params
+  IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG"
+  for param in "${metrics_params[@]}"; do
+    IFS='=' read -r -a pair <<< "$param"
+    if [[ ${#pair[@]} != 2 ]]; then
+      echo Error: invalid metrics parameter: "$param" >&2
+    else
+      declare name="${pair[0]}"
+      declare value="${pair[1]}"
+      case "$name" in
+      host)
+        export INFLUX_HOST="$value"
+        echo INFLUX_HOST="$INFLUX_HOST" >&2
+        ;;
+      db)
+        export INFLUX_DATABASE="$value"
+        echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2
+        ;;
+      u)
+        export INFLUX_USERNAME="$value"
+        echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2
+        ;;
+      p)
+        export INFLUX_PASSWORD="$value"
+        echo INFLUX_PASSWORD="********" >&2
+        ;;
+      *)
+        echo Error: Unknown metrics parameter name: "$name" >&2
+        ;;
+      esac
+    fi
+  done
+}
+configure_metrics

scripts/oom-monitor.sh

@@ -4,8 +4,8 @@
 #
 
 here=$(dirname "$0")
-# shellcheck source=multinode-demo/common.sh
+# shellcheck source=scripts/oom-score-adj.sh
-source "$here"/common.sh
+source "$here"/oom-score-adj.sh
 
 if [[ $(uname) != Linux ]]; then
   exit 0
@@ -24,7 +24,7 @@ oom_score_adj "self" -500
 
 while read -r victim; do
   echo "Out of memory event detected, $victim killed"
-  "$here"/metrics_write_datapoint.sh "oom-killer,victim=$victim killed=1"
+  "$here"/metrics-write-datapoint.sh "oom-killer,victim=$victim killed=1"
 done < <( \
   tail --follow=name --retry -n0 $syslog \
   | sed --unbuffered -n 's/^.* Out of memory: Kill process [1-9][0-9]* (\([^)]*\)) .*/\1/p' \

scripts/oom-score-adj.sh

@@ -0,0 +1,20 @@
+# |source| this file
+#
+# Adjusts the OOM score for the specified process.  Linux only
+#
+# usage: oom_score_adj [pid] [score]
+#
+oom_score_adj() {
+  declare pid=$1
+  declare score=$2
+  if [[ $(uname) != Linux ]]; then
+    return
+  fi
+
+  echo "$score" > "/proc/$pid/oom_score_adj" || true
+  declare currentScore
+  currentScore=$(cat "/proc/$pid/oom_score_adj" || true)
+  if [[ $score != "$currentScore" ]]; then
+    echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)"
+  fi
+}

snap/hooks/configure

@@ -18,16 +18,16 @@ setup_args="$(snapctl get setup-args)"
 
 case $mode in
 leader+drone)
-  "$SNAP"/bin/setup.sh -t leader $num_tokens -p $setup_args
+  "$SNAP"/multinode-demo/setup.sh -t leader $num_tokens -p $setup_args
   snapctl start --enable solana.daemon-drone
   snapctl start --enable solana.daemon-leader
   ;;
 leader)
-  "$SNAP"/bin/setup.sh -t leader $num_tokens -p $setup_args
+  "$SNAP"/multinode-demo/setup.sh -t leader $num_tokens -p $setup_args
   snapctl start --enable solana.daemon-leader
   ;;
 validator)
-  "$SNAP"/bin/setup.sh -t validator -p $setup_args
+  "$SNAP"/multinode-demo/setup.sh -t validator -p $setup_args
   snapctl start --enable solana.daemon-validator
   ;;
 *)

snap/snapcraft.yaml

@@ -50,7 +50,7 @@ apps:
       - home
   bench-tps:
     # TODO: Merge client.sh functionality into solana-bench-tps proper
-    command: client.sh
+    command: multinode-demo/client.sh
     #command: solana-bench-tps
     plugs:
       - network
@@ -58,32 +58,32 @@ apps:
       - home
   wallet:
     # TODO: Merge wallet.sh functionality into solana-wallet proper
-    command: wallet.sh
+    command: multinode-demo/wallet.sh
     #command: solana-wallet
     plugs:
       - network
       - home
   daemon-validator:
     daemon: simple
-    command: validator.sh
+    command: multinode-demo/validator.sh
     plugs:
       - network
       - network-bind
   daemon-leader:
     daemon: simple
-    command: leader.sh
+    command: multinode-demo/leader.sh
     plugs:
       - network
       - network-bind
   daemon-drone:
     daemon: simple
-    command: drone.sh
+    command: multinode-demo/drone.sh
     plugs:
       - network
       - network-bind
   daemon-oom-monitor:
     daemon: simple
-    command: oom_monitor.sh
+    command: scripts/oom-monitor.sh
     plugs:
       - network
 
@@ -92,6 +92,8 @@ parts:
     plugin: nil
     prime:
       - bin
+      - multinode-demo
+      - scripts
       - usr/lib
     override-build: |
       # Install CUDA 9.2 runtime
@@ -114,9 +116,13 @@ parts:
       # Build/install all other programs
       cargo install --root $SNAPCRAFT_PART_INSTALL --bins
 
-      # Install multinode scripts
+      # Install multinode-demo/
-      mkdir -p $SNAPCRAFT_PART_INSTALL/bin
+      mkdir -p $SNAPCRAFT_PART_INSTALL/multinode-demo/
-      cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/bin/
+      cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/multinode-demo/
+
+      # Install scripts/
+      mkdir -p $SNAPCRAFT_PART_INSTALL/scripts/
+      cp -av scripts/* $SNAPCRAFT_PART_INSTALL/scripts/
 
       # TODO: build curl,rsync/multilog from source instead of sneaking it in from the host
       # system...