diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index bc30c58458..82bc0d974f 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -6,9 +6,9 @@ cd "$(dirname "$0")"/.. source ci/upload_ci_artifact.sh -source multinode-demo/common.sh +source scripts/configure-metrics.sh -./multinode-demo/setup.sh +multinode-demo/setup.sh backgroundCommands="drone leader validator validator-x" pids=() @@ -16,7 +16,7 @@ pids=() for cmd in $backgroundCommands; do echo "--- Start $cmd" rm -f log-"$cmd".txt - ./multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 & + multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 & declare pid=$! pids+=("$pid") echo "pid: $pid" @@ -67,11 +67,11 @@ flag_error() { # set -x # multinode-demo/test/wallet-sanity.sh # ) || flag_error -# +# # echo "--- Node count" # ( # set -x -# ./multinode-demo/client.sh "$PWD" 3 -c --addr 127.0.0.1 +# multinode-demo/client.sh "$PWD" 3 -c --addr 127.0.0.1 # ) || flag_error killBackgroundCommands @@ -79,6 +79,7 @@ killBackgroundCommands echo "--- Ledger verification" ( set -x + source multinode-demo/common.sh $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/ledger verify ) || flag_error diff --git a/ci/testnet-deploy.sh b/ci/testnet-deploy.sh index e70e15d9b8..da3bb79bab 100755 --- a/ci/testnet-deploy.sh +++ b/ci/testnet-deploy.sh @@ -6,7 +6,7 @@ # with GCP and has sufficient permission. # here=$(dirname "$0") -metrics_write_datapoint="$here"/../multinode-demo/metrics_write_datapoint.sh +metrics_write_datapoint="$here"/../scripts/metrics-write-datapoint.sh # TODO: Switch over to rolling updates ROLLING_UPDATE=false diff --git a/ci/testnet-sanity.sh b/ci/testnet-sanity.sh index c9da4d0a92..40dce414ab 100755 --- a/ci/testnet-sanity.sh +++ b/ci/testnet-sanity.sh @@ -4,7 +4,7 @@ # cd "$(dirname "$0")/.." -source multinode-demo/common.sh +source scripts/metrics-write-datapoint.sh NET_URL=$1 if [[ -z $NET_URL ]]; then @@ -45,7 +45,7 @@ echo "--- $NET_URL: node count" if [[ -n "$USE_SNAP" ]]; then # TODO: Merge client.sh functionality into solana-bench-tps proper and # remove this USE_SNAP case - cmd=$solana_bench_tps + cmd=solana.bench-tps else cmd=multinode-demo/client.sh fi diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index ceedf529ed..ae2bc8a71d 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -102,50 +102,8 @@ solana_ledger_tool=$(solana_program ledger-tool) export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info export RUST_BACKTRACE=1 - -# The SOLANA_METRICS_CONFIG environment variable is formatted as a -# comma-delimited list of parameters. All parameters are optional. -# -# Example: -# export SOLANA_METRICS_CONFIG="host=,db=,u=,p=" -# -configure_metrics() { - [[ -n $SOLANA_METRICS_CONFIG ]] || return 0 - - declare metrics_params - IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG" - for param in "${metrics_params[@]}"; do - IFS='=' read -r -a pair <<< "$param" - if [[ ${#pair[@]} != 2 ]]; then - echo Error: invalid metrics parameter: "$param" >&2 - else - declare name="${pair[0]}" - declare value="${pair[1]}" - case "$name" in - host) - export INFLUX_HOST="$value" - echo INFLUX_HOST="$INFLUX_HOST" >&2 - ;; - db) - export INFLUX_DATABASE="$value" - echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2 - ;; - u) - export INFLUX_USERNAME="$value" - echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2 - ;; - p) - export INFLUX_PASSWORD="$value" - echo INFLUX_PASSWORD="********" >&2 - ;; - *) - echo Error: Unknown metrics parameter name: "$name" >&2 - ;; - esac - fi - done -} -configure_metrics +# shellcheck source=scripts/configure-metrics.sh +source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh tune_networking() { # Skip in CI @@ -177,20 +135,6 @@ tune_networking() { fi } -oom_score_adj() { - declare pid=$1 - declare score=$2 - if [[ $(uname) != Linux ]]; then - return - fi - - echo "$score" > "/proc/$pid/oom_score_adj" || true - declare currentScore - currentScore=$(cat "/proc/$pid/oom_score_adj" || true) - if [[ $score != "$currentScore" ]]; then - echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)" - fi -} SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config SOLANA_CONFIG_PRIVATE_DIR=${SNAP_DATA:-$PWD}/config-private diff --git a/multinode-demo/drone.sh b/multinode-demo/drone.sh index 3895c71e09..f57c755931 100755 --- a/multinode-demo/drone.sh +++ b/multinode-demo/drone.sh @@ -10,6 +10,9 @@ here=$(dirname "$0") source "$here"/common.sh SOLANA_CONFIG_DIR="$SOLANA_CONFIG_DIR"-drone +# shellcheck source=scripts/oom-score-adj.sh +source "$here"/../scripts/oom-score-adj.sh + if [[ -d "$SNAP" ]]; then # Exit if mode is not yet configured # (typically the case after the Snap is first installed) diff --git a/multinode-demo/leader.sh b/multinode-demo/leader.sh index d5b9a9f40e..0530a48f67 100755 --- a/multinode-demo/leader.sh +++ b/multinode-demo/leader.sh @@ -7,6 +7,9 @@ here=$(dirname "$0") # shellcheck source=multinode-demo/common.sh source "$here"/common.sh +# shellcheck source=scripts/oom-score-adj.sh +source "$here"/../scripts/oom-score-adj.sh + if [[ -d "$SNAP" ]]; then # Exit if mode is not yet configured # (typically the case after the Snap is first installed) diff --git a/multinode-demo/wallet.sh b/multinode-demo/wallet.sh index 69a8bcec0b..d43b0e8762 100755 --- a/multinode-demo/wallet.sh +++ b/multinode-demo/wallet.sh @@ -9,6 +9,9 @@ here=$(dirname "$0") # shellcheck source=multinode-demo/common.sh source "$here"/common.sh +# shellcheck source=scripts/oom-score-adj.sh +source "$here"/../scripts/oom-score-adj.sh + # if $1 isn't host:path, something.com, or a valid local path if [[ ${1%:} != "$1" || "$1" =~ [^.]\.[^.] || -d $1 ]]; then leader=$1 # interpret diff --git a/scripts/configure-metrics.sh b/scripts/configure-metrics.sh new file mode 100644 index 0000000000..95b6e803d4 --- /dev/null +++ b/scripts/configure-metrics.sh @@ -0,0 +1,45 @@ +# |source| this file +# +# The SOLANA_METRICS_CONFIG environment variable is formatted as a +# comma-delimited list of parameters. All parameters are optional. +# +# Example: +# export SOLANA_METRICS_CONFIG="host=,db=,u=,p=" +# +configure_metrics() { + [[ -n $SOLANA_METRICS_CONFIG ]] || return 0 + + declare metrics_params + IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG" + for param in "${metrics_params[@]}"; do + IFS='=' read -r -a pair <<< "$param" + if [[ ${#pair[@]} != 2 ]]; then + echo Error: invalid metrics parameter: "$param" >&2 + else + declare name="${pair[0]}" + declare value="${pair[1]}" + case "$name" in + host) + export INFLUX_HOST="$value" + echo INFLUX_HOST="$INFLUX_HOST" >&2 + ;; + db) + export INFLUX_DATABASE="$value" + echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2 + ;; + u) + export INFLUX_USERNAME="$value" + echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2 + ;; + p) + export INFLUX_PASSWORD="$value" + echo INFLUX_PASSWORD="********" >&2 + ;; + *) + echo Error: Unknown metrics parameter name: "$name" >&2 + ;; + esac + fi + done +} +configure_metrics diff --git a/multinode-demo/metrics_write_datapoint.sh b/scripts/metrics-write-datapoint.sh similarity index 100% rename from multinode-demo/metrics_write_datapoint.sh rename to scripts/metrics-write-datapoint.sh diff --git a/multinode-demo/oom_monitor.sh b/scripts/oom-monitor.sh similarity index 82% rename from multinode-demo/oom_monitor.sh rename to scripts/oom-monitor.sh index 5fb392e3a2..a3e663f669 100755 --- a/multinode-demo/oom_monitor.sh +++ b/scripts/oom-monitor.sh @@ -4,8 +4,8 @@ # here=$(dirname "$0") -# shellcheck source=multinode-demo/common.sh -source "$here"/common.sh +# shellcheck source=scripts/oom-score-adj.sh +source "$here"/oom-score-adj.sh if [[ $(uname) != Linux ]]; then exit 0 @@ -24,7 +24,7 @@ oom_score_adj "self" -500 while read -r victim; do echo "Out of memory event detected, $victim killed" - "$here"/metrics_write_datapoint.sh "oom-killer,victim=$victim killed=1" + "$here"/metrics-write-datapoint.sh "oom-killer,victim=$victim killed=1" done < <( \ tail --follow=name --retry -n0 $syslog \ | sed --unbuffered -n 's/^.* Out of memory: Kill process [1-9][0-9]* (\([^)]*\)) .*/\1/p' \ diff --git a/scripts/oom-score-adj.sh b/scripts/oom-score-adj.sh new file mode 100644 index 0000000000..1f492bb993 --- /dev/null +++ b/scripts/oom-score-adj.sh @@ -0,0 +1,20 @@ +# |source| this file +# +# Adjusts the OOM score for the specified process. Linux only +# +# usage: oom_score_adj [pid] [score] +# +oom_score_adj() { + declare pid=$1 + declare score=$2 + if [[ $(uname) != Linux ]]; then + return + fi + + echo "$score" > "/proc/$pid/oom_score_adj" || true + declare currentScore + currentScore=$(cat "/proc/$pid/oom_score_adj" || true) + if [[ $score != "$currentScore" ]]; then + echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)" + fi +} diff --git a/snap/hooks/configure b/snap/hooks/configure index 59135c9445..f1e6c68774 100755 --- a/snap/hooks/configure +++ b/snap/hooks/configure @@ -18,16 +18,16 @@ setup_args="$(snapctl get setup-args)" case $mode in leader+drone) - "$SNAP"/bin/setup.sh -t leader $num_tokens -p $setup_args + "$SNAP"/multinode-demo/setup.sh -t leader $num_tokens -p $setup_args snapctl start --enable solana.daemon-drone snapctl start --enable solana.daemon-leader ;; leader) - "$SNAP"/bin/setup.sh -t leader $num_tokens -p $setup_args + "$SNAP"/multinode-demo/setup.sh -t leader $num_tokens -p $setup_args snapctl start --enable solana.daemon-leader ;; validator) - "$SNAP"/bin/setup.sh -t validator -p $setup_args + "$SNAP"/multinode-demo/setup.sh -t validator -p $setup_args snapctl start --enable solana.daemon-validator ;; *) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index daf3b456aa..da516fc4cb 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -50,7 +50,7 @@ apps: - home bench-tps: # TODO: Merge client.sh functionality into solana-bench-tps proper - command: client.sh + command: multinode-demo/client.sh #command: solana-bench-tps plugs: - network @@ -58,32 +58,32 @@ apps: - home wallet: # TODO: Merge wallet.sh functionality into solana-wallet proper - command: wallet.sh + command: multinode-demo/wallet.sh #command: solana-wallet plugs: - network - home daemon-validator: daemon: simple - command: validator.sh + command: multinode-demo/validator.sh plugs: - network - network-bind daemon-leader: daemon: simple - command: leader.sh + command: multinode-demo/leader.sh plugs: - network - network-bind daemon-drone: daemon: simple - command: drone.sh + command: multinode-demo/drone.sh plugs: - network - network-bind daemon-oom-monitor: daemon: simple - command: oom_monitor.sh + command: scripts/oom-monitor.sh plugs: - network @@ -92,6 +92,8 @@ parts: plugin: nil prime: - bin + - multinode-demo + - scripts - usr/lib override-build: | # Install CUDA 9.2 runtime @@ -114,9 +116,13 @@ parts: # Build/install all other programs cargo install --root $SNAPCRAFT_PART_INSTALL --bins - # Install multinode scripts - mkdir -p $SNAPCRAFT_PART_INSTALL/bin - cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/bin/ + # Install multinode-demo/ + mkdir -p $SNAPCRAFT_PART_INSTALL/multinode-demo/ + cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/multinode-demo/ + + # Install scripts/ + mkdir -p $SNAPCRAFT_PART_INSTALL/scripts/ + cp -av scripts/* $SNAPCRAFT_PART_INSTALL/scripts/ # TODO: build curl,rsync/multilog from source instead of sneaking it in from the host # system...