Stable dashboard can now actually come from the stable channel

This reverts commit ca12faca9c.

.buildkite/env/README.md

@@ -0,0 +1,31 @@
+
+[ejson](https://github.com/Shopify/ejson) and
+[ejson2env](https://github.com/Shopify/ejson2env) are used to manage access
+tokens and other secrets required for CI.
+
+#### Setup
+```bash
+$ sudo gem install ejson ejson2env
+```
+
+then obtain the necessary keypair and place it in `/opt/ejson/keys/`.
+
+#### Usage
+Run the following command to decrypt the secrets into the environment:
+```bash
+eval $(ejson2env secrets.ejson)
+```
+
+#### Managing secrets.ejson
+To decrypt `secrets.ejson` for modification, run:
+```bash
+$ ejson decrypt secrets.ejson -o secrets_unencrypted.ejson
+```
+
+Edit, then run the following to re-encrypt the file **BEFORE COMMITING YOUR
+CHANGES**:
+```bash
+$ ejson encrypt secrets_unencrypted.ejson
+$ mv secrets_unencrypted.ejson secrets.ejson
+```
+

.buildkite/env/secrets.ejson

@@ -0,0 +1,10 @@
+{
+    "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
+    "environment": {
+      "CODECOV_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:EzVa4Gpj2Qn5OhZQlVfGFchuROgupvnW:CbWc6sNh1GCrAbrncxDjW00zUAD/Sa+ccg7CFSz8Ua6LnCYnSddTBxJWcJEbEs0MrjuZRQ==]",
+      "CRATES_IO_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:qF7QrUM8j+19mptcE1YS71CqmrCM13Ah:TZCatJeT1egCHiufE6cGFC1VsdJkKaaqV6QKWkEsMPBKvOAdaZbbVz9Kl+lGnIsF]",
+      "INFLUX_DATABASE": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:PetD/4c/EbkQmFEcK21g3cBBAPwFqHEw:wvYmDZRajy2WngVFs9AlwyHk]",
+      "INFLUX_USERNAME": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:WcnqZdmDFtJJ01Zu5LbeGgbYGfRzBdFc:a7c5zDDtCOu5L1Qd2NKkxT6kljyBcbck]",
+      "INFLUX_PASSWORD": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:LIZgP9Tp9yE9OlpV8iogmLOI7iW7SiU3:x0nYdT1A6sxu+O+MMLIN19d2t6rrK1qJ3+HnoWG3PDodsXjz06YJWQKU/mx6saqH+QbGtGV5mk0=]"
+    }
+}

.buildkite/hooks/pre-command

@@ -1,4 +1,7 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
+
+eval "$(ejson2env .buildkite/env/secrets.ejson)"
 
 # Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
 # interprets this as the start of a log group.
@@ -24,4 +27,3 @@ export PS4="++"
   set -x
   rsync -a --delete --link-dest="$d" "$d"/target .
 )
-

.buildkite/pipeline-upload.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# This script is used to upload the full buildkite pipeline. The steps defined
+# in the buildkite UI should simply be:
+#
+#   steps:
+#    - command: "ci/buildkite-pipeline-upload.sh"
+#
+
+set -e
+cd "$(dirname "$0")"/..
+
+buildkite-agent pipeline upload ci/buildkite.yml
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  # Add helpful link back to the corresponding Github Pull Request
+  buildkite-agent annotate --style "info" \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+fi
+

.gitignore

@@ -1,4 +1,3 @@
-Cargo.lock
 /target/
 
 **/*.rs.bk

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.10.3"
+version = "0.10.5"
 documentation = "https://docs.rs/solana"
 homepage = "http://solana.com/"
 readme = "README.md"
@@ -104,14 +104,14 @@ serde_cbor = "0.9.0"
 serde_derive = "1.0.27"
 serde_json = "1.0.10"
 socket2 = "0.3.8"
-solana-sdk = { path = "sdk", version = "0.10.3" }
+solana-sdk = { path = "sdk", version = "0.10.5" }
 sys-info = "0.5.6"
 tokio = "0.1"
 tokio-codec = "0.1"
 untrusted = "0.6.2"
-solana-noop = { path = "programs/native/noop", version = "0.10.3" }
+solana-noop = { path = "programs/native/noop", version = "0.10.5" }
-solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.3" }
+solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.5" }
-solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.3" }
+solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.5" }
 
 [[bench]]
 name = "bank"

build.rs

@@ -8,7 +8,7 @@ fn main() {
     // Ensure target/perf-libs/ exists.  It's been observed that
     // a cargo:rerun-if-changed= directive with a non-existent
     // directory triggers a rebuild on every |cargo build| invocation
-    fs::create_dir("target/perf-libs").unwrap_or_else(|err| {
+    fs::create_dir_all("target/perf-libs").unwrap_or_else(|err| {
         if err.kind() != std::io::ErrorKind::AlreadyExists {
             panic!("Unable to create target/perf-libs: {:?}", err);
         }

ci/publish-metrics-dashboard.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z $BUILDKITE ]]; then
+  echo BUILDKITE not defined
+  exit 1
+fi
+
+if [[ -z $CHANNEL ]]; then
+  CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
+fi
+
+if [[ -z $CHANNEL ]]; then
+  (
+    cat <<EOF
+steps:
+  - block: "Select Dashboard"
+    fields:
+      - select: "Channel"
+        key: "channel"
+        options:
+          - label: "stable"
+            value: "stable"
+          - label: "edge"
+            value: "edge"
+          - label: "beta"
+            value: "beta"
+  - command: "ci/$(basename "$0")"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+
+ci/channel-info.sh
+eval "$(ci/channel-info.sh)"
+
+case $CHANNEL in
+edge)
+  CHANNEL_BRANCH=$EDGE_CHANNEL
+  ;;
+beta)
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+stable)
+  CHANNEL_BRANCH=$STABLE_CHANNEL
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+  (
+    cat <<EOF
+steps:
+  - trigger: "$BUILDKITE_PIPELINE_SLUG"
+    async: true
+    build:
+      message: "$BUILDKITE_MESSAGE"
+      branch: "$CHANNEL_BRANCH"
+      env:
+        CHANNEL: "$CHANNEL"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+set -x
+exec metrics/publish-metrics-dashboard.sh "$CHANNEL"

ci/testnet-manager.sh

@@ -83,10 +83,11 @@ testnet-beta|testnet-beta-perf)
 testnet|testnet-perf)
   if [[ -n $BETA_CHANNEL_LATEST_TAG ]]; then
     CHANNEL_OR_TAG=$BETA_CHANNEL_LATEST_TAG
+    CHANNEL_BRANCH=$BETA_CHANNEL
   else
     CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+    CHANNEL_BRANCH=$STABLE_CHANNEL
   fi
-  CHANNEL_BRANCH=$BETA_CHANNEL
   ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"

ci/testnet-sanity.sh

@@ -32,10 +32,10 @@ zone=$3
 
 set -x
 echo "--- $cloudProvider.sh config"
-net/"$cloudProvider".sh config -p "$netName" -z "$zone"
+timeout 5m net/"$cloudProvider".sh config -p "$netName" -z "$zone"
 net/init-metrics.sh -e
 echo --- net.sh sanity
-net/net.sh sanity \
+timeout 5m net/net.sh sanity \
   ${NO_LEDGER_VERIFY:+-o noLedgerVerify} \
   ${NO_VALIDATOR_SANITY:+-o noValidatorSanity} \
   ${REJECT_EXTRA_NODES:+-o rejectExtraNodes} \

metrics/README.md

@@ -0,0 +1,39 @@
+# Metrics
+
+## Testnet Grafana Dashboard
+
+There are three versions of the testnet dashboard, corresponding to the three
+release channels:
+* https://metrics.solana.com:3000/d/testnet-edge/testnet-monitor-edge
+* https://metrics.solana.com:3000/d/testnet-beta/testnet-monitor-beta
+* https://metrics.solana.com:3000/d/testnet/testnet-monitor
+
+The dashboard for each channel is defined from the
+`metrics/testnet-monitor.json` source file in the git branch associated with
+that channel, and deployed by automation running `ci/publish-metrics-dashboard.sh`.
+
+A deploy can be triggered at any time via the `New Build` button of
+https://buildkite.com/solana-labs/publish-metrics-dashboard.
+
+### Modifying a Dashboard
+
+Dashboard updates are accomplished by modifying `metrics/testnet-monitor.json`,
+**manual edits made directly in Grafana will be overwritten**.
+
+1. Open the desired dashboard in Grafana
+2. Create a development copy of the dashboard by selecting `Save As..` in the
+   `Settings` menu for the dashboard
+3. Edit dashboard as desired
+4. Extract the JSON Model by selecting `JSON Model` in the `Settings` menu.  Copy the JSON to the clipboard
+    and paste into `metrics/testnet-monitor.json`
+5. Delete your development dashboard: `Settings` => `Delete`
+
+### Deploying a Dashboard Manually
+
+If you need to immediately deploy a dashboard using the contents of
+`metrics/testnet-monitor.json` in your local workspace,
+```
+$ export GRAFANA_API_TOKEN="an API key from https://metrics.solana.com:3000/org/apikeys"
+$ metrics/publish-metrics-dashboard.sh (edge|beta|stable)
+```
+Note that automation will eventually overwrite your manual deploy.

metrics/adjust-dashboard-for-channel.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Adjusts the testnet monitor dashboard for the specified release channel
+#
+
+import sys
+import json
+
+if len(sys.argv) != 3:
+    print('Error: Dashboard or Channel not specified')
+    sys.exit(1)
+
+dashboard_json = sys.argv[1]
+channel = sys.argv[2]
+if channel not in ['edge', 'beta', 'stable']:
+    print('Error: Unknown channel:', channel)
+    sys.exit(2)
+
+with open(dashboard_json, 'r') as read_file:
+    data = json.load(read_file)
+
+if channel == 'stable':
+    # Stable dashboard only allows the user to select between the stable
+    # testnet databases
+    data['title'] = 'Testnet Monitor'
+    data['uid'] = 'testnet'
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [{'selected': False,
+                                                'text': 'testnet',
+                                                'value': 'testnet'},
+                                               {'selected': True,
+                                                'text': 'testnet-perf',
+                                                'value': 'testnet-perf'}],
+                                   'query': 'testnet,testnet-perf',
+                                   'type': 'custom'}]
+else:
+    # Non-stable dashboard only allows the user to select between all testnet
+    # databases
+    data['title'] = 'Testnet Monitor ({})'.format(channel)
+    data['uid'] = 'testnet-' + channel
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'datasource': 'Solana Metrics (read-only)',
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [],
+                                   'query': 'show databases',
+                                   'refresh': 1,
+                                   'regex': 'testnet.*',
+                                   'sort': 1,
+                                   'tagValuesQuery': '',
+                                   'tags': [],
+                                   'tagsQuery': '',
+                                   'type': 'query',
+                                   'useTags': False}]
+
+with open(dashboard_json, 'w') as write_file:
+    json.dump(data, write_file, indent=2)

metrics/grafcli.conf

@@ -0,0 +1,15 @@
+[grafcli]
+editor = vim
+mergetool = vimdiff
+verbose = on
+force = on
+
+[resources]
+
+[hosts]
+metrics = on
+
+[metrics]
+type = api
+url = https://metrics.solana.com:3000/api
+ssl = off

metrics/publish-metrics-dashboard.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")"
+
+CHANNEL=$1
+if [[ -z $CHANNEL ]]; then
+  echo "usage: $0 [channel]"
+  exit 1
+fi
+
+case $CHANNEL in
+edge)
+  DASHBOARD=testnet-monitor-edge
+  ;;
+beta)
+  DASHBOARD=testnet-monitor-beta
+  ;;
+stable)
+  DASHBOARD=testnet-monitor
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+
+if [[ -z $GRAFANA_API_TOKEN ]]; then
+  echo Error: GRAFANA_API_TOKEN not defined
+  exit 1
+fi
+
+DASHBOARD_JSON=./testnet-monitor.json
+if [[ ! -r $DASHBOARD_JSON ]]; then
+  echo Error: $DASHBOARD_JSON not found
+fi
+
+(
+  set -x
+  ./adjust-dashboard-for-channel.py "$DASHBOARD_JSON" "$CHANNEL"
+)
+
+rm -rf venv
+python3 -m venv venv
+# shellcheck source=/dev/null
+source venv/bin/activate
+
+echo --- Fetch/build grafcli
+(
+  set -x
+  git clone git@github.com:mvines/grafcli.git -b experimental-v5 venv/grafcli
+  cd venv/grafcli
+  python3 setup.py install
+)
+
+echo --- Take a backup of existing dashboard if possible
+(
+  set -x +e
+  grafcli export remote/metrics/$DASHBOARD $DASHBOARD_JSON.org
+  grafcli rm remote/metrics/$DASHBOARD
+  :
+)
+
+echo --- Publish $DASHBOARD_JSON to $DASHBOARD
+(
+  set -x
+  grafcli import $DASHBOARD_JSON remote/metrics
+)
+
+exit 0

net/scripts/install-earlyoom.sh

@@ -13,8 +13,8 @@ sysctl -w kernel.sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
 if command -v earlyoom; then
   systemctl status earlyoom
 else
-  wget  -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.1-*_amd64.deb' -e robots=off -nd
+  wget  -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.2-*_amd64.deb' -e robots=off -nd
-  apt install --quiet --yes ./earlyoom_1.1-*_amd64.deb
+  apt install --quiet --yes ./earlyoom_1.2-*_amd64.deb
 
   cat > earlyoom <<OOM
   # use the kernel OOM killer, trigger at 20% available RAM,

net/scripts/install-libssl-compatability.sh

@@ -12,7 +12,6 @@ apt-get --assume-yes install libssl-dev
 #
 # cc: https://github.com/solana-labs/solana/issues/1090
 # cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
-wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+dpkg -i libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+rm libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-

programs/bpf/rust/noop/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-noop"
-version = "0.10.3"
+version = "0.10.5"
 description = "Solana BPF noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -8,4 +8,4 @@ license = "Apache-2.0"
 
 [dependencies]
 rbpf = "0.1.0"
-solana-sdk = { path = "../../../../sdk", version = "0.10.3" }
+solana-sdk = { path = "../../../../sdk", version = "0.10.5" }

programs/native/bpf_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpfloader"
-version = "0.10.3"
+version = "0.10.5"
 description = "Solana BPF Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,7 +16,7 @@ log = "0.4.2"
 solana_rbpf = "0.1.3"
 serde = "1.0.27"
 serde_derive = "1.0.27"
-solana-sdk = { path = "../../../sdk", version = "0.10.3" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [lib]
 name = "solana_bpf_loader"

programs/native/lua_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-lualoader"
-version = "0.10.3"
+version = "0.10.5"
 description = "Solana Lua Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ log = "0.4.2"
 rlua = "0.15.2"
 serde = "1.0.27"
 serde_derive = "1.0.27"
-solana-sdk = { path = "../../../sdk", version = "0.10.3" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [dev-dependencies]
 bincode = "1.0.0"

programs/native/noop/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "solana-noop"
-version = "0.10.3"
+version = "0.10.5"
 description = "Solana noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 
 [dependencies]
-solana-sdk = { path = "../../../sdk", version = "0.10.3" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [lib]
 name = "noop"

sdk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk"
-version = "0.10.3"
+version = "0.10.5"
 description = "Solana SDK"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"