Stable dashboard can now actually come from the stable channel

This reverts commit ca12faca9c.

.buildkite/env/README.md

@@ -0,0 +1,31 @@
+
+[ejson](https://github.com/Shopify/ejson) and
+[ejson2env](https://github.com/Shopify/ejson2env) are used to manage access
+tokens and other secrets required for CI.
+
+#### Setup
+```bash
+$ sudo gem install ejson ejson2env
+```
+
+then obtain the necessary keypair and place it in `/opt/ejson/keys/`.
+
+#### Usage
+Run the following command to decrypt the secrets into the environment:
+```bash
+eval $(ejson2env secrets.ejson)
+```
+
+#### Managing secrets.ejson
+To decrypt `secrets.ejson` for modification, run:
+```bash
+$ ejson decrypt secrets.ejson -o secrets_unencrypted.ejson
+```
+
+Edit, then run the following to re-encrypt the file **BEFORE COMMITING YOUR
+CHANGES**:
+```bash
+$ ejson encrypt secrets_unencrypted.ejson
+$ mv secrets_unencrypted.ejson secrets.ejson
+```
+

.buildkite/env/secrets.ejson

@@ -0,0 +1,10 @@
+{
+    "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
+    "environment": {
+      "CODECOV_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:EzVa4Gpj2Qn5OhZQlVfGFchuROgupvnW:CbWc6sNh1GCrAbrncxDjW00zUAD/Sa+ccg7CFSz8Ua6LnCYnSddTBxJWcJEbEs0MrjuZRQ==]",
+      "CRATES_IO_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:qF7QrUM8j+19mptcE1YS71CqmrCM13Ah:TZCatJeT1egCHiufE6cGFC1VsdJkKaaqV6QKWkEsMPBKvOAdaZbbVz9Kl+lGnIsF]",
+      "INFLUX_DATABASE": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:PetD/4c/EbkQmFEcK21g3cBBAPwFqHEw:wvYmDZRajy2WngVFs9AlwyHk]",
+      "INFLUX_USERNAME": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:WcnqZdmDFtJJ01Zu5LbeGgbYGfRzBdFc:a7c5zDDtCOu5L1Qd2NKkxT6kljyBcbck]",
+      "INFLUX_PASSWORD": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:LIZgP9Tp9yE9OlpV8iogmLOI7iW7SiU3:x0nYdT1A6sxu+O+MMLIN19d2t6rrK1qJ3+HnoWG3PDodsXjz06YJWQKU/mx6saqH+QbGtGV5mk0=]"
+    }
+}

.buildkite/hooks/pre-command

@@ -1,4 +1,7 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
+
+eval "$(ejson2env .buildkite/env/secrets.ejson)"
 
 # Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
 # interprets this as the start of a log group.
@@ -24,4 +27,3 @@ export PS4="++"
   set -x
   rsync -a --delete --link-dest="$d" "$d"/target .
 )
-

.buildkite/pipeline-upload.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# This script is used to upload the full buildkite pipeline. The steps defined
+# in the buildkite UI should simply be:
+#
+#   steps:
+#    - command: "ci/buildkite-pipeline-upload.sh"
+#
+
+set -e
+cd "$(dirname "$0")"/..
+
+buildkite-agent pipeline upload ci/buildkite.yml
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  # Add helpful link back to the corresponding Github Pull Request
+  buildkite-agent annotate --style "info" \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+fi
+

.gitignore

@@ -1,4 +1,3 @@
-Cargo.lock
 /target/
 
 **/*.rs.bk

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.10.1"
+version = "0.10.5"
 documentation = "https://docs.rs/solana"
 homepage = "http://solana.com/"
 readme = "README.md"
@@ -104,14 +104,14 @@ serde_cbor = "0.9.0"
 serde_derive = "1.0.27"
 serde_json = "1.0.10"
 socket2 = "0.3.8"
-solana-sdk = { path = "sdk", version = "0.10.1" }
+solana-sdk = { path = "sdk", version = "0.10.5" }
 sys-info = "0.5.6"
 tokio = "0.1"
 tokio-codec = "0.1"
 untrusted = "0.6.2"
-solana-noop = { path = "programs/native/noop", version = "0.10.1" }
+solana-noop = { path = "programs/native/noop", version = "0.10.5" }
-solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.1" }
+solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.5" }
-solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.1" }
+solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.5" }
 
 [[bench]]
 name = "bank"

build.rs

@@ -8,7 +8,7 @@ fn main() {
     // Ensure target/perf-libs/ exists.  It's been observed that
     // a cargo:rerun-if-changed= directive with a non-existent
     // directory triggers a rebuild on every |cargo build| invocation
-    fs::create_dir("target/perf-libs").unwrap_or_else(|err| {
+    fs::create_dir_all("target/perf-libs").unwrap_or_else(|err| {
         if err.kind() != std::io::ErrorKind::AlreadyExists {
             panic!("Unable to create target/perf-libs: {:?}", err);
         }

ci/buildkite.yml

@@ -1,5 +1,5 @@
 steps:
-  - command: "ci/docker-run.sh solanalabs/rust:1.30.0 ci/test-stable.sh"
+  - command: "ci/docker-run.sh solanalabs/rust:1.30.1 ci/test-stable.sh"
     name: "stable [public]"
     env:
       CARGO_TARGET_CACHE_NAME: "stable"

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/buildkite.yml to pick up the new image tag
-FROM rust:1.30.0
+FROM rust:1.30.1
 
 RUN set -x && \
     apt update && \

ci/publish-metrics-dashboard.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z $BUILDKITE ]]; then
+  echo BUILDKITE not defined
+  exit 1
+fi
+
+if [[ -z $CHANNEL ]]; then
+  CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
+fi
+
+if [[ -z $CHANNEL ]]; then
+  (
+    cat <<EOF
+steps:
+  - block: "Select Dashboard"
+    fields:
+      - select: "Channel"
+        key: "channel"
+        options:
+          - label: "stable"
+            value: "stable"
+          - label: "edge"
+            value: "edge"
+          - label: "beta"
+            value: "beta"
+  - command: "ci/$(basename "$0")"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+
+ci/channel-info.sh
+eval "$(ci/channel-info.sh)"
+
+case $CHANNEL in
+edge)
+  CHANNEL_BRANCH=$EDGE_CHANNEL
+  ;;
+beta)
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+stable)
+  CHANNEL_BRANCH=$STABLE_CHANNEL
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+  (
+    cat <<EOF
+steps:
+  - trigger: "$BUILDKITE_PIPELINE_SLUG"
+    async: true
+    build:
+      message: "$BUILDKITE_MESSAGE"
+      branch: "$CHANNEL_BRANCH"
+      env:
+        CHANNEL: "$CHANNEL"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+set -x
+exec metrics/publish-metrics-dashboard.sh "$CHANNEL"

ci/publish-solana-tar.sh

@@ -5,6 +5,7 @@ cd "$(dirname "$0")/.."
 DRYRUN=
 if [[ -z $BUILDKITE_BRANCH ]]; then
   DRYRUN="echo"
+  CHANNEL=unknown
 fi
 
 eval "$(ci/channel-info.sh)"
@@ -17,11 +18,6 @@ elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
   CHANNEL=beta
 fi
 
-if [[ -z $CHANNEL ]]; then
-  echo Unable to determine channel to publish into, exiting.
-  exit 0
-fi
-
 if [[ -n "$BUILDKITE_TAG" ]]; then
   CHANNEL_OR_TAG=$BUILDKITE_TAG
 elif [[ -n "$TRIGGERED_BUILDKITE_TAG" ]]; then
@@ -30,37 +26,46 @@ else
   CHANNEL_OR_TAG=$CHANNEL
 fi
 
+if [[ -z $CHANNEL_OR_TAG ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+
 echo --- Creating tarball
-if [[ -z $DRYRUN ]]; then
 (
   set -x
   rm -rf solana-release/
   mkdir solana-release/
   (
-    echo "$CHANNEL"
+    echo "$CHANNEL_OR_TAG"
     git rev-parse HEAD
   ) > solana-release/version.txt
 
+  cargo install --root solana-release
+  ./scripts/install-native-programs.sh solana-release/bin
   ./fetch-perf-libs.sh
-  cargo install --features=cuda --root solana-release
+  cargo install --features=cuda --root solana-release-cuda
-  ./scripts/install-native-programs.sh solana-release
+  cp solana-release-cuda/bin/solana-fullnode solana-release/bin/solana-fullnode-cuda
 
   tar jvcf solana-release.tar.bz2 solana-release/
 )
-fi
-
 
 echo --- AWS S3 Store
+if [[ -z $DRYRUN ]]; then
+  (
+    set -x
+    if [[ ! -r s3cmd-2.0.1/s3cmd ]]; then
+      rm -rf s3cmd-2.0.1.tar.gz s3cmd-2.0.1
+      $DRYRUN wget https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gz
+      $DRYRUN tar zxf s3cmd-2.0.1.tar.gz
+    fi
 
-set -x
+    $DRYRUN python ./s3cmd-2.0.1/s3cmd --acl-public put solana-release.tar.bz2 \
-if [[ ! -r s3cmd-2.0.1/s3cmd ]]; then
+      s3://solana-release/"$CHANNEL_OR_TAG"/solana-release.tar.bz2
-  rm -rf s3cmd-2.0.1.tar.gz s3cmd-2.0.1
+  )
-  $DRYRUN wget https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gz
+else
-  $DRYRUN tar zxf s3cmd-2.0.1.tar.gz
+  echo Skipped due to DRYRUN
 fi
-
-$DRYRUN python ./s3cmd-2.0.1/s3cmd --acl-public put solana-release.tar.bz2 \
-  s3://solana-release/"$CHANNEL_OR_TAG"/solana-release.tar.bz2
-
 exit 0
 

ci/testnet-deploy.sh

@@ -9,10 +9,10 @@ clientNodeCount=0
 validatorNodeCount=10
 publicNetwork=false
 snapChannel=edge
-releaseChannel=edge
+tarChannelOrTag=edge
 delete=false
 enableGpu=false
-useReleaseChannel=false
+useTarReleaseChannel=false
 
 usage() {
   exitcode=0
@@ -32,8 +32,10 @@ Deploys a CD testnet
   options:
    -s edge|beta|stable  - Deploy the specified Snap release channel
                           (default: $snapChannel)
-   -t edge|beta|stable  - Deploy the specified prebuilt tar from channel
+   -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
-                          (default: $releaseChannel)
+                                 specified release channel (edge|beta|stable) or release tag
+                                 (vX.Y.Z)
+                                 (default: $tarChannelOrTag)
    -n [number]          - Number of validator nodes (default: $validatorNodeCount)
    -c [number]          - Number of client nodes (default: $clientNodeCount)
    -P                   - Use public network IP addresses (default: $publicNetwork)
@@ -82,9 +84,9 @@ while getopts "h?p:Pn:c:s:t:gG:a:d" opt; do
     ;;
   t)
     case $OPTARG in
-    edge|beta|stable)
+    edge|beta|stable|v*)
-      releaseChannel=$OPTARG
+      tarChannelOrTag=$OPTARG
-      useReleaseChannel=true
+      useTarReleaseChannel=true
       ;;
     *)
       usage "Invalid release channel: $OPTARG"
@@ -157,9 +159,9 @@ if [[ -n $NO_LEDGER_VERIFY ]]; then
   maybeNoLedgerVerify="-o noLedgerVerify"
 fi
 # shellcheck disable=SC2086 # Don't want to double quote maybeRejectExtraNodes
-if ! $useReleaseChannel; then
+if $useTarReleaseChannel; then
-  time net/net.sh start -s "$snapChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+  time net/net.sh start -t "$tarChannelOrTag" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
 else
-  time net/net.sh start -t "$releaseChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+  time net/net.sh start -s "$snapChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
 fi
 exit 0

ci/testnet-manager.sh

@@ -83,10 +83,11 @@ testnet-beta|testnet-beta-perf)
 testnet|testnet-perf)
   if [[ -n $BETA_CHANNEL_LATEST_TAG ]]; then
     CHANNEL_OR_TAG=$BETA_CHANNEL_LATEST_TAG
+    CHANNEL_BRANCH=$BETA_CHANNEL
   else
     CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+    CHANNEL_BRANCH=$STABLE_CHANNEL
   fi
-  CHANNEL_BRANCH=$BETA_CHANNEL
   ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"
@@ -122,7 +123,7 @@ sanity() {
       set -ex
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-east-1b
+      ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-edge-perf)
@@ -133,7 +134,7 @@ sanity() {
       export REJECT_EXTRA_NODES=1
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-east-1a
+      ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
     )
     ;;
   testnet-beta)
@@ -185,7 +186,8 @@ sanity() {
       set -ex
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-sanity.sh testnet-solana-com gce us-east1-c
+      #ci/testnet-sanity.sh testnet-solana-com gce us-east1-c
+      ci/testnet-sanity.sh testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-perf)
@@ -196,6 +198,7 @@ sanity() {
       export REJECT_EXTRA_NODES=1
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
+      #ci/testnet-sanity.sh perf-testnet-solana-com ec2 us-east-1a
       ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
     )
     ;;
@@ -223,8 +226,8 @@ start() {
       set -ex
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-east-1b \
+      ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
-        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P \
+        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0ccd4f2239886fa94 \
         ${maybeDelete:+-d}
     )
     ;;
@@ -235,7 +238,7 @@ start() {
       set -ex
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-east-1a \
+      ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
         -g -t "$CHANNEL_OR_TAG" -c 2 \
         ${maybeDelete:+-d}
     )
@@ -248,7 +251,7 @@ start() {
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
       ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
-        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P \
+        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0f286cf8a0771ce35 \
         ${maybeDelete:+-d}
     )
     ;;
@@ -296,8 +299,11 @@ start() {
       set -ex
       export NO_LEDGER_VERIFY=1
       export NO_VALIDATOR_SANITY=1
-      ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+      #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
-        -s "$CHANNEL_OR_TAG" -n 3 -g -c 0 -P -a testnet-solana-com  \
+      #  -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+      #  ${maybeDelete:+-d}
+      ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
+        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0fa502bf95f6f18b2 \
         ${maybeDelete:+-d}
     )
     ;;
@@ -312,6 +318,10 @@ start() {
         -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
         -t "$CHANNEL_OR_TAG" -c 2 \
         ${maybeDelete:+-d}
+      #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
+      #  -g \
+      #  -t "$CHANNEL_OR_TAG" -c 2 \
+      #  ${maybeDelete:+-d}
     )
     ;;
   *)

ci/testnet-sanity.sh

@@ -32,10 +32,10 @@ zone=$3
 
 set -x
 echo "--- $cloudProvider.sh config"
-net/"$cloudProvider".sh config -p "$netName" -z "$zone"
+timeout 5m net/"$cloudProvider".sh config -p "$netName" -z "$zone"
 net/init-metrics.sh -e
 echo --- net.sh sanity
-net/net.sh sanity \
+timeout 5m net/net.sh sanity \
   ${NO_LEDGER_VERIFY:+-o noLedgerVerify} \
   ${NO_VALIDATOR_SANITY:+-o noValidatorSanity} \
   ${REJECT_EXTRA_NODES:+-o rejectExtraNodes} \

metrics/README.md

@@ -0,0 +1,39 @@
+# Metrics
+
+## Testnet Grafana Dashboard
+
+There are three versions of the testnet dashboard, corresponding to the three
+release channels:
+* https://metrics.solana.com:3000/d/testnet-edge/testnet-monitor-edge
+* https://metrics.solana.com:3000/d/testnet-beta/testnet-monitor-beta
+* https://metrics.solana.com:3000/d/testnet/testnet-monitor
+
+The dashboard for each channel is defined from the
+`metrics/testnet-monitor.json` source file in the git branch associated with
+that channel, and deployed by automation running `ci/publish-metrics-dashboard.sh`.
+
+A deploy can be triggered at any time via the `New Build` button of
+https://buildkite.com/solana-labs/publish-metrics-dashboard.
+
+### Modifying a Dashboard
+
+Dashboard updates are accomplished by modifying `metrics/testnet-monitor.json`,
+**manual edits made directly in Grafana will be overwritten**.
+
+1. Open the desired dashboard in Grafana
+2. Create a development copy of the dashboard by selecting `Save As..` in the
+   `Settings` menu for the dashboard
+3. Edit dashboard as desired
+4. Extract the JSON Model by selecting `JSON Model` in the `Settings` menu.  Copy the JSON to the clipboard
+    and paste into `metrics/testnet-monitor.json`
+5. Delete your development dashboard: `Settings` => `Delete`
+
+### Deploying a Dashboard Manually
+
+If you need to immediately deploy a dashboard using the contents of
+`metrics/testnet-monitor.json` in your local workspace,
+```
+$ export GRAFANA_API_TOKEN="an API key from https://metrics.solana.com:3000/org/apikeys"
+$ metrics/publish-metrics-dashboard.sh (edge|beta|stable)
+```
+Note that automation will eventually overwrite your manual deploy.

metrics/adjust-dashboard-for-channel.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Adjusts the testnet monitor dashboard for the specified release channel
+#
+
+import sys
+import json
+
+if len(sys.argv) != 3:
+    print('Error: Dashboard or Channel not specified')
+    sys.exit(1)
+
+dashboard_json = sys.argv[1]
+channel = sys.argv[2]
+if channel not in ['edge', 'beta', 'stable']:
+    print('Error: Unknown channel:', channel)
+    sys.exit(2)
+
+with open(dashboard_json, 'r') as read_file:
+    data = json.load(read_file)
+
+if channel == 'stable':
+    # Stable dashboard only allows the user to select between the stable
+    # testnet databases
+    data['title'] = 'Testnet Monitor'
+    data['uid'] = 'testnet'
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [{'selected': False,
+                                                'text': 'testnet',
+                                                'value': 'testnet'},
+                                               {'selected': True,
+                                                'text': 'testnet-perf',
+                                                'value': 'testnet-perf'}],
+                                   'query': 'testnet,testnet-perf',
+                                   'type': 'custom'}]
+else:
+    # Non-stable dashboard only allows the user to select between all testnet
+    # databases
+    data['title'] = 'Testnet Monitor ({})'.format(channel)
+    data['uid'] = 'testnet-' + channel
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'datasource': 'Solana Metrics (read-only)',
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [],
+                                   'query': 'show databases',
+                                   'refresh': 1,
+                                   'regex': 'testnet.*',
+                                   'sort': 1,
+                                   'tagValuesQuery': '',
+                                   'tags': [],
+                                   'tagsQuery': '',
+                                   'type': 'query',
+                                   'useTags': False}]
+
+with open(dashboard_json, 'w') as write_file:
+    json.dump(data, write_file, indent=2)

metrics/grafcli.conf

@@ -0,0 +1,15 @@
+[grafcli]
+editor = vim
+mergetool = vimdiff
+verbose = on
+force = on
+
+[resources]
+
+[hosts]
+metrics = on
+
+[metrics]
+type = api
+url = https://metrics.solana.com:3000/api
+ssl = off

metrics/publish-metrics-dashboard.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")"
+
+CHANNEL=$1
+if [[ -z $CHANNEL ]]; then
+  echo "usage: $0 [channel]"
+  exit 1
+fi
+
+case $CHANNEL in
+edge)
+  DASHBOARD=testnet-monitor-edge
+  ;;
+beta)
+  DASHBOARD=testnet-monitor-beta
+  ;;
+stable)
+  DASHBOARD=testnet-monitor
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+
+if [[ -z $GRAFANA_API_TOKEN ]]; then
+  echo Error: GRAFANA_API_TOKEN not defined
+  exit 1
+fi
+
+DASHBOARD_JSON=./testnet-monitor.json
+if [[ ! -r $DASHBOARD_JSON ]]; then
+  echo Error: $DASHBOARD_JSON not found
+fi
+
+(
+  set -x
+  ./adjust-dashboard-for-channel.py "$DASHBOARD_JSON" "$CHANNEL"
+)
+
+rm -rf venv
+python3 -m venv venv
+# shellcheck source=/dev/null
+source venv/bin/activate
+
+echo --- Fetch/build grafcli
+(
+  set -x
+  git clone git@github.com:mvines/grafcli.git -b experimental-v5 venv/grafcli
+  cd venv/grafcli
+  python3 setup.py install
+)
+
+echo --- Take a backup of existing dashboard if possible
+(
+  set -x +e
+  grafcli export remote/metrics/$DASHBOARD $DASHBOARD_JSON.org
+  grafcli rm remote/metrics/$DASHBOARD
+  :
+)
+
+echo --- Publish $DASHBOARD_JSON to $DASHBOARD
+(
+  set -x
+  grafcli import $DASHBOARD_JSON remote/metrics
+)
+
+exit 0

multinode-demo/common.sh

@@ -49,8 +49,6 @@ elif [[ -n $USE_INSTALL ]]; then # Assume |cargo install| was run
     declare program="$1"
     printf "solana-%s" "$program"
   }
-  # CUDA was/wasn't selected at build time, can't affect CUDA state here
-  unset SOLANA_CUDA
 else
   solana_program() {
     declare program="$1"

net/gce.sh

@@ -11,13 +11,6 @@ gce)
   # shellcheck source=net/scripts/gce-provider.sh
   source "$here"/scripts/gce-provider.sh
 
-  cpuImageName="ubuntu-1804-bionic-v20181029 --image-project ubuntu-os-cloud"
-
-  # TODO: GPU image is still 16.04-based pending resolution of
-  #       https://github.com/solana-labs/solana/issues/1702
-  gpuImageName="ubuntu-16-04-cuda-9-2-new"
-  imageName=$cpuImageName
-
   cpuLeaderMachineType=n1-standard-16
   gpuLeaderMachineType="$cpuLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80"
   leaderMachineType=$cpuLeaderMachineType
@@ -28,11 +21,6 @@ ec2)
   # shellcheck source=net/scripts/ec2-provider.sh
   source "$here"/scripts/ec2-provider.sh
 
-  # Deep Learning AMI (Ubuntu 16.04-based)
-  cpuImageName="ami-0466e26ccc0e752c1"
-  gpuImageName="$cpuImageName"
-  imageName=$cpuImageName
-
   cpuLeaderMachineType=m4.4xlarge
   gpuLeaderMachineType=p2.xlarge
   leaderMachineType=$cpuLeaderMachineType
@@ -129,12 +117,10 @@ while getopts "h?p:Pn:c:z:gG:a:d:" opt; do
   g)
     enableGpu=true
     leaderMachineType=$gpuLeaderMachineType
-    imageName=$gpuImageName
     ;;
   G)
     enableGpu=true
     leaderMachineType="$OPTARG"
-    imageName=$gpuImageName
     ;;
   a)
     leaderAddress=$OPTARG
@@ -143,14 +129,53 @@ while getopts "h?p:Pn:c:z:gG:a:d:" opt; do
     bootDiskType=$OPTARG
     ;;
   *)
-    usage "Error: unhandled option: $opt"
+    usage "unhandled option: $opt"
     ;;
   esac
 done
 shift $((OPTIND - 1))
 
 [[ -z $1 ]] || usage "Unexpected argument: $1"
-sshPrivateKey="$netConfigDir/id_$prefix"
+if [[ $cloudProvider = ec2 ]]; then
+  # EC2 keys can't be retrieved from running instances like GCE keys can so save
+  # EC2 keys in the user's home directory so |./ec2.sh config| can at least be
+  # used on the same host that ran |./ec2.sh create| .
+  sshPrivateKey="$HOME/.ssh/solana-net-id_$prefix"
+else
+  sshPrivateKey="$netConfigDir/id_$prefix"
+fi
+
+case $cloudProvider in
+gce)
+  if $enableGpu; then
+    # TODO: GPU image is still 16.04-based pending resolution of
+    #       https://github.com/solana-labs/solana/issues/1702
+    imageName="ubuntu-16-04-cuda-9-2-new"
+  else
+    imageName="ubuntu-1804-bionic-v20181029 --image-project ubuntu-os-cloud"
+  fi
+  ;;
+ec2)
+  # Deep Learning AMI (Ubuntu 16.04-based)
+  case $region in # (region global variable is set by cloud_SetZone)
+  us-east-1)
+    imageName="ami-047daf3f2b162fc35"
+    ;;
+  us-west-1)
+    imageName="ami-08c8c7c4a57a6106d"
+    ;;
+  us-west-2)
+    imageName="ami-0b63040ee445728bf"
+    ;;
+  *)
+    usage "Unsupported region: $region"
+    ;;
+  esac
+  ;;
+*)
+  echo "Error: Unknown cloud provider: $cloudProvider"
+  ;;
+esac
 
 
 # cloud_ForEachInstance [cmd] [extra args to cmd]
@@ -218,9 +243,11 @@ EOF
 
     echo "Waiting for $name to finish booting..."
     (
-      set -x
+      set -x +e
       for i in $(seq 1 60); do
-        if timeout 20s ssh "${sshOptions[@]}" "$publicIp" "test -f /.instance-startup-complete"; then
+        timeout 20s ssh "${sshOptions[@]}" "$publicIp" "ls -l /.instance-startup-complete"
+        ret=$?
+        if [[ $ret -eq 0 ]]; then
           exit 0
         fi
         sleep 2
@@ -245,7 +272,7 @@ EOF
     IFS=: read -r leaderName leaderIp _ < <(echo "${instances[0]}")
 
     # Try to ping the machine first.
-    timeout 60s bash -c "set -o pipefail; until ping -c 3 $leaderIp | tr - _; do echo .; done"
+    timeout 90s bash -c "set -o pipefail; until ping -c 3 $leaderIp | tr - _; do echo .; done"
 
     if [[ ! -r $sshPrivateKey ]]; then
       echo "Fetching $sshPrivateKey from $leaderName"
@@ -392,6 +419,8 @@ $(
     install-libssl-compatability.sh \
     install-rsync.sh \
     network-config.sh \
+    remove-docker-interface.sh \
+    update-default-cuda.sh \
 
 )
 

net/net.sh

@@ -25,6 +25,7 @@ Operate a configured testnet
  start-specific options:
    -S [snapFilename]           - Deploy the specified Snap file
    -s edge|beta|stable         - Deploy the latest Snap on the specified Snap release channel
+   -T [tarFilename]            - Deploy the specified release tarball
    -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
                                  specified release channel (edge|beta|stable) or release tag
                                  (vX.Y.Z)
@@ -57,7 +58,7 @@ command=$1
 [[ -n $command ]] || usage
 shift
 
-while getopts "h?S:s:t:o:f:" opt; do
+while getopts "h?S:s:T:t:o:f:" opt; do
   case $opt in
   h | \?)
     usage
@@ -78,6 +79,11 @@ while getopts "h?S:s:t:o:f:" opt; do
       ;;
     esac
     ;;
+  T)
+    tarballFilename=$OPTARG
+    [[ -f $tarballFilename ]] || usage "Snap not readable: $tarballFilename"
+    deployMethod=tar
+    ;;
   t)
     case $OPTARG in
     edge|beta|stable|v*)
@@ -266,8 +272,9 @@ start() {
 
       set -x
       curl -o solana-release.tar.bz2 http://solana-release.s3.amazonaws.com/"$releaseChannel"/solana-release.tar.bz2
-      tar jxvf solana-release.tar.bz2
+      tarballFilename=solana-release.tar.bz2
     fi
+    tar jxvf $tarballFilename
     ;;
   local)
     build
@@ -320,15 +327,28 @@ start() {
   clientDeployTime=$SECONDS
   $metricsWriteDatapoint "testnet-deploy net-start-complete=1"
 
-  if [[ $deployMethod = "snap" ]]; then
+  declare networkVersion=unknown
-    declare networkVersion=unknown
+  case $deployMethod in
+  snap)
     IFS=\  read -r _ networkVersion _ < <(
       ssh "${sshOptions[@]}" "$leaderIp" \
         "snap info solana | grep \"^installed:\""
     )
     networkVersion=${networkVersion/0+git./}
-    $metricsWriteDatapoint "testnet-deploy version=\"$networkVersion\""
+    ;;
-  fi
+  tar)
+    networkVersion="$(
+      tail -n1 "$SOLANA_ROOT"/solana-release/version.txt || echo "tar-unknown"
+    )"
+    ;;
+  local)
+    networkVersion="$(git rev-parse HEAD || echo local-unknown)"
+    ;;
+  *)
+    usage "Internal error: invalid deployMethod: $deployMethod"
+    ;;
+  esac
+  $metricsWriteDatapoint "testnet-deploy version=\"${networkVersion:0:9}\""
 
   echo
   echo "+++ Deployment Successful"

net/remote/remote-node.sh

@@ -84,7 +84,7 @@ local|tar)
   export SOLANA_DEFAULT_METRICS_RATE=1
 
   ./fetch-perf-libs.sh
-  export LD_LIBRARY_PATH="$PWD/target/perf-libs:$LD_LIBRARY_PATH"
+  export LD_LIBRARY_PATH="$PWD/target/perf-libs:/usr/local/cuda/lib64:$LD_LIBRARY_PATH"
   echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
 
   scripts/oom-monitor.sh  > oom-monitor.log 2>&1 &
@@ -92,6 +92,10 @@ local|tar)
 
   case $nodeType in
   leader)
+    if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then
+      echo Selecting solana-fullnode-cuda
+      export SOLANA_CUDA=1
+    fi
     ./multinode-demo/setup.sh -t leader $setupArgs
     ./multinode-demo/drone.sh > drone.log 2>&1 &
     ./multinode-demo/leader.sh > leader.log 2>&1 &
@@ -99,6 +103,11 @@ local|tar)
   validator)
     net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
 
+    if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then
+      echo Selecting solana-fullnode-cuda
+      export SOLANA_CUDA=1
+    fi
+
     ./multinode-demo/setup.sh -t validator $setupArgs
     ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp:8001" >validator.log 2>&1 &
     ;;

net/scripts/install-earlyoom.sh

@@ -13,8 +13,8 @@ sysctl -w kernel.sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
 if command -v earlyoom; then
   systemctl status earlyoom
 else
-  wget  -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.1-*_amd64.deb' -e robots=off -nd
+  wget  -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.2-*_amd64.deb' -e robots=off -nd
-  apt install --quiet --yes ./earlyoom_1.1-*_amd64.deb
+  apt install --quiet --yes ./earlyoom_1.2-*_amd64.deb
 
   cat > earlyoom <<OOM
   # use the kernel OOM killer, trigger at 20% available RAM,

net/scripts/install-libssl-compatability.sh

@@ -12,7 +12,6 @@ apt-get --assume-yes install libssl-dev
 #
 # cc: https://github.com/solana-labs/solana/issues/1090
 # cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
-wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+dpkg -i libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+rm libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
-

net/scripts/remove-docker-interface.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+#
+# Some instances have docker running and docker0 network interface confuses
+# gossip and airdrops fail.  As a workaround for now simply remove the docker0
+# interface
+#
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+ip link delete docker0 || true

net/scripts/update-default-cuda.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+#
+# Updates the default cuda symlink to the supported version
+#
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+ln -sfT /usr/local/cuda-9.2 /usr/local/cuda

programs/bpf/rust/noop/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-noop"
-version = "0.10.1"
+version = "0.10.5"
 description = "Solana BPF noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -8,4 +8,4 @@ license = "Apache-2.0"
 
 [dependencies]
 rbpf = "0.1.0"
-solana-sdk = { path = "../../../../sdk", version = "0.10.1" }
+solana-sdk = { path = "../../../../sdk", version = "0.10.5" }

programs/native/bpf_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpfloader"
-version = "0.10.1"
+version = "0.10.5"
 description = "Solana BPF Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,7 +16,7 @@ log = "0.4.2"
 solana_rbpf = "0.1.3"
 serde = "1.0.27"
 serde_derive = "1.0.27"
-solana-sdk = { path = "../../../sdk", version = "0.10.1" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [lib]
 name = "solana_bpf_loader"

programs/native/lua_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-lualoader"
-version = "0.10.1"
+version = "0.10.5"
 description = "Solana Lua Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ log = "0.4.2"
 rlua = "0.15.2"
 serde = "1.0.27"
 serde_derive = "1.0.27"
-solana-sdk = { path = "../../../sdk", version = "0.10.1" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [dev-dependencies]
 bincode = "1.0.0"

programs/native/noop/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "solana-noop"
-version = "0.10.1"
+version = "0.10.5"
 description = "Solana noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 
 [dependencies]
-solana-sdk = { path = "../../../sdk", version = "0.10.1" }
+solana-sdk = { path = "../../../sdk", version = "0.10.5" }
 
 [lib]
 name = "noop"

rfcs/rfc-006-budget-contract-language.md

@@ -117,6 +117,14 @@ $ solana-wallet send-timestamp <PUBKEY> <PROCESS_ID> --date 2018-12-24T23:59:00
 <TX_SIGNATURE>
 ```
 
+### Deploy program
+```
+// Command
+$ solana-wallet deploy <PATH>
+
+// Return
+<PROGRAM_ID>
+```
 
 ## Javascript solana-web3.js Interface
 

sdk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk"
-version = "0.10.1"
+version = "0.10.5"
 description = "Solana SDK"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

src/wallet.rs

@@ -7,7 +7,6 @@ use chrono::prelude::*;
 use clap::ArgMatches;
 use cluster_info::NodeInfo;
 use drone::DroneRequest;
-use elf;
 use fullnode::Config;
 use hash::Hash;
 use loader_transaction::LoaderTransaction;
@@ -31,7 +30,6 @@ use std::{error, fmt, mem};
 use system_transaction::SystemTransaction;
 use transaction::Transaction;
 
-const PLATFORM_SECTION_C: &str = ".text.entrypoint";
 const USERDATA_CHUNK_SIZE: usize = 256;
 
 #[derive(Debug, PartialEq)]
@@ -396,16 +394,14 @@ pub fn process_command(config: &WalletConfig) -> Result<String, Box<error::Error
 
             let last_id = get_last_id(&config)?;
             let program = Keypair::new();
-            let program_userdata = elf::File::open_path(program_location)
+            let mut program_userdata = Vec::new();
+            File::open(program_location)
                 .map_err(|_| {
                     WalletError::DynamicProgramError("Could not parse program file".to_string())
-                })?.get_section(PLATFORM_SECTION_C)
+                })?.read_to_end(&mut program_userdata)
-                .ok_or_else(|| {
+                .map_err(|_| {
-                    WalletError::DynamicProgramError(
+                    WalletError::DynamicProgramError("Could not read program file".to_string())
-                        "Could not find entrypoint in program file".to_string(),
+                })?;
-                    )
-                })?.data
-                .clone();
 
             let tx = Transaction::system_create(
                 &config.id,