Compare commits

..

17 Commits

Author SHA1 Message Date
Michael Vines
14306a33e7 Stable dashboard can now actually come from the stable channel 2018-12-20 08:05:32 -08:00
Michael Vines
babc3847d7 Select correct branch for {testnet,-perf} when using a stable channel tag 2018-12-19 17:50:15 -08:00
Michael Vines
40fd1befa5 Add pipeline upload script 2018-12-15 16:06:36 -08:00
Michael Vines
7808af9a65 Regenerate secrets 2018-12-15 15:27:42 -08:00
Michael Vines
3c17732826 Use ejson to manage build secrets 2018-12-15 15:10:44 -08:00
Michael Vines
77aee571ad crdt-vote-count metric is now named cluster_info-vote-count 2018-12-09 19:28:30 -08:00
Michael Vines
a01b55c580 Add newline at end of file 2018-12-06 17:47:00 -08:00
Carl
0ecdc64302 Update versions in install-libssl-compatibility.sh 2018-12-06 16:35:31 -08:00
Michael Vines
ba06082d58 Move testnet metrics dashboard management out of the Grafana UI 2018-11-25 16:10:29 -08:00
Michael Vines
08e9c1a96e Add Cargo.lock 2018-11-17 16:06:40 -08:00
Michael Vines
9f38b86df8 Revert "Temporarily disable clippy"
This reverts commit ca12faca9c.
2018-11-17 16:06:40 -08:00
Michael Vines
ca12faca9c Temporarily disable clippy 2018-11-17 09:43:33 -08:00
Michael Vines
97a0791f3f Add timeouts 2018-11-17 09:09:49 -08:00
Michael Vines
4791c7e0a7 Bump earlyoom version 2018-11-10 15:56:37 -08:00
Michael Vines
1ba13fe180 v0.10.5 2018-11-09 11:55:30 -08:00
Michael Vines
9a30100a9c Create target/ if it doesn't exist yet 2018-11-09 11:52:19 -08:00
Michael Vines
aa741b3147 v0.10.4 2018-11-09 10:29:32 -08:00
23 changed files with 8396 additions and 27 deletions

31
.buildkite/env/README.md vendored Normal file
View File

@@ -0,0 +1,31 @@
[ejson](https://github.com/Shopify/ejson) and
[ejson2env](https://github.com/Shopify/ejson2env) are used to manage access
tokens and other secrets required for CI.
#### Setup
```bash
$ sudo gem install ejson ejson2env
```
then obtain the necessary keypair and place it in `/opt/ejson/keys/`.
#### Usage
Run the following command to decrypt the secrets into the environment:
```bash
eval $(ejson2env secrets.ejson)
```
#### Managing secrets.ejson
To decrypt `secrets.ejson` for modification, run:
```bash
$ ejson decrypt secrets.ejson -o secrets_unencrypted.ejson
```
Edit, then run the following to re-encrypt the file **BEFORE COMMITING YOUR
CHANGES**:
```bash
$ ejson encrypt secrets_unencrypted.ejson
$ mv secrets_unencrypted.ejson secrets.ejson
```

10
.buildkite/env/secrets.ejson vendored Normal file
View File

@@ -0,0 +1,10 @@
{
"_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
"environment": {
"CODECOV_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:EzVa4Gpj2Qn5OhZQlVfGFchuROgupvnW:CbWc6sNh1GCrAbrncxDjW00zUAD/Sa+ccg7CFSz8Ua6LnCYnSddTBxJWcJEbEs0MrjuZRQ==]",
"CRATES_IO_TOKEN": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:qF7QrUM8j+19mptcE1YS71CqmrCM13Ah:TZCatJeT1egCHiufE6cGFC1VsdJkKaaqV6QKWkEsMPBKvOAdaZbbVz9Kl+lGnIsF]",
"INFLUX_DATABASE": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:PetD/4c/EbkQmFEcK21g3cBBAPwFqHEw:wvYmDZRajy2WngVFs9AlwyHk]",
"INFLUX_USERNAME": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:WcnqZdmDFtJJ01Zu5LbeGgbYGfRzBdFc:a7c5zDDtCOu5L1Qd2NKkxT6kljyBcbck]",
"INFLUX_PASSWORD": "EJ[1:Kqnm+k1Z4p8nr7GqMczXnzh6azTk39tj3bAbCKPitUc=:LIZgP9Tp9yE9OlpV8iogmLOI7iW7SiU3:x0nYdT1A6sxu+O+MMLIN19d2t6rrK1qJ3+HnoWG3PDodsXjz06YJWQKU/mx6saqH+QbGtGV5mk0=]"
}
}

View File

@@ -1,4 +1,7 @@
#!/bin/bash -e
#!/usr/bin/env bash
set -e
eval "$(ejson2env .buildkite/env/secrets.ejson)"
# Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
# interprets this as the start of a log group.
@@ -24,4 +27,3 @@ export PS4="++"
set -x
rsync -a --delete --link-dest="$d" "$d"/target .
)

20
.buildkite/pipeline-upload.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/env bash
#
# This script is used to upload the full buildkite pipeline. The steps defined
# in the buildkite UI should simply be:
#
# steps:
# - command: "ci/buildkite-pipeline-upload.sh"
#
set -e
cd "$(dirname "$0")"/..
buildkite-agent pipeline upload ci/buildkite.yml
if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
# Add helpful link back to the corresponding Github Pull Request
buildkite-agent annotate --style "info" \
"Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
fi

1
.gitignore vendored
View File

@@ -1,4 +1,3 @@
Cargo.lock
/target/
**/*.rs.bk

2464
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
[package]
name = "solana"
description = "Blockchain, Rebuilt for Scale"
version = "0.10.3"
version = "0.10.5"
documentation = "https://docs.rs/solana"
homepage = "http://solana.com/"
readme = "README.md"
@@ -104,14 +104,14 @@ serde_cbor = "0.9.0"
serde_derive = "1.0.27"
serde_json = "1.0.10"
socket2 = "0.3.8"
solana-sdk = { path = "sdk", version = "0.10.3" }
solana-sdk = { path = "sdk", version = "0.10.5" }
sys-info = "0.5.6"
tokio = "0.1"
tokio-codec = "0.1"
untrusted = "0.6.2"
solana-noop = { path = "programs/native/noop", version = "0.10.3" }
solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.3" }
solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.3" }
solana-noop = { path = "programs/native/noop", version = "0.10.5" }
solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.5" }
solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.5" }
[[bench]]
name = "bank"

View File

@@ -8,7 +8,7 @@ fn main() {
// Ensure target/perf-libs/ exists. It's been observed that
// a cargo:rerun-if-changed= directive with a non-existent
// directory triggers a rebuild on every |cargo build| invocation
fs::create_dir("target/perf-libs").unwrap_or_else(|err| {
fs::create_dir_all("target/perf-libs").unwrap_or_else(|err| {
if err.kind() != std::io::ErrorKind::AlreadyExists {
panic!("Unable to create target/perf-libs: {:?}", err);
}

73
ci/publish-metrics-dashboard.sh Executable file
View File

@@ -0,0 +1,73 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/.."
if [[ -z $BUILDKITE ]]; then
echo BUILDKITE not defined
exit 1
fi
if [[ -z $CHANNEL ]]; then
CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
fi
if [[ -z $CHANNEL ]]; then
(
cat <<EOF
steps:
- block: "Select Dashboard"
fields:
- select: "Channel"
key: "channel"
options:
- label: "stable"
value: "stable"
- label: "edge"
value: "edge"
- label: "beta"
value: "beta"
- command: "ci/$(basename "$0")"
EOF
) | buildkite-agent pipeline upload
exit 0
fi
ci/channel-info.sh
eval "$(ci/channel-info.sh)"
case $CHANNEL in
edge)
CHANNEL_BRANCH=$EDGE_CHANNEL
;;
beta)
CHANNEL_BRANCH=$BETA_CHANNEL
;;
stable)
CHANNEL_BRANCH=$STABLE_CHANNEL
;;
*)
echo "Error: Invalid CHANNEL=$CHANNEL"
exit 1
;;
esac
if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
(
cat <<EOF
steps:
- trigger: "$BUILDKITE_PIPELINE_SLUG"
async: true
build:
message: "$BUILDKITE_MESSAGE"
branch: "$CHANNEL_BRANCH"
env:
CHANNEL: "$CHANNEL"
EOF
) | buildkite-agent pipeline upload
exit 0
fi
set -x
exec metrics/publish-metrics-dashboard.sh "$CHANNEL"

View File

@@ -83,10 +83,11 @@ testnet-beta|testnet-beta-perf)
testnet|testnet-perf)
if [[ -n $BETA_CHANNEL_LATEST_TAG ]]; then
CHANNEL_OR_TAG=$BETA_CHANNEL_LATEST_TAG
CHANNEL_BRANCH=$BETA_CHANNEL
else
CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
CHANNEL_BRANCH=$STABLE_CHANNEL
fi
CHANNEL_BRANCH=$BETA_CHANNEL
;;
*)
echo "Error: Invalid TESTNET=$TESTNET"

View File

@@ -32,10 +32,10 @@ zone=$3
set -x
echo "--- $cloudProvider.sh config"
net/"$cloudProvider".sh config -p "$netName" -z "$zone"
timeout 5m net/"$cloudProvider".sh config -p "$netName" -z "$zone"
net/init-metrics.sh -e
echo --- net.sh sanity
net/net.sh sanity \
timeout 5m net/net.sh sanity \
${NO_LEDGER_VERIFY:+-o noLedgerVerify} \
${NO_VALIDATOR_SANITY:+-o noValidatorSanity} \
${REJECT_EXTRA_NODES:+-o rejectExtraNodes} \

39
metrics/README.md Normal file
View File

@@ -0,0 +1,39 @@
# Metrics
## Testnet Grafana Dashboard
There are three versions of the testnet dashboard, corresponding to the three
release channels:
* https://metrics.solana.com:3000/d/testnet-edge/testnet-monitor-edge
* https://metrics.solana.com:3000/d/testnet-beta/testnet-monitor-beta
* https://metrics.solana.com:3000/d/testnet/testnet-monitor
The dashboard for each channel is defined from the
`metrics/testnet-monitor.json` source file in the git branch associated with
that channel, and deployed by automation running `ci/publish-metrics-dashboard.sh`.
A deploy can be triggered at any time via the `New Build` button of
https://buildkite.com/solana-labs/publish-metrics-dashboard.
### Modifying a Dashboard
Dashboard updates are accomplished by modifying `metrics/testnet-monitor.json`,
**manual edits made directly in Grafana will be overwritten**.
1. Open the desired dashboard in Grafana
2. Create a development copy of the dashboard by selecting `Save As..` in the
`Settings` menu for the dashboard
3. Edit dashboard as desired
4. Extract the JSON Model by selecting `JSON Model` in the `Settings` menu. Copy the JSON to the clipboard
and paste into `metrics/testnet-monitor.json`
5. Delete your development dashboard: `Settings` => `Delete`
### Deploying a Dashboard Manually
If you need to immediately deploy a dashboard using the contents of
`metrics/testnet-monitor.json` in your local workspace,
```
$ export GRAFANA_API_TOKEN="an API key from https://metrics.solana.com:3000/org/apikeys"
$ metrics/publish-metrics-dashboard.sh (edge|beta|stable)
```
Note that automation will eventually overwrite your manual deploy.

View File

@@ -0,0 +1,69 @@
#!/usr/bin/env python3
#
# Adjusts the testnet monitor dashboard for the specified release channel
#
import sys
import json
if len(sys.argv) != 3:
print('Error: Dashboard or Channel not specified')
sys.exit(1)
dashboard_json = sys.argv[1]
channel = sys.argv[2]
if channel not in ['edge', 'beta', 'stable']:
print('Error: Unknown channel:', channel)
sys.exit(2)
with open(dashboard_json, 'r') as read_file:
data = json.load(read_file)
if channel == 'stable':
# Stable dashboard only allows the user to select between the stable
# testnet databases
data['title'] = 'Testnet Monitor'
data['uid'] = 'testnet'
data['templating']['list'] = [{'allValue': None,
'current': {'text': 'testnet',
'value': 'testnet'},
'hide': 1,
'includeAll': False,
'label': 'Testnet',
'multi': False,
'name': 'testnet',
'options': [{'selected': False,
'text': 'testnet',
'value': 'testnet'},
{'selected': True,
'text': 'testnet-perf',
'value': 'testnet-perf'}],
'query': 'testnet,testnet-perf',
'type': 'custom'}]
else:
# Non-stable dashboard only allows the user to select between all testnet
# databases
data['title'] = 'Testnet Monitor ({})'.format(channel)
data['uid'] = 'testnet-' + channel
data['templating']['list'] = [{'allValue': None,
'current': {'text': 'testnet',
'value': 'testnet'},
'datasource': 'Solana Metrics (read-only)',
'hide': 1,
'includeAll': False,
'label': 'Testnet',
'multi': False,
'name': 'testnet',
'options': [],
'query': 'show databases',
'refresh': 1,
'regex': 'testnet.*',
'sort': 1,
'tagValuesQuery': '',
'tags': [],
'tagsQuery': '',
'type': 'query',
'useTags': False}]
with open(dashboard_json, 'w') as write_file:
json.dump(data, write_file, indent=2)

15
metrics/grafcli.conf Normal file
View File

@@ -0,0 +1,15 @@
[grafcli]
editor = vim
mergetool = vimdiff
verbose = on
force = on
[resources]
[hosts]
metrics = on
[metrics]
type = api
url = https://metrics.solana.com:3000/api
ssl = off

View File

@@ -0,0 +1,71 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"
CHANNEL=$1
if [[ -z $CHANNEL ]]; then
echo "usage: $0 [channel]"
exit 1
fi
case $CHANNEL in
edge)
DASHBOARD=testnet-monitor-edge
;;
beta)
DASHBOARD=testnet-monitor-beta
;;
stable)
DASHBOARD=testnet-monitor
;;
*)
echo "Error: Invalid CHANNEL=$CHANNEL"
exit 1
;;
esac
if [[ -z $GRAFANA_API_TOKEN ]]; then
echo Error: GRAFANA_API_TOKEN not defined
exit 1
fi
DASHBOARD_JSON=./testnet-monitor.json
if [[ ! -r $DASHBOARD_JSON ]]; then
echo Error: $DASHBOARD_JSON not found
fi
(
set -x
./adjust-dashboard-for-channel.py "$DASHBOARD_JSON" "$CHANNEL"
)
rm -rf venv
python3 -m venv venv
# shellcheck source=/dev/null
source venv/bin/activate
echo --- Fetch/build grafcli
(
set -x
git clone git@github.com:mvines/grafcli.git -b experimental-v5 venv/grafcli
cd venv/grafcli
python3 setup.py install
)
echo --- Take a backup of existing dashboard if possible
(
set -x +e
grafcli export remote/metrics/$DASHBOARD $DASHBOARD_JSON.org
grafcli rm remote/metrics/$DASHBOARD
:
)
echo --- Publish $DASHBOARD_JSON to $DASHBOARD
(
set -x
grafcli import $DASHBOARD_JSON remote/metrics
)
exit 0

5576
metrics/testnet-monitor.json Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -13,8 +13,8 @@ sysctl -w kernel.sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
if command -v earlyoom; then
systemctl status earlyoom
else
wget -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.1-*_amd64.deb' -e robots=off -nd
apt install --quiet --yes ./earlyoom_1.1-*_amd64.deb
wget -r -l1 -np http://ftp.us.debian.org/debian/pool/main/e/earlyoom/ -A 'earlyoom_1.2-*_amd64.deb' -e robots=off -nd
apt install --quiet --yes ./earlyoom_1.2-*_amd64.deb
cat > earlyoom <<OOM
# use the kernel OOM killer, trigger at 20% available RAM,

View File

@@ -12,7 +12,6 @@ apt-get --assume-yes install libssl-dev
#
# cc: https://github.com/solana-labs/solana/issues/1090
# cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
dpkg -i libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb
rm libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-bpf-noop"
version = "0.10.3"
version = "0.10.5"
description = "Solana BPF noop program"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -8,4 +8,4 @@ license = "Apache-2.0"
[dependencies]
rbpf = "0.1.0"
solana-sdk = { path = "../../../../sdk", version = "0.10.3" }
solana-sdk = { path = "../../../../sdk", version = "0.10.5" }

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-bpfloader"
version = "0.10.3"
version = "0.10.5"
description = "Solana BPF Loader"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -16,7 +16,7 @@ log = "0.4.2"
solana_rbpf = "0.1.3"
serde = "1.0.27"
serde_derive = "1.0.27"
solana-sdk = { path = "../../../sdk", version = "0.10.3" }
solana-sdk = { path = "../../../sdk", version = "0.10.5" }
[lib]
name = "solana_bpf_loader"

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-lualoader"
version = "0.10.3"
version = "0.10.5"
description = "Solana Lua Loader"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ log = "0.4.2"
rlua = "0.15.2"
serde = "1.0.27"
serde_derive = "1.0.27"
solana-sdk = { path = "../../../sdk", version = "0.10.3" }
solana-sdk = { path = "../../../sdk", version = "0.10.5" }
[dev-dependencies]
bincode = "1.0.0"

View File

@@ -1,13 +1,13 @@
[package]
name = "solana-noop"
version = "0.10.3"
version = "0.10.5"
description = "Solana noop program"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
[dependencies]
solana-sdk = { path = "../../../sdk", version = "0.10.3" }
solana-sdk = { path = "../../../sdk", version = "0.10.5" }
[lib]
name = "noop"

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-sdk"
version = "0.10.3"
version = "0.10.5"
description = "Solana SDK"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"