From dd1ed323e71176e8251c8dea0c15423d045cb8fd Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Thu, 30 May 2019 13:40:42 -0700 Subject: [PATCH] v0.15: cherrypick crates.io-related fixes from master (#4486) * Remove runtime dependency on storage (#4480) * Break noop_program -> runtime dependency (#4481) * Clean up crates.io publishing (#4478) * Clean up crates.io publishing * Cargo.lock --- Cargo.lock | 3 - ci/order-crates-for-publishing.py | 65 +++++++++++++++++++ ci/publish-crate.sh | 47 +++----------- ci/test-checks.sh | 1 + core/src/local_cluster.rs | 4 +- genesis/src/main.rs | 12 +--- programs/noop_program/Cargo.toml | 3 - .../storage_program/src/genesis_block_util.rs | 19 ++++++ programs/storage_program/src/lib.rs | 2 + runtime/Cargo.toml | 2 - runtime/src/genesis_utils.rs | 15 +---- runtime/src/lib.rs | 3 - .../noop_program => runtime}/tests/noop.rs | 0 13 files changed, 102 insertions(+), 74 deletions(-) create mode 100755 ci/order-crates-for-publishing.py create mode 100644 programs/storage_program/src/genesis_block_util.rs rename {programs/noop_program => runtime}/tests/noop.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b6b396c9e8..b5e289d2a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2584,7 +2584,6 @@ version = "0.15.1" dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "solana-logger 0.15.1", - "solana-runtime 0.15.1", "solana-sdk 0.15.1", ] @@ -2623,8 +2622,6 @@ dependencies = [ "solana-sdk 0.15.1", "solana-stake-api 0.15.1", "solana-stake-program 0.15.1", - "solana-storage-api 0.15.1", - "solana-storage-program 0.15.1", "solana-vote-api 0.15.1", "solana-vote-program 0.15.1", ] diff --git a/ci/order-crates-for-publishing.py b/ci/order-crates-for-publishing.py new file mode 100755 index 0000000000..5a6bc2f7a5 --- /dev/null +++ b/ci/order-crates-for-publishing.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# This script figures the order in which workspace crates must be published to +# crates.io. Along the way it also ensures there are no circular dependencies +# that would cause a |cargo publish| to fail. +# +# On success an ordered list of Cargo.toml files is written to stdout +# + +import os +import json +import subprocess +import sys; + +def load_metadata(): + return json.loads(subprocess.Popen( + 'cargo metadata --no-deps --format-version=1', + shell=True, stdout=subprocess.PIPE).communicate()[0]) + +def get_packages(): + metadata = load_metadata() + + manifest_path = dict() + + # Build dictionary of packages and their immediate solana-only dependencies + dependency_graph = dict() + for pkg in metadata['packages']: + manifest_path[pkg['name']] = pkg['manifest_path']; + dependency_graph[pkg['name']] = [x['name'] for x in pkg['dependencies'] if x['name'].startswith('solana-')]; + + # Check for direct circular dependencies + circular_dependencies = set() + for package, dependencies in dependency_graph.items(): + for dependency in dependencies: + if dependency in dependency_graph and package in dependency_graph[dependency]: + circular_dependencies.add(' <--> '.join(sorted([package, dependency]))) + + for dependency in circular_dependencies: + sys.stderr.write('Error: Circular dependency: {}\n'.format(dependency)) + + if len(circular_dependencies) != 0: + sys.exit(1) + + # Order dependencies + sorted_dependency_graph = [] + max_iterations = pow(len(dependency_graph),2) + while dependency_graph: + if max_iterations == 0: + # TODO: Be more helpful and find the actual cycle for the user + sys.exit('Error: Circular dependency suspected between these packages: {}\n'.format(' '.join(dependency_graph.keys()))) + + max_iterations -= 1 + for package, dependencies in dependency_graph.items(): + for dependency in dependencies: + if dependency in dependency_graph: + break + else: + del dependency_graph[package] + sorted_dependency_graph.append((package, manifest_path[package])) + + + return sorted_dependency_graph + +for package, manifest in get_packages(): + print os.path.relpath(manifest) diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index d9c5ea28ce..6426f8df67 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -3,36 +3,6 @@ set -e cd "$(dirname "$0")/.." source ci/semver_bash/semver.sh -# List of internal crates to publish -# -# IMPORTANT: the order of the CRATES *is* significant. Crates must be published -# before the crates that depend on them. Note that this information is already -# expressed in the various Cargo.toml files, and ideally would not be duplicated -# here. (TODO: figure the crate ordering dynamically) -# -CRATES=( - kvstore - logger - netutil - sdk - keygen - metrics - client - drone - programs/{budget_api,config_api,stake_api,storage_api,token_api,vote_api,exchange_api} - programs/{vote_program,budget_program,bpf_loader,config_program,exchange_program,failure_program} - programs/{noop_program,stake_program,storage_program,token_program} - runtime - vote-signer - core - validator - genesis - gossip - ledger-tool - wallet - install -) - # Only package/publish if this is a tagged release [[ -n $TRIGGERED_BUILDKITE_TAG ]] || { echo TRIGGERED_BUILDKITE_TAG unset, skipped @@ -49,25 +19,24 @@ expectedCrateVersion="$MAJOR.$MINOR.$PATCH$SPECIAL" cargoCommand="cargo publish --token $CRATES_IO_TOKEN" -for crate in "${CRATES[@]}"; do - if [[ ! -r $crate/Cargo.toml ]]; then - echo "Error: $crate/Cargo.toml does not exist" - exit 1 - fi - echo "-- $crate" - grep -q "^version = \"$expectedCrateVersion\"$" "$crate"/Cargo.toml || { - echo "Error: $crate/Cargo.toml version is not $expectedCrateVersion" +Cargo_tomls=$(ci/order-crates-for-publishing.py) + +for Cargo_toml in "${Cargo_tomls[@]}"; do + echo "-- $Cargo_toml" + grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || { + echo "Error: $Cargo_toml version is not $expectedCrateVersion" exit 1 } ( set -x + crate=$(dirname "$Cargo_toml") # TODO: the rocksdb package does not build with the stock rust docker image, # so use the solana rust docker image until this is resolved upstream source ci/rust-version.sh ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand" #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand" - ) + ) || true # <-- Don't fail. We want to be able to retry the job in cases when a publish fails halfway due to network/cloud issues done exit 0 diff --git a/ci/test-checks.sh b/ci/test-checks.sh index ddc9e05457..0e63e9d83c 100755 --- a/ci/test-checks.sh +++ b/ci/test-checks.sh @@ -14,6 +14,7 @@ _ cargo +"$rust_stable" clippy --all -- --version _ cargo +"$rust_stable" clippy --all -- --deny=warnings _ cargo +"$rust_stable" audit _ ci/nits.sh +_ ci/order-crates-for-publishing.py _ book/build.sh echo --- ok diff --git a/core/src/local_cluster.rs b/core/src/local_cluster.rs index ebfbd6e097..faef41d152 100644 --- a/core/src/local_cluster.rs +++ b/core/src/local_cluster.rs @@ -21,6 +21,7 @@ use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT; use solana_sdk::transaction::Transaction; use solana_stake_api::stake_instruction; use solana_storage_api::storage_instruction; +use solana_storage_program::genesis_block_util::GenesisBlockUtil; use solana_vote_api::vote_instruction; use solana_vote_api::vote_state::VoteState; use std::collections::HashMap; @@ -124,12 +125,13 @@ impl LocalCluster { mut genesis_block, mint_keypair, voting_keypair, - storage_keypair, } = create_genesis_block_with_leader( config.cluster_lamports, &leader_pubkey, config.node_stakes[0], ); + let storage_keypair = Keypair::new(); + genesis_block.add_storage_program(&storage_keypair.pubkey()); genesis_block.ticks_per_slot = config.ticks_per_slot; genesis_block.slots_per_epoch = config.slots_per_epoch; genesis_block.stakers_slot_offset = config.stakers_slot_offset; diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 8f7501fe63..f51bf4120c 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -6,8 +6,6 @@ extern crate solana_stake_program; #[macro_use] extern crate solana_budget_program; #[macro_use] -extern crate solana_storage_program; -#[macro_use] extern crate solana_token_program; #[macro_use] extern crate solana_config_program; @@ -25,7 +23,7 @@ use solana_sdk::signature::{read_keypair, KeypairUtil}; use solana_sdk::system_program; use solana_sdk::timing; use solana_stake_api::stake_state; -use solana_storage_api::storage_contract; +use solana_storage_program::genesis_block_util::GenesisBlockUtil; use solana_vote_api::vote_state; use std::error; use std::time::{Duration, Instant}; @@ -213,22 +211,18 @@ fn main() -> Result<(), Box> { bootstrap_leader_stake_lamports, ), ), - // storage account - ( - bootstrap_storage_keypair.pubkey(), - storage_contract::create_validator_storage_account(1), - ), ], &[ solana_vote_program!(), solana_stake_program!(), solana_budget_program!(), - solana_storage_program!(), solana_token_program!(), solana_config_program!(), solana_exchange_program!(), ], ); + genesis_block.add_storage_program(&bootstrap_storage_keypair.pubkey()); + genesis_block.fee_calculator.lamports_per_signature = value_t_or_exit!(matches, "lamports_per_signature", u64); genesis_block.ticks_per_slot = value_t_or_exit!(matches, "ticks_per_slot", u64); diff --git a/programs/noop_program/Cargo.toml b/programs/noop_program/Cargo.toml index ea9b2a9f8f..1cbe32f45f 100644 --- a/programs/noop_program/Cargo.toml +++ b/programs/noop_program/Cargo.toml @@ -13,9 +13,6 @@ solana-sdk = { path = "../../sdk", version = "0.15.1" } solana-logger = { path = "../../logger", version = "0.15.1" } log = "0.4.2" -[dev-dependencies] -solana-runtime = { path = "../../runtime", version = "0.15.1" } - [lib] name = "solana_noop_program" crate-type = ["cdylib"] diff --git a/programs/storage_program/src/genesis_block_util.rs b/programs/storage_program/src/genesis_block_util.rs new file mode 100644 index 0000000000..c5c35b52ec --- /dev/null +++ b/programs/storage_program/src/genesis_block_util.rs @@ -0,0 +1,19 @@ +use crate::solana_storage_program; +use solana_sdk::genesis_block::GenesisBlock; +use solana_sdk::pubkey::Pubkey; +use solana_storage_api::storage_contract; + +pub trait GenesisBlockUtil { + fn add_storage_program(&mut self, validator_storage_pubkey: &Pubkey); +} + +impl GenesisBlockUtil for GenesisBlock { + fn add_storage_program(&mut self, validator_storage_pubkey: &Pubkey) { + self.accounts.push(( + *validator_storage_pubkey, + storage_contract::create_validator_storage_account(1), + )); + self.native_instruction_processors + .push(solana_storage_program!()); + } +} diff --git a/programs/storage_program/src/lib.rs b/programs/storage_program/src/lib.rs index 7868bfd582..15b965d4fb 100644 --- a/programs/storage_program/src/lib.rs +++ b/programs/storage_program/src/lib.rs @@ -1,3 +1,5 @@ +pub mod genesis_block_util; + #[macro_export] macro_rules! solana_storage_program { () => { diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index af310ee4e4..ed830cd363 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -27,11 +27,9 @@ solana-logger = { path = "../logger", version = "0.15.1" } solana-metrics = { path = "../metrics", version = "0.15.1" } solana-sdk = { path = "../sdk", version = "0.15.1" } solana-stake-api = { path = "../programs/stake_api", version = "0.15.1" } -solana-storage-api = { path = "../programs/storage_api", version = "0.15.1" } solana-vote-api = { path = "../programs/vote_api", version = "0.15.1" } solana-vote-program = { path = "../programs/vote_program", version = "0.15.1" } solana-stake-program = { path = "../programs/stake_program", version = "0.15.1" } -solana-storage-program = { path = "../programs/storage_program", version = "0.15.1" } solana-noop-program = { path = "../programs/noop_program", version = "0.15.1" } [lib] diff --git a/runtime/src/genesis_utils.rs b/runtime/src/genesis_utils.rs index 826c5b762b..21cbb8ee31 100644 --- a/runtime/src/genesis_utils.rs +++ b/runtime/src/genesis_utils.rs @@ -4,7 +4,6 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_program; use solana_stake_api::stake_state; -use solana_storage_api::storage_contract; use solana_vote_api::vote_state; // The default stake placed with the bootstrap leader @@ -14,7 +13,6 @@ pub struct GenesisBlockInfo { pub genesis_block: GenesisBlock, pub mint_keypair: Keypair, pub voting_keypair: Keypair, - pub storage_keypair: Keypair, } pub fn create_genesis_block_with_leader( @@ -25,7 +23,6 @@ pub fn create_genesis_block_with_leader( let mint_keypair = Keypair::new(); let voting_keypair = Keypair::new(); let staking_keypair = Keypair::new(); - let storage_keypair = Keypair::new(); // TODO: de-duplicate the stake once passive staking // is fully implemented @@ -61,23 +58,13 @@ pub fn create_genesis_block_with_leader( bootstrap_leader_stake_lamports, ), ), - // storage account - ( - storage_keypair.pubkey(), - storage_contract::create_validator_storage_account(1), - ), - ], - &[ - solana_vote_program!(), - solana_stake_program!(), - solana_storage_program!(), // TODO: storage program is only needed by core/, move this line into core/src/genesis_utils.rs ], + &[solana_vote_program!(), solana_stake_program!()], ); GenesisBlockInfo { genesis_block, mint_keypair, voting_keypair, - storage_keypair, } } diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 9760effa7f..0e44d53aef 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -25,8 +25,5 @@ extern crate solana_vote_program; #[macro_use] extern crate solana_stake_program; -#[macro_use] -extern crate solana_storage_program; - #[macro_use] extern crate serde_derive; diff --git a/programs/noop_program/tests/noop.rs b/runtime/tests/noop.rs similarity index 100% rename from programs/noop_program/tests/noop.rs rename to runtime/tests/noop.rs