v0.15: cherrypick crates.io-related fixes from master (#4486)

* Remove runtime dependency on storage (#4480)

* Break noop_program -> runtime dependency (#4481)

* Clean up crates.io publishing (#4478)

* Clean up crates.io publishing

* Cargo.lock

Cargo.lock

@@ -2584,7 +2584,6 @@ version = "0.15.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-logger 0.15.1",
- "solana-runtime 0.15.1",
  "solana-sdk 0.15.1",
 ]
 
@@ -2623,8 +2622,6 @@ dependencies = [
  "solana-sdk 0.15.1",
  "solana-stake-api 0.15.1",
  "solana-stake-program 0.15.1",
- "solana-storage-api 0.15.1",
- "solana-storage-program 0.15.1",
  "solana-vote-api 0.15.1",
  "solana-vote-program 0.15.1",
 ]

ci/order-crates-for-publishing.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+#
+# This script figures the order in which workspace crates must be published to
+# crates.io.  Along the way it also ensures there are no circular dependencies
+# that would cause a |cargo publish| to fail.
+#
+# On success an ordered list of Cargo.toml files is written to stdout
+#
+
+import os
+import json
+import subprocess
+import sys;
+
+def load_metadata():
+    return json.loads(subprocess.Popen(
+        'cargo metadata --no-deps --format-version=1',
+        shell=True, stdout=subprocess.PIPE).communicate()[0])
+
+def get_packages():
+    metadata = load_metadata()
+
+    manifest_path = dict()
+
+    # Build dictionary of packages and their immediate solana-only dependencies
+    dependency_graph = dict()
+    for pkg in metadata['packages']:
+        manifest_path[pkg['name']] = pkg['manifest_path'];
+        dependency_graph[pkg['name']] = [x['name'] for x in pkg['dependencies'] if x['name'].startswith('solana-')];
+
+    # Check for direct circular dependencies
+    circular_dependencies = set()
+    for package, dependencies in dependency_graph.items():
+        for dependency in dependencies:
+            if dependency in dependency_graph and package in dependency_graph[dependency]:
+                circular_dependencies.add(' <--> '.join(sorted([package, dependency])))
+
+    for dependency in circular_dependencies:
+        sys.stderr.write('Error: Circular dependency: {}\n'.format(dependency))
+
+    if len(circular_dependencies) != 0:
+        sys.exit(1)
+
+    # Order dependencies
+    sorted_dependency_graph = []
+    max_iterations = pow(len(dependency_graph),2)
+    while dependency_graph:
+        if max_iterations == 0:
+            # TODO: Be more helpful and find the actual cycle for the user
+            sys.exit('Error: Circular dependency suspected between these packages: {}\n'.format(' '.join(dependency_graph.keys())))
+
+        max_iterations -= 1
+        for package, dependencies in dependency_graph.items():
+            for dependency in dependencies:
+                if dependency in dependency_graph:
+                    break
+            else:
+                del dependency_graph[package]
+                sorted_dependency_graph.append((package, manifest_path[package]))
+
+
+    return sorted_dependency_graph
+
+for package, manifest in get_packages():
+    print os.path.relpath(manifest)

ci/publish-crate.sh

@@ -3,36 +3,6 @@ set -e
 cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 
-# List of internal crates to publish
-#
-# IMPORTANT: the order of the CRATES *is* significant.  Crates must be published
-# before the crates that depend on them.  Note that this information is already
-# expressed in the various Cargo.toml files, and ideally would not be duplicated
-# here. (TODO: figure the crate ordering dynamically)
-#
-CRATES=(
-  kvstore
-  logger
-  netutil
-  sdk
-  keygen
-  metrics
-  client
-  drone
-  programs/{budget_api,config_api,stake_api,storage_api,token_api,vote_api,exchange_api}
-  programs/{vote_program,budget_program,bpf_loader,config_program,exchange_program,failure_program}
-  programs/{noop_program,stake_program,storage_program,token_program}
-  runtime
-  vote-signer
-  core
-  validator
-  genesis
-  gossip
-  ledger-tool
-  wallet
-  install
-)
-
 # Only package/publish if this is a tagged release
 [[ -n $TRIGGERED_BUILDKITE_TAG ]] || {
   echo TRIGGERED_BUILDKITE_TAG unset, skipped
@@ -49,25 +19,24 @@ expectedCrateVersion="$MAJOR.$MINOR.$PATCH$SPECIAL"
 
 cargoCommand="cargo publish --token $CRATES_IO_TOKEN"
 
-for crate in "${CRATES[@]}"; do
-  if [[ ! -r $crate/Cargo.toml ]]; then
-    echo "Error: $crate/Cargo.toml does not exist"
-    exit 1
-  fi
-  echo "-- $crate"
-  grep -q "^version = \"$expectedCrateVersion\"$" "$crate"/Cargo.toml || {
-    echo "Error: $crate/Cargo.toml version is not $expectedCrateVersion"
+Cargo_tomls=$(ci/order-crates-for-publishing.py)
+
+for Cargo_toml in "${Cargo_tomls[@]}"; do
+  echo "-- $Cargo_toml"
+  grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || {
+    echo "Error: $Cargo_toml version is not $expectedCrateVersion"
     exit 1
   }
 
   (
     set -x
+    crate=$(dirname "$Cargo_toml")
     # TODO: the rocksdb package does not build with the stock rust docker image,
     # so use the solana rust docker image until this is resolved upstream
     source ci/rust-version.sh
     ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"
     #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand"
-  )
+  ) || true # <-- Don't fail.  We want to be able to retry the job in cases when a publish fails halfway due to network/cloud issues
 done
 
 exit 0

ci/test-checks.sh

@@ -14,6 +14,7 @@ _ cargo +"$rust_stable" clippy --all -- --version
 _ cargo +"$rust_stable" clippy --all -- --deny=warnings
 _ cargo +"$rust_stable" audit
 _ ci/nits.sh
+_ ci/order-crates-for-publishing.py
 _ book/build.sh
 
 echo --- ok

core/src/local_cluster.rs

@@ -21,6 +21,7 @@ use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::transaction::Transaction;
 use solana_stake_api::stake_instruction;
 use solana_storage_api::storage_instruction;
+use solana_storage_program::genesis_block_util::GenesisBlockUtil;
 use solana_vote_api::vote_instruction;
 use solana_vote_api::vote_state::VoteState;
 use std::collections::HashMap;
@@ -124,12 +125,13 @@ impl LocalCluster {
             mut genesis_block,
             mint_keypair,
             voting_keypair,
-            storage_keypair,
         } = create_genesis_block_with_leader(
             config.cluster_lamports,
             &leader_pubkey,
             config.node_stakes[0],
         );
+        let storage_keypair = Keypair::new();
+        genesis_block.add_storage_program(&storage_keypair.pubkey());
         genesis_block.ticks_per_slot = config.ticks_per_slot;
         genesis_block.slots_per_epoch = config.slots_per_epoch;
         genesis_block.stakers_slot_offset = config.stakers_slot_offset;

genesis/src/main.rs

@@ -6,8 +6,6 @@ extern crate solana_stake_program;
 #[macro_use]
 extern crate solana_budget_program;
 #[macro_use]
-extern crate solana_storage_program;
-#[macro_use]
 extern crate solana_token_program;
 #[macro_use]
 extern crate solana_config_program;
@@ -25,7 +23,7 @@ use solana_sdk::signature::{read_keypair, KeypairUtil};
 use solana_sdk::system_program;
 use solana_sdk::timing;
 use solana_stake_api::stake_state;
-use solana_storage_api::storage_contract;
+use solana_storage_program::genesis_block_util::GenesisBlockUtil;
 use solana_vote_api::vote_state;
 use std::error;
 use std::time::{Duration, Instant};
@@ -213,22 +211,18 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     bootstrap_leader_stake_lamports,
                 ),
             ),
-            // storage account
-            (
-                bootstrap_storage_keypair.pubkey(),
-                storage_contract::create_validator_storage_account(1),
-            ),
         ],
         &[
             solana_vote_program!(),
             solana_stake_program!(),
             solana_budget_program!(),
-            solana_storage_program!(),
             solana_token_program!(),
             solana_config_program!(),
             solana_exchange_program!(),
         ],
     );
+    genesis_block.add_storage_program(&bootstrap_storage_keypair.pubkey());
+
     genesis_block.fee_calculator.lamports_per_signature =
         value_t_or_exit!(matches, "lamports_per_signature", u64);
     genesis_block.ticks_per_slot = value_t_or_exit!(matches, "ticks_per_slot", u64);

programs/noop_program/Cargo.toml

@@ -13,9 +13,6 @@ solana-sdk = { path = "../../sdk", version = "0.15.1"   }
 solana-logger = { path = "../../logger", version = "0.15.1"   }
 log = "0.4.2"
 
-[dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.15.1"   }
-
 [lib]
 name = "solana_noop_program"
 crate-type = ["cdylib"]

programs/storage_program/src/genesis_block_util.rs

@@ -0,0 +1,19 @@
+use crate::solana_storage_program;
+use solana_sdk::genesis_block::GenesisBlock;
+use solana_sdk::pubkey::Pubkey;
+use solana_storage_api::storage_contract;
+
+pub trait GenesisBlockUtil {
+    fn add_storage_program(&mut self, validator_storage_pubkey: &Pubkey);
+}
+
+impl GenesisBlockUtil for GenesisBlock {
+    fn add_storage_program(&mut self, validator_storage_pubkey: &Pubkey) {
+        self.accounts.push((
+            *validator_storage_pubkey,
+            storage_contract::create_validator_storage_account(1),
+        ));
+        self.native_instruction_processors
+            .push(solana_storage_program!());
+    }
+}

programs/storage_program/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod genesis_block_util;
+
 #[macro_export]
 macro_rules! solana_storage_program {
     () => {

runtime/Cargo.toml

@@ -27,11 +27,9 @@ solana-logger = { path = "../logger", version = "0.15.1"   }
 solana-metrics = { path = "../metrics", version = "0.15.1"   }
 solana-sdk = { path = "../sdk", version = "0.15.1"   }
 solana-stake-api = { path = "../programs/stake_api", version = "0.15.1"   }
-solana-storage-api = { path = "../programs/storage_api", version = "0.15.1"   }
 solana-vote-api = { path = "../programs/vote_api", version = "0.15.1"   }
 solana-vote-program = { path = "../programs/vote_program", version = "0.15.1"   }
 solana-stake-program = { path = "../programs/stake_program", version = "0.15.1"   }
-solana-storage-program = { path = "../programs/storage_program", version = "0.15.1"   }
 solana-noop-program = { path = "../programs/noop_program", version = "0.15.1"   }
 
 [lib]

runtime/src/genesis_utils.rs

@@ -4,7 +4,6 @@ use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, KeypairUtil};
 use solana_sdk::system_program;
 use solana_stake_api::stake_state;
-use solana_storage_api::storage_contract;
 use solana_vote_api::vote_state;
 
 // The default stake placed with the bootstrap leader
@@ -14,7 +13,6 @@ pub struct GenesisBlockInfo {
     pub genesis_block: GenesisBlock,
     pub mint_keypair: Keypair,
     pub voting_keypair: Keypair,
-    pub storage_keypair: Keypair,
 }
 
 pub fn create_genesis_block_with_leader(
@@ -25,7 +23,6 @@ pub fn create_genesis_block_with_leader(
     let mint_keypair = Keypair::new();
     let voting_keypair = Keypair::new();
     let staking_keypair = Keypair::new();
-    let storage_keypair = Keypair::new();
 
     // TODO: de-duplicate the stake once passive staking
     //  is fully implemented
@@ -61,23 +58,13 @@ pub fn create_genesis_block_with_leader(
                     bootstrap_leader_stake_lamports,
                 ),
             ),
-            // storage account
-            (
-                storage_keypair.pubkey(),
-                storage_contract::create_validator_storage_account(1),
-            ),
-        ],
-        &[
-            solana_vote_program!(),
-            solana_stake_program!(),
-            solana_storage_program!(), // TODO: storage program is only needed by core/, move this line into core/src/genesis_utils.rs
         ],
+        &[solana_vote_program!(), solana_stake_program!()],
     );
 
     GenesisBlockInfo {
         genesis_block,
         mint_keypair,
         voting_keypair,
-        storage_keypair,
     }
 }

runtime/src/lib.rs

@@ -25,8 +25,5 @@ extern crate solana_vote_program;
 #[macro_use]
 extern crate solana_stake_program;
 
-#[macro_use]
-extern crate solana_storage_program;
-
 #[macro_use]
 extern crate serde_derive;