Compare commits

..

272 Commits

Author SHA1 Message Date
mergify[bot]
02a83e7c6e Allow lower shred count (#9410) (#9451)
automerge
2020-04-11 14:49:32 -07:00
sakridge
83263e1737 Calculate account refs fix (#9448) 2020-04-11 12:56:20 -07:00
mergify[bot]
1f7ac22b60 Don't subject authorizing a new stake authority to lockup (#9434) (#9441)
automerge
2020-04-10 17:25:15 -07:00
Michael Vines
747debae56 Cache downloads to speed up CI
(cherry picked from commit b4e00275b2)
2020-04-10 12:25:49 -07:00
mergify[bot]
00b4186469 Improve coverage.sh usability when used locally (#9054) (#9424)
automerge
2020-04-10 05:59:35 -07:00
mergify[bot]
b087dabf4f Rpc: Add getConfirmedSignaturesForAddress (#9407) (#9417)
automerge
2020-04-09 21:20:28 -07:00
mergify[bot]
e00eb0a069 Remove Trust Wallet Beta install instructions (#9396) (#9397)
automerge
2020-04-09 08:52:04 -07:00
mergify[bot]
d4e49ffd06 Rpc: Add getConfirmedTransaction (#9381) (#9392)
automerge
2020-04-09 01:00:34 -07:00
Michael Vines
0f34a190ea Bump version to 1.0.16 2020-04-09 00:05:16 -07:00
mergify[bot]
df2fb8f5b3 Add --no-wait arg to transfer (#9388) (#9390)
automerge
2020-04-08 23:49:40 -07:00
mergify[bot]
80d2a6046b Moar vm.max_map_count (#9389)
automerge
2020-04-08 23:18:30 -07:00
mergify[bot]
24273b826f Add blockstore address-to-signature index (#9367) (#9378)
automerge
2020-04-08 13:55:53 -07:00
mergify[bot]
68d2616e35 stake-monitor: Add 1 SOL grace, to allow for a complaint system account to fund a reasonable number of transactions. (bp #9359) (#9363)
automerge
2020-04-08 11:56:01 -07:00
mergify[bot]
f506d39339 Improve ledger-tool/accounts for easier debuging (#9370) (#9371)
automerge
2020-04-08 11:31:41 -07:00
Michael Vines
bc58c9ec7e Cache solana-perf.tgz to speed up CI (#9360)
automerge

(cherry picked from commit dc91698b3a)
2020-04-07 13:32:13 -07:00
Michael Vines
b57a52cd85 Bump version to 1.0.15 2020-04-07 09:36:47 -07:00
Michael Vines
8631be42ac Add support for monitoring system account balances (#9345)
automerge

(cherry picked from commit 03978ac5a5)
2020-04-06 22:57:47 -07:00
Tyera Eulberg
09367369ef Reinstate commitment param to support old clients (#9324) (#9329)
automerge
2020-04-06 12:21:19 -07:00
mergify[bot]
23b8c95cc4 Update getSignatureStatuses to return historical statuses (#9314) (#9321)
automerge
2020-04-06 04:24:19 -07:00
Tyera Eulberg
e61392b057 Rework TransactionStatus index in blockstore (#9281) (#9304) (#9312)
automerge

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2020-04-05 12:52:15 -06:00
mergify[bot]
7deba20395 Deprecate confirmTransaction, getSignatureStatus, and getSignatureConfirmation (bp #9298) (#9308)
automerge
2020-04-05 02:41:10 -07:00
Michael Vines
274c097f84 RPC: add err field to TransactionStatus, alongside the now deprecated status field (#9296) (#9307)
automerge
2020-04-05 00:21:15 -07:00
Michael Vines
1c7cea1af4 Add stake-monitor 2020-04-03 20:37:28 -07:00
Michael Vines
4406496d2f Add log before opening database
(cherry picked from commit b557b3170e)
2020-04-03 15:10:57 -07:00
sakridge
a15fa4840c Fix repair dos (#9056) 2020-04-03 13:20:39 -07:00
Michael Vines
c9030660d6 Bump version to 1.0.14 2020-04-02 22:42:28 -07:00
Dan Albert
fdeda769d0 Backport wallet doc changes to v1.0 (#9268)
* Add ledger live screenshots and reduce duplicate instructions (#9258)

automerge

* Add instructions for Trust Wallet Beta for Android (#9261)

automerge
2020-04-02 15:37:26 -06:00
mergify[bot]
53edd26578 Add instructions for Trust Wallet Beta for Android (#9261) (#9264)
automerge
2020-04-02 12:25:32 -07:00
Dan Albert
2433cdd6d6 Set checks timeout to 20 minutes 2020-04-02 13:11:03 -06:00
Dan Albert
77b34c278e Do not trigger tests if only docs were modified (#9240) (#9257) 2020-04-02 10:44:51 -06:00
mergify[bot]
90e993fd9a Add epoch subcommand (#9249) (#9254)
automerge
2020-04-01 22:29:18 -07:00
Tyera Eulberg
251054d8c9 Backport confirmations fix. (#9252)
automerge
2020-04-01 19:42:04 -07:00
Tyera Eulberg
d4bb7cec69 Undo breaking rpc removal of getSignatureConfirmation (#9247) 2020-04-01 18:04:11 -06:00
mergify[bot]
a3f6b04345 Add fee-payer option to docs (bp #9230) (#9236)
automerge
2020-04-01 15:11:47 -07:00
Justin Starry
ba4a5053dd Undo getSignatureStatus breaking change, add getSignatureStatuses (#9231)
automerge
2020-04-01 12:21:56 -07:00
mergify[bot]
6b47a259c3 Add a support page for wallet docs (#9229) (#9234)
automerge
2020-04-01 11:22:50 -07:00
mergify[bot]
c9ec13cf1f Tune udp buffers and vmmap immediately (#9194) (#9216)
automerge
2020-04-01 00:52:50 -07:00
mergify[bot]
906a6ab837 Fix error with account hash list getting too big for gossip (#9197) (#9214)
automerge
2020-03-31 23:27:18 -07:00
mergify[bot]
d0e478a9f8 Fix panic (#9195) (#9208)
automerge
2020-03-31 22:03:46 -07:00
mergify[bot]
b560b64d33 Remove unecessary security exception and add a new one (bp #9200) (#9205)
* Remove unecessary exception and add a new one (#9200)

(cherry picked from commit 62e12e3af5)

# Conflicts:
#	ci/test-checks.sh

* Update test-checks.sh

Co-authored-by: sakridge <sakridge@gmail.com>
Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-31 19:50:04 -07:00
mergify[bot]
057af41716 Fix links (#9184) (#9187)
automerge
2020-03-31 09:57:23 -07:00
Michael Vines
a44b8abd48 Bump version to v1.0.13 2020-03-30 23:05:41 -07:00
mergify[bot]
8778ecaed5 Ledger cleanup fixes (#9131) (#9175)
automerge
2020-03-30 20:42:17 -07:00
Michael Vines
a02542ada3 Bump version to v1.0.12 2020-03-30 08:45:50 -07:00
Michael Vines
ea17c6883f Remove chatty 'setting snapshot root:' info log 2020-03-30 08:44:24 -07:00
Michael Vines
706306645b solana account now displays the account's rent epoch 2020-03-30 08:44:24 -07:00
sakridge
da9e930788 Calculate ref counts earlier to prevent bad clean (#9153) 2020-03-29 14:42:57 -07:00
mergify[bot]
8b8e066bbe Add RPC subscription api for rooted slots (#9118) (#9126)
automerge
2020-03-27 13:09:32 -07:00
mergify[bot]
3473350b62 fix links (#9125) (#9128)
automerge
2020-03-27 10:28:23 -07:00
mergify[bot]
59d7eb5216 Fix links in docs (#9119) (#9127)
automerge
2020-03-27 09:50:28 -07:00
mergify[bot]
55ba934137 Document transaction field in getConfirmedBlock responses (#9121) (#9124)
automerge
2020-03-27 09:45:30 -07:00
mergify[bot]
4c3dcb7f7e Exclude all executable accounts from rent collection (#9116) (#9120)
automerge
2020-03-27 09:27:07 -07:00
mergify[bot]
3a879db8af Fix broken gitbook links (#9107) (#9108)
automerge
2020-03-26 20:33:20 -07:00
mergify[bot]
d2107270ea Consolidate signature-status rpcs (bp #9069) (#9105)
automerge
2020-03-26 20:30:46 -07:00
mergify[bot]
007afe22d0 Add docs for app wallets (#9098) (#9103)
automerge
2020-03-26 18:26:23 -07:00
mergify[bot]
93a1d10e15 Revert setting the default toolchain (#9093) (#9097)
automerge
2020-03-26 15:59:26 -07:00
mergify[bot]
d57a7c8f21 Restructure wallet docs to prep for app wallet content (#9088) (#9095)
automerge
2020-03-26 14:39:32 -07:00
mergify[bot]
6db39829c8 Install xargo using CI dictated cargo version if available (#9068) (#9092)
automerge
2020-03-26 14:02:09 -07:00
mergify[bot]
ec76826493 Unflake rpc subscriptions test by reducing sub count (#9078) (#9082)
automerge
2020-03-25 22:56:52 -07:00
carllin
d4ddb6265b Convert Banks (#9033)
* Store and compute needed state in EpochStakes struct
Co-authored-by: Carl <carl@solana.com>
2020-03-25 20:43:48 -07:00
Michael Vines
7a8233d7ca Bump version to 1.0.11 2020-03-25 19:17:06 -07:00
mergify[bot]
95bc051129 Cargo update bumpalo (#9067) (#9077)
automerge
2020-03-25 18:51:38 -07:00
mergify[bot]
02bcf4f8e2 ledger-tool can now decode stake instructions (bp #9045) (#9076)
automerge
2020-03-25 18:02:31 -07:00
sakridge
4b0d4e9834 Remove accounts unwrap (#9063)
automerge
2020-03-25 10:59:52 -07:00
mergify[bot]
bf4cdc091a Fix xargo to version 0.3.19 to avoid unstable feature (#9065) (#9066)
automerge

(cherry picked from commit c558db2a48)

Co-authored-by: Justin Starry <justin@solana.com>
2020-03-25 08:48:22 -07:00
mergify[bot]
5f2cf2b44d Increase vmap count in sys-tuner (#8940) (#9064)
(cherry picked from commit a70008cc5c)

Co-authored-by: sakridge <sakridge@gmail.com>
2020-03-25 08:06:36 -07:00
mergify[bot]
03aae5eb5f Ignore RUSTSEC-2020-0006 for the moment (#9057) (#9059)
automerge
2020-03-24 21:06:15 -07:00
mergify[bot]
5f1ce81fbc ledger tool now outputs transaction status information if available (#9024) (#9026)
automerge
2020-03-24 12:23:25 -07:00
mergify[bot]
fc582aa57c Refactor how pubsub subscriptions are added (#9042) (#9049)
automerge
2020-03-24 11:05:33 -07:00
mergify[bot]
13676e9614 Fix timeout for subscriptions test (#9043) (#9044)
automerge
2020-03-24 02:57:25 -07:00
Michael Vines
7636a0521f solana-install-init: --pubkey is no longer required on platforms without a default update manifest 2020-03-23 22:40:53 -07:00
Michael Vines
ad06354a18 Remove , 2020-03-23 22:11:56 -07:00
Michael Vines
953cb93e44 Bump version to 1.0.10 2020-03-23 21:58:45 -07:00
Dan Albert
38b2957a8e Remove unused default update manifest pubkeys (#9041)
automerge
2020-03-23 20:57:43 -07:00
Tyera Eulberg
9eb39df93f Backport: add slot to signature notification & respect confirmations param (#9036)
automerge
2020-03-23 18:32:05 -07:00
mergify[bot]
f34ce94347 Remove Ledger-specific analysis of hardware wallets (#9028) (#9030)
automerge
2020-03-23 14:48:58 -07:00
mergify[bot]
5c6411fc06 Fix link in gitbook (#9027) (#9029)
automerge
2020-03-23 14:48:41 -07:00
Michael Vines
3ab428693a Manual v1.0 backports (#9025)
automerge
2020-03-23 13:55:03 -07:00
Michael Vines
7ffaf2ad29 Manual v1.0 backports (#9015)
automerge
2020-03-22 22:44:55 -07:00
sakridge
a5e4a1f2d8 Drop storage lock (#8997) 2020-03-21 18:55:59 -07:00
mergify[bot]
6ae99848f4 Revert "Move Install Solana doc into the Command-line Guide (#8982)" (#8992) (#8993)
This reverts commit 5fa36bbab3.

(cherry picked from commit 1aab959d4e)

Co-authored-by: Greg Fitzgerald <greg@solana.com>
2020-03-21 14:52:53 -06:00
mergify[bot]
5342b3a53f Shred fetch comment and debug message tweak (#8980) (#8990)
automerge

(cherry picked from commit 909321928c)

Co-authored-by: sakridge <sakridge@gmail.com>
2020-03-21 09:25:08 -07:00
sakridge
50a3c98d21 Shred filter (#8975) (#8987)
Thread bank_forks into shred fetch and filter by slot.
2020-03-20 11:33:19 -07:00
Michael Vines
ca2b7a2c5b Cargo.lock 2020-03-20 11:05:45 -07:00
mergify[bot]
e900623817 Update value names in docs (#8983) (#8985)
automerge
2020-03-20 09:36:55 -07:00
mergify[bot]
a46e953ebb Move Install Solana doc into the Command-line Guide (#8982) (#8984)
automerge
2020-03-20 09:24:56 -07:00
mergify[bot]
3bbc51a6ae Improve CLI usage messages (#8972) (#8977)
automerge
2020-03-19 21:52:37 -07:00
Michael Vines
7ae2464cf3 Fix windows build by removing sys-info (#8860) (#8973)
automerge
2020-03-19 18:14:01 -07:00
Michael Vines
58a36ce484 Bump version to 1.0.9 2020-03-19 17:10:35 -07:00
Dan Albert
61bd9e6a28 Fix windows binary build on v1.0 (#8968)
Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-19 15:58:34 -07:00
Tyera Eulberg
495ab631d6 Some Cli polish (#8966) (#8970)
automerge
2020-03-19 14:11:52 -07:00
Tyera Eulberg
93906847f9 Cli: polish transaction progress bar (#8963) (#8967)
automerge
2020-03-19 12:54:50 -07:00
Dan Albert
70f4c4a974 Build windows binaries on v1.0 (#8965) 2020-03-19 11:06:17 -07:00
mergify[bot]
ad6078da2d CLI: Fix create-nonce-account with seed (#8929) (#8961)
automerge
2020-03-19 10:32:35 -07:00
Michael Vines
2a617f2d07 v1.0: Backport of #8939 (#8957)
automerge
2020-03-19 02:08:36 -07:00
mergify[bot]
6af3c6ecbc CLI: Add multi-session signing support (#8927) (#8953)
* SDK: Add `NullSigner` implementation

* SDK: Split `Transaction::verify()` to gain access to results

* CLI: Minor refactor of --sign_only result parsing

* CLI: Enable paritial signing

Signers specified by pubkey, but without a matching --signer arg
supplied fall back to a `NullSigner` when --sign-only is in effect.
This allows their pubkey to be used for TX construction as usual,
but leaves their `sign_message()` a NOP. As such, with --sign-only
in effect, signing and verification must be done separately, with
the latter's per-signature results considered

* CLI: Surface/report missing/bad signers to user

* CLI: Suppress --sign-only JSON output

* nits

* Docs for multi-session offline signing

(cherry picked from commit 98228c392e)

Co-authored-by: Trent Nelson <trent@solana.com>
2020-03-18 23:39:44 -07:00
mergify[bot]
a5938e5a21 Bump bv from 0.11.0 to 0.11.1 (bp #8952) (#8955)
automerge
2020-03-18 22:33:05 -07:00
mergify[bot]
8eaa8d8788 Add docs on wallets and generating keys (bp #8905) (#8946)
automerge
2020-03-18 19:15:34 -07:00
mergify[bot]
51940eec9b Delete broken link (#8950) (#8951)
automerge
2020-03-18 18:20:36 -07:00
mergify[bot]
d1e16b2bc4 Remove product string from device keypair URL (#8942) (#8945)
automerge
2020-03-18 14:11:41 -07:00
mergify[bot]
97051c87b4 Document account/signer requirements for vote instructions (#8941)
automerge
2020-03-18 12:20:02 -07:00
mergify[bot]
2521913654 Add counter for accounts hash verification. (#8928) (#8934)
automerge
2020-03-18 09:57:38 -07:00
mergify[bot]
803c87b4bd Add docs for --trusted-validator options (#8911) (#8933)
automerge
2020-03-18 08:37:15 -07:00
mergify[bot]
8ff9ee3a06 Cli: add spinner progress bar when waiting for transaction confirmation (#8916) (#8920)
automerge
2020-03-17 20:06:01 -07:00
mergify[bot]
25bf2c6062 Extend local-cluster CI timeout (#8921) (#8923)
automerge
2020-03-17 19:48:57 -07:00
mergify[bot]
fc80b77fc4 Increase buffer on low SOL fault to over a week (#8903) (#8904)
automerge
2020-03-17 10:32:28 -07:00
mergify[bot]
5a7707362c Sort device paths for select (#8896) (#8897)
automerge
2020-03-16 19:43:11 -07:00
mergify[bot]
0102ee3fa9 Hoist USB URL docs (#8894) (#8895)
automerge
2020-03-16 16:31:35 -07:00
mergify[bot]
d29cb19a73 Cli: enable flexible flexible signer paths for pubkey args (#8892) (#8893)
automerge
2020-03-16 16:16:58 -07:00
sakridge
1cc66f0cd7 Add Accounts hash consistency halting (#8772) (#8889)
* Accounts hash consistency halting

* Add option to inject account hash faults for testing.

Enable option in local cluster test to see that node halts.
2020-03-16 14:29:44 -07:00
mergify[bot]
e2cfc513eb Add genesis token counter test to system test (#8824) (#8891)
automerge
2020-03-16 13:32:58 -07:00
mergify[bot]
8115a962e9 Cleanup CLI types (#8888) (#8890)
automerge
2020-03-16 12:28:22 -07:00
mergify[bot]
0c804e2ef2 Use types for CLI value names (#8878) (#8886)
automerge
2020-03-16 09:23:15 -07:00
mergify[bot]
cbe36a7a63 Lower error level of InvalidTickCount (bp #8880) (#8885)
automerge
2020-03-16 08:53:33 -07:00
mergify[bot]
174825fecf Fix faucet command in run.sh (#8883) (#8884)
automerge
2020-03-16 05:38:31 -07:00
Michael Vines
c7e80bebdc Bump version to 1.0.8 2020-03-15 21:50:06 -07:00
Michael Vines
57abc370fa Quietly re-introduce legacy --voting-keypair/--identity-keypair args for v1.0.6 compatibility
(cherry picked from commit 49706172f3)
2020-03-15 20:21:23 -07:00
mergify[bot]
42ab421a87 Rename leader to validator, drop _keypair/-keypair suffix (#8876) (#8877)
automerge
2020-03-15 14:14:32 -07:00
mergify[bot]
9ab27291f5 Validators now run a full gossip node while looking for a snapshot (#8872)
automerge
2020-03-15 10:29:13 -07:00
mergify[bot]
27e4e9cb8d Cli: Add resolve-signer subcommand (#8859) (#8870)
automerge
2020-03-14 22:14:24 -07:00
mergify[bot]
b0cf65dfc8 Refactor system tests dir structure (#8865) (#8869)
automerge
2020-03-14 19:38:19 -07:00
Dan Albert
bfc97c682c Apply s/faucet-keypair/faucet renaming to net scripts (#8868)
automerge
2020-03-14 18:14:49 -07:00
mergify[bot]
231c9b25d1 Rework validator vote account defaults to half voting fees (#8858)
automerge
2020-03-13 21:47:32 -07:00
Michael Vines
f536d805ed Rework cluster metrics dashboard to support the modern clusters
(cherry picked from commit 5f5824d78d)

# Conflicts:
#	system-test/automation_utils.sh
2020-03-13 20:18:06 -07:00
Michael Vines
cb6e7426a4 Drop :8899 port from http://devnet.solana.com references
(cherry picked from commit 9e0a26628b)
2020-03-13 20:14:02 -07:00
mergify[bot]
71f391ae04 Enable any signer in various cli subcommands (#8844) (#8856)
automerge
2020-03-13 18:11:05 -07:00
mergify[bot]
12bf34f059 Remove holding Poh lock (#8838) (#8850)
automerge
2020-03-13 16:56:22 -07:00
Grimes
a3a14d6b5b Docs: Use correct flag in keypair verification instructions (#8677)
automerge

(cherry picked from commit 542691c4e4)
2020-03-13 16:00:33 -07:00
Michael Vines
5e22db017a Surface the missing pubkey
(cherry picked from commit ce88602ced)
2020-03-13 16:00:01 -07:00
mergify[bot]
2cd09957b4 Cli: add subcommand to withdraw from vote account (#8550) (#8840)
automerge
2020-03-13 15:31:47 -07:00
mergify[bot]
d1ec6c0b8b Upgrade to Rust 1.42 (#8836) (#8839)
automerge
2020-03-13 14:29:18 -07:00
Michael Vines
7722723400 Bump version to 1.0.7 2020-03-13 08:10:09 -07:00
Trent Nelson
4a42cfc42a Cli error cleanup 1.0 (#8834)
* Don't use move semantics if not needed (#8793)

* SDK: Deboilerplate `TransportError` with thiserror

* Enable interchange between `TransportError` and `ClientError`

* SDK: Retval consistency between `Client` and `AsyncClient` traits

* Client: Introduce/use `Result` type

* Client: Remove unused `RpcResponseIn` type

* Client: Rename `RpcResponse` to more appropriate `RpcResult`

* Client: Death to `io::Result` return types

* Client: Struct-ify `ClientError`

* Client: Add optional `command` parameter to `ClientError`

* RpcClient: Stop abusing `io::Error` (low-fruit)

* ClientError: Use `thiserror`'s `Display` impl

* Extend `RpcError`'s utility

* RpcClient: Stop abusing `io::Error` (the rest)

* CLI: Shim `main()` so we can `Display` format errors

* claputils: format input validator errors with `Display`

They are intended to be displayed to users

* SDK: `thiserror` for hash and sig parse erros

* Keygen: Shim main to format errors with `Display`

* SDK: `thiserror` for `InstructionError`

* CLI: `thiserror` for `CliError`

* CLI: Format user messages with `Display`

* Client: Tweak `Display` for `ClientError`

* RpcClient: Improve messaging when TX cannot be confirmed

* fu death io res retval

* CLI/Keygen - fix shell return value on error

* Tweak `InstructionError` `Display` messages as per review

* Cleanup hackjob return code fix

* Embrace that which you hate most

* Too much...

Co-authored-by: Jack May <jack@solana.com>
2020-03-13 07:42:25 -06:00
mergify[bot]
976d744b0d Enable conservative out-of-bound snapshot cleaning (#8811) (#8832)
automerge
2020-03-12 23:40:17 -07:00
mergify[bot]
62de02c8d4 Avoid early clean and bad snapshot by ref-counting (#8724) (#8831)
automerge
2020-03-12 23:09:45 -07:00
mergify[bot]
2741a5ce3b Move history out of intro (bp #8825) (#8826)
automerge
2020-03-12 18:21:56 -07:00
mergify[bot]
0c818acd90 Move intro out of README (#8735) (#8827)
automerge

(cherry picked from commit c0fd017906)

Co-authored-by: Greg Fitzgerald <greg@solana.com>
2020-03-12 18:49:51 -06:00
Michael Vines
4d04915302 Update source markdown in CI 2020-03-12 14:56:00 -07:00
Michael Vines
7576411c59 Don't tell users to install unreleased software versions 2020-03-12 14:55:46 -07:00
Michael Vines
ab2bed6e8f Add all of docs/src 2020-03-12 14:45:48 -07:00
mergify[bot]
f66a5ba579 Update keys (#8821) (#8822)
automerge
2020-03-12 14:15:32 -07:00
Dan Albert
9253d786f8 Revert over-zealous versioning text (#8819)
automerge
2020-03-12 09:49:26 -07:00
mergify[bot]
d1be1ee49e Fix malformed doc link (#8817) (#8818)
automerge
2020-03-12 09:46:26 -07:00
mergify[bot]
65833eacd8 Update keys (#8814) (#8816)
automerge
2020-03-12 09:39:31 -07:00
Dan Albert
921994e588 Restrict which nodes can run stable and coverage (#8807)
automerge
2020-03-11 19:18:10 -07:00
mergify[bot]
91dfd962e5 Update keys (#8800) (#8806)
automerge
2020-03-11 19:04:46 -07:00
Trent Nelson
18067dcb55 CLI: Plumb nonce-stored fees (#8750)
automerge

(cherry picked from commit 0422af2aae)
2020-03-11 13:09:40 -07:00
Greg Fitzgerald
7501e1b0f0 Update keys (#8791)
(cherry picked from commit a0d0d4c0e9)
2020-03-11 13:08:48 -07:00
mergify[bot]
1bd2c1de20 Notify when validator balance goes below 1 SOL (#8792)
automerge
2020-03-11 11:47:08 -07:00
mergify[bot]
a9ef20d43f Update keys (#8783) (#8784)
automerge
2020-03-10 21:39:34 -07:00
mergify[bot]
00a9d66fed Improve install messaging (#8477) (#8785)
automerge
2020-03-10 20:30:39 -07:00
mergify[bot]
7508ffe703 Add checkmark (#8781) (#8782)
automerge
2020-03-10 18:32:53 -07:00
Michael Vines
6c2368bfc9 Permit fee-payer/split-stake accounts to be the same when using --seed
(cherry picked from commit 775ce3a03f)
2020-03-10 16:16:26 -07:00
Michael Vines
6bcb797260 Revert to a computed websocket_url value when json_rpc_url is changed
(cherry picked from commit f655372b08)
2020-03-10 16:12:23 -07:00
mergify[bot]
329945e56f Print approved msg after Ledger interaction (#8771) (#8775)
automerge
2020-03-10 15:06:43 -07:00
mergify[bot]
1f80346d97 watchtower: Add --monitor-active-stake flag (bp #8765) (#8769)
automerge
2020-03-10 13:27:19 -07:00
mergify[bot]
38e661d7ba CLI Nonce account access dereplicode (#8743) (#8767)
automerge
2020-03-10 13:15:18 -07:00
mergify[bot]
592c4efd17 Configure the cluster right after installing it (#8761) (#8762)
automerge
2020-03-10 10:14:55 -07:00
mergify[bot]
1629745bd3 Move docs to imperative mood (bp #8643) (#8763)
automerge
2020-03-10 09:50:59 -07:00
mergify[bot]
02e078c22e Fix Gitbook's markdown rendering (#8759) (#8760)
automerge
2020-03-10 08:09:51 -07:00
Michael Vines
ea9e7c710a Bump version to 1.0.6 2020-03-09 22:38:57 -07:00
Michael Vines
ee5aa0c1a2 Support monitoring multiple validators 2020-03-09 21:07:32 -07:00
HM
55dee2901e watchtower: flag to suppress duplicate notifications (#8549)
* watchtower: send error message as notification

* watchtower: send all clear notification when ok again

* watchtower: add twilio sms notifications

* watchtower: flag to suppress duplicate notifications

* remove trailing space character

* changes as per suggestion on PR

* all changes together

* cargo fmt
2020-03-09 21:07:32 -07:00
Michael Vines
2b0824d18b watchtower now uses cli-config/
(cherry picked from commit 74e7da214a)
2020-03-09 20:40:18 -07:00
Michael Vines
b0709ea0ac Move cli-config default out of cli/ into cli-config/
(cherry picked from commit 756ba07b16)
2020-03-09 20:40:18 -07:00
Michael Vines
81b5499f7a Rename 'url' to 'json_rpc_url'
(cherry picked from commit 5c236fd06c)
2020-03-09 20:40:18 -07:00
Michael Vines
c96ce99705 Wait for 80% of the active stake instead of 75% 2020-03-09 20:31:44 -07:00
mergify[bot]
8dcd2d11e1 Docs: Fix missing CLI usage.md (#8745) (#8749)
automerge
2020-03-09 20:05:24 -07:00
mergify[bot]
777aae9059 Remove --derivation-path option (#8741) (#8747)
automerge
2020-03-09 19:17:14 -07:00
mergify[bot]
4dd1340236 Limit waiting-message to single- or last-chunk apdus (#8730) (#8733)
automerge
2020-03-09 15:44:17 -07:00
mergify[bot]
889b06e1d4 Allow passing of program_id to programs (bp #8639) (#8670)
automerge
2020-03-09 12:36:34 -07:00
mergify[bot]
f511296ee8 Fix account tests (#8615) (#8729)
automerge
2020-03-09 11:19:14 -07:00
mergify[bot]
c19eb717b4 Update rust-bpf to include matching cargo (#8598) (#8727)
automerge
2020-03-09 10:42:19 -07:00
mergify[bot]
dd54369e1b Cli: Fix create-with-seed (#8706) (#8723)
automerge
2020-03-09 00:16:29 -07:00
Michael Vines
bb563b4835 Permit --no-untrusted-rpc without any --trusted-validators 2020-03-08 22:34:53 -07:00
mergify[bot]
d061fadede Add purge function to ledger-tool (#8719) (#8720)
(cherry picked from commit de34187db0)

Co-authored-by: sakridge <sakridge@gmail.com>
2020-03-08 21:38:38 -07:00
mergify[bot]
577cd2bd3a Remove unnecessary snapshot hash verification (#8711) (#8714)
(cherry picked from commit f992ee3140)

Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-07 14:41:45 -07:00
mergify[bot]
9d1c8657e2 Groom ledger-tool bounds output (#8710) (#8715)
(cherry picked from commit acb23e8ef0)

Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-07 14:15:50 -07:00
mergify[bot]
67216ac7f5 Split staker infos (#8683)
automerge
2020-03-06 23:44:06 -08:00
mergify[bot]
e63b24c39f Remove ask-seed-phrase arg from validator, archiver (#8697) (#8713)
automerge
2020-03-06 23:34:20 -08:00
Greg Fitzgerald
d963f7afb4 Set withdrawer keys (#8707)
automerge
2020-03-06 21:21:00 -08:00
mergify[bot]
a07bf4870a Fix Ledger docs (#8705) (#8708)
automerge
2020-03-06 21:03:15 -08:00
mergify[bot]
8422d4b3fb Disable setLogFilter RPC API by default (#8693) (#8699)
automerge
2020-03-06 18:16:04 -08:00
mergify[bot]
ff4731cce2 RPC: Add getFeeCalculatorForBlockhash method call (#8687) (#8698)
automerge
2020-03-06 17:25:33 -08:00
mergify[bot]
659aaafff6 Ledger: return specific error if ledger-app-solana is not running (#8684) (#8695)
automerge
2020-03-06 16:28:53 -08:00
mergify[bot]
175651c497 Add shred version support to net/ (#8689) (#8694)
automerge
2020-03-06 15:41:16 -08:00
mergify[bot]
085e773f27 Properly escape current version (#8686) (#8688)
(cherry picked from commit a78a339407)

Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-06 14:37:41 -07:00
Michael Vines
5d3140c040 Cargo.lock 2020-03-06 14:25:54 -07:00
Michael Vines
d8d7238920 Bump version to 1.0.5 2020-03-06 14:14:53 -07:00
mergify[bot]
418c3cd4cf Delete Archiver installation docs (#8665) (#8669)
automerge
2020-03-06 12:49:52 -08:00
Michael Vines
9f532cb50f Publish initial snapshot hash in gossip on validator startup (#8678) 2020-03-05 23:01:35 -07:00
Michael Vines
c35f4927cd Bump version to 1.0.4 2020-03-05 16:10:24 -07:00
Michael Vines
2d0f4b5c8c Bump version to 1.0.3 2020-03-05 13:05:12 -07:00
Grimes
fe59ee61e6 Add find_incomplete_slots (#8654) (#8671)
automerge
2020-03-05 11:52:20 -08:00
Michael Vines
bda2acb06d SDK: Allow RecentBlockhashes to hold the entire BlockhashQueue (#8632) (#8672)
automerge

(cherry picked from commit 9d667db634)

Co-authored-by: Grimes <39311140+solana-grimes@users.noreply.github.com>
2020-03-05 12:09:13 -07:00
Grimes
ac3fe1da02 Fix docs build (#8663) (#8667)
automerge
2020-03-05 10:25:43 -08:00
Michael Vines
57813041d2 Remove solana-archiver from release artifacts 2020-03-05 11:02:18 -07:00
Grimes
7e446da82c Choose a cluster before checking balances (#8666) (#8668)
automerge
2020-03-05 10:00:53 -08:00
Grimes
d94f5c94a3 Nonce fees 1.0 (#8664)
automerge
2020-03-05 09:41:45 -08:00
Michael Vines
4d9aee4794 Always and fully normalize stored 0-lamport accts. (#8657) (#8661)
(cherry picked from commit f146c92e88)

Co-authored-by: Michael Vines <mvines@gmail.com>
2020-03-05 09:49:26 -07:00
Grimes
f32c152bce genesis: Add support for multiple bootstrap validators (#8656) (#8658)
automerge
2020-03-05 00:53:40 -08:00
Grimes
5caf9110e0 Add --bind-address and --rpc-bind-address validator arguments (#8628) (#8655)
automerge
2020-03-04 22:39:38 -08:00
Grimes
d1c80143ea Add NextSlotsIterator (#8652) (#8653)
automerge
2020-03-04 21:59:04 -08:00
Grimes
bb132df121 Add orphan iterator (#8636) (#8651)
automerge
2020-03-04 20:02:46 -08:00
Grimes
b3af1c7e57 Connect partition flag to validators (#8622) (#8646)
automerge
2020-03-04 18:16:54 -08:00
Grimes
14cba53338 Install Solana before using it (#8638) (#8644)
automerge
2020-03-04 16:08:25 -08:00
Grimes
dc8abbe9e3 solana catchup now detects when you try to catchup to yourself (#8635) (#8641)
automerge
2020-03-04 16:00:20 -08:00
carllin
dd06001ed8 compute_bank_stats needs to return newly computed ForkStats (#8608) (#8634)
* Fix broken confirmation, add test

(cherry picked from commit f23dc11a86)

Co-authored-by: carllin <wumu727@gmail.com>
2020-03-04 14:28:17 -08:00
Grimes
298b7de2e2 catchup now supports an optional RPC URL argument for validators with private RPC (#8629) (#8633)
automerge
2020-03-04 12:48:18 -08:00
Grimes
74cbc6953f Expose executable and rent_epoch in AccountInfo (#8619) (#8631)
automerge
2020-03-04 12:09:14 -08:00
mergify[bot]
27e5203078 Fix sendTransaction doc (#8625) (#8626)
automerge
2020-03-04 08:53:30 -08:00
mergify[bot]
73787e162c Keep GenesisConfig binary compatible with v0.23 (#8617) (#8620)
automerge
2020-03-04 02:02:28 -08:00
mergify[bot]
74ae40be41 Check transaction signatures in entry verify (#8596) (#8614)
automerge
2020-03-03 22:10:35 -08:00
mergify[bot]
8dd58e9cea Remove accounts hack and correctly restore accounts store counts (#8569) (#8613)
automerge
2020-03-03 22:05:50 -08:00
mergify[bot]
7d86179c60 Split signature throughput tracking out of FeeCalculator (#8447) (#8610)
automerge
2020-03-03 20:36:52 -08:00
mergify[bot]
8115cf1360 Docs: Update CLI offline cmds (#8548) (#8558)
automerge
2020-03-03 20:26:28 -08:00
Michael Vines
3e5d45053d Cargo.lock 2020-03-03 21:14:00 -07:00
mergify[bot]
061319f35a Add commitment flag to vote-account and validators commands (#8597) (#8606)
automerge
2020-03-03 17:47:14 -08:00
mergify[bot]
dc75837500 Use fs::rename which is much faster than move_items (#8579) (#8595)
automerge
2020-03-03 11:23:11 -08:00
mergify[bot]
dfe26f5275 Add Ledger wallet installation instructions (#8581) (#8592)
automerge
2020-03-03 08:29:12 -08:00
mergify[bot]
f4385f7ad2 Do periodic inbound cleaning for rooted slots (#8436) (#8583)
automerge
2020-03-02 23:04:57 -08:00
Michael Vines
8df4cf2895 Don't advertise the snapshot that the node was loaded from
snapshot_packager_service will remove this snapshot hash from gossip
when it starts
2020-03-02 19:16:06 -07:00
mergify[bot]
dad62e132e Remove granularity from genesis (bp #8514) (#8518)
automerge
2020-03-02 15:42:46 -08:00
Michael Vines
0d4131ae68 Bump version to 1.0.2 2020-03-02 15:38:26 -07:00
mergify[bot]
a2539e1892 Allow stake lockup fields to be updated independently (#8568) (#8574)
automerge
2020-03-02 14:19:18 -08:00
Michael Vines
210659e6c3 Only gossip packaged snapshots
(cherry picked from commit 42c5c59800)
2020-03-02 14:21:04 -07:00
Michael Vines
15a0fb1fa9 --wait-for-supermajority now requires a SLOT
(cherry picked from commit 13551885c2)
2020-03-02 13:40:52 -07:00
Michael Vines
4db31f5b48 Add ---no-untrusted-rpc flag
(cherry picked from commit d677e83ed4)
2020-03-02 11:53:15 -07:00
mergify[bot]
b38a535c63 Hack to skip cleanup_dead_slots upon snapshot load (#8562)
automerge
2020-03-02 10:40:41 -08:00
Michael Vines
218b02aaf8 Demote gossip responder error log messages to info! 2020-03-01 10:42:32 -07:00
mergify[bot]
d6e7cbd4e8 feat: implement websocket_url as a get/set-able global parameter w/ value computation (#8553)
automerge
2020-03-01 01:09:09 -08:00
mergify[bot]
f2fda14333 Reduce max snapshot hashes to stay under MTU (bp #8541) (#8544)
automerge
2020-02-29 09:17:09 -08:00
Michael Vines
1c576d4a68 Upgrade to Rust 1.41.1
(cherry picked from commit 7d27be2a73)
2020-02-29 09:27:11 -07:00
mergify[bot]
f6232e1b3c Fix skipping own leader slots (#8533) (#8540)
automerge
2020-02-29 01:27:46 -08:00
mergify[bot]
ad71fa3f12 rpc: GET for /snapshot.tar.bz2 now redirects to the latest snapshot (bp #8536) (#8538)
automerge
2020-02-28 23:45:15 -08:00
mergify[bot]
9c326c7c71 Ensure the validator's identity pubkey is not provided as a --trusted-validator (#8525) (#8527)
automerge
2020-02-27 22:07:07 -08:00
mergify[bot]
ac545cadaf Add versioning (#8348) (#8524)
automerge
2020-02-27 20:12:49 -08:00
mergify[bot]
082d8fff36 Use legit solana message in verify (#8513) (#8523)
automerge
2020-02-27 19:36:37 -08:00
mergify[bot]
2c3632a042 Determine vote_state ahead of time (#8303) (#8521)
automerge
2020-02-27 18:32:27 -08:00
mergify[bot]
7b23e79922 Add snapshot hash of full accounts state (#8295) (#8515)
* Add snapshot hash of full accounts state

* Use normal hashing for the accounts delta state

* Add merkle

(cherry picked from commit 947a339714)

Co-authored-by: sakridge <sakridge@gmail.com>
2020-02-27 17:36:28 -08:00
mergify[bot]
9afd14a0c6 Import Tour de SOL docs (#8516) (#8519)
automerge
2020-02-27 17:15:36 -08:00
Michael Vines
f42aa34ab6 Reorder InstructionError to remain compatible with v0.23
(cherry picked from commit 7dac8e2dde)
2020-02-27 18:07:00 -07:00
mergify[bot]
43af91ff4b Ledger messaging cleanup (#8506) (#8508)
automerge
2020-02-27 12:38:09 -08:00
mergify[bot]
a920a0f946 Fix cluster economics figures and spelling in docs (#8502) (#8505)
automerge
2020-02-27 02:49:10 -08:00
mergify[bot]
4a8a6d0b3f Remove loop (#8496)
automerge
2020-02-27 00:38:27 -08:00
mergify[bot]
a64b8a2705 Rename snapshot.tar.bz2 to snapshot-<slot>-<hash>.tar.bz2 (bp #8482) (#8501)
automerge
2020-02-26 23:31:44 -08:00
Michael Vines
0198f9e8af Peg snapshot version to 1.0.0 2020-02-26 22:07:50 -07:00
Michael Vines
1582a3a927 Cargo.lock 2020-02-26 21:17:40 -07:00
mergify[bot]
e713f0e5fb Update voting simulation (#8489) 2020-02-26 20:00:20 -08:00
mergify[bot]
77031000c4 Choose more appropriate options for pubsub websocket server (#8354) (#8492)
automerge
2020-02-26 18:23:50 -08:00
Michael Vines
635a962fba Reference the v1.0.0 installer 2020-02-26 19:20:58 -07:00
mergify[bot]
c59ec2dcff Add flag to confirm key on device (#8478) (#8490)
automerge
2020-02-26 17:31:52 -08:00
mergify[bot]
abc6c5e264 Limit leader schedule search space (#8468) (#8486)
automerge
2020-02-26 16:11:14 -08:00
mergify[bot]
87cfac12dd Validate the genesis config downloaded over RPC before accepting it (bp #8474) (#8481)
automerge
2020-02-26 15:12:06 -08:00
Tyera Eulberg
60b43e34b6 Ledger hardware wallet docs (#8472) (#8479)
automerge
2020-02-26 14:42:08 -08:00
mergify[bot]
9ab6222f03 Move docs from book/ to docs/ (#8469) (#8471)
automerge
2020-02-26 08:01:44 -08:00
mergify[bot]
2298dd5c07 Use runtime executor to send pubsub notifications (#8353) (#8465)
automerge
2020-02-25 21:45:18 -08:00
mergify[bot]
822d166115 live-slots now displays the rate the root slot is advancing (#8464)
automerge
2020-02-25 21:18:10 -08:00
mergify[bot]
8f71580615 Allow withdrawer to change the authorized stake key (#8456) (#8462)
automerge
2020-02-25 19:30:59 -08:00
mergify[bot]
cc3352ff06 Ledger key path rework (#8453) (#8457)
automerge
2020-02-25 18:00:53 -08:00
mergify[bot]
8f5928b7c7 Promote dangerous cond. from just warning to panic (#8439) (#8449)
automerge
2020-02-25 15:31:06 -08:00
Michael Vines
888e9617ff 🐌🐌 Publish crates for even longer longer 2020-02-25 09:23:20 -07:00
mergify[bot]
4695c4cf7d Add --no-check-vote-account argument (#8430) (#8435)
automerge
2020-02-25 00:23:50 -08:00
mergify[bot]
bbfc56ff7f Make solana root key accessible on Ledger (#8421) (#8431)
automerge
2020-02-24 22:28:28 -08:00
Michael Vines
4103d99018 Bump version to 1.0.1 2020-02-24 23:24:27 -07:00
mergify[bot]
c375ce1fcd CLI: collect and deduplicate signers (#8398) (#8423)
automerge
2020-02-24 17:29:34 -08:00
Jack May
df813b31c5 Fix SDK deps 2020-02-24 17:30:46 -07:00
Michael Vines
7db92e2951 Drop print- prefix from slot/accounts command
(cherry picked from commit 89baa94002)
2020-02-24 17:28:36 -07:00
Michael Vines
6585518f78 Add genesis subcommand
(cherry picked from commit 1ef3478709)
2020-02-24 17:28:36 -07:00
Michael Vines
6398e256e4 Move shred_version module to sdk/
(cherry picked from commit 73063544bd)
2020-02-24 17:28:36 -07:00
mergify[bot]
b64ebb45e7 validator: snapshot fetching cleanup (bp #8413) (#8417)
automerge
2020-02-24 15:22:34 -08:00
mergify[bot]
307ac66d9c Reinstate create-stale-account w/ seed test (#8401) (#8402)
automerge
2020-02-22 10:56:39 -08:00
332 changed files with 12923 additions and 17887 deletions

42
.appveyor.yml Normal file
View File

@@ -0,0 +1,42 @@
version: '{build}'
branches:
only:
- master
- /^v[0-9.]+\.[0-9.]+/
cache:
- '%USERPROFILE%\.cargo'
- '%APPVEYOR_BUILD_FOLDER%\target'
clone_folder: d:\projects\solana
build_script:
- bash ci/publish-tarball.sh
notifications:
- provider: Slack
incoming_webhook:
secure: GJsBey+F5apAtUm86MHVJ68Uqa6WN1SImcuIc4TsTZrDhA8K1QWUNw9FFQPybUWDyOcS5dly3kubnUqlGt9ux6Ad2efsfRIQYWv0tOVXKeY=
channel: ci-status
on_build_success: false
on_build_failure: true
on_build_status_changed: true
deploy:
- provider: S3
access_key_id:
secure: fTbJl6JpFebR40J7cOWZ2mXBa3kIvEiXgzxAj6L3N7A=
secret_access_key:
secure: vItsBXb2rEFLvkWtVn/Rcxu5a5+2EwC+b7GsA0waJy9hXh6XuBAD0lnHd9re3g/4
bucket: release.solana.com
region: us-west-1
set_public: true
- provider: GitHub
auth_token:
secure: 81fEmPZ0cV1wLtNuUrcmtgxKF6ROQF1+/ft5m+fHX21z6PoeCbaNo8cTyLioWBj7
draft: false
prerelease: false
on:
appveyor_repo_tag: true

1
.gitignore vendored
View File

@@ -1,6 +1,5 @@
/docs/html/
/docs/src/tests.ok
/docs/src/cli/usage.md
/docs/src/.gitbook/assets/*.svg
/farf/
/solana-release/

View File

@@ -19,13 +19,20 @@ pull_request_rules:
label:
add:
- automerge
- name: v0.23 backport
conditions:
- base=master
- label=v0.23
actions:
backport:
branches:
- v0.23
- name: v1.0 backport
conditions:
- base=master
- label=v1.0
actions:
backport:
ignore_conflicts: true
branches:
- v1.0
- name: v1.1 backport
@@ -34,7 +41,6 @@ pull_request_rules:
- label=v1.1
actions:
backport:
ignore_conflicts: true
branches:
- v1.1
- name: v1.2 backport
@@ -43,6 +49,5 @@ pull_request_rules:
- label=v1.2
actions:
backport:
ignore_conflicts: true
branches:
- v1.2

View File

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
If there are no functional changes, PRs can be very large and that's no
problem. If, however, your changes are making meaningful changes or additions,
then about 1,000 lines of changes is about the most you should ask a Solana
then about 1000 lines of changes is about the most you should ask a Solana
maintainer to review.
### Should I send small PRs as I develop large, new components?

1774
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -3,7 +3,6 @@ members = [
"bench-exchange",
"bench-streamer",
"bench-tps",
"accounts-bench",
"banking-bench",
"chacha",
"chacha-cuda",
@@ -11,8 +10,6 @@ members = [
"cli-config",
"client",
"core",
"dos",
"download-utils",
"faucet",
"perf",
"validator",
@@ -27,7 +24,6 @@ members = [
"logger",
"log-analyzer",
"merkle-tree",
"streamer",
"measure",
"metrics",
"net-shaper",
@@ -52,7 +48,6 @@ members = [
"sdk",
"sdk-c",
"scripts",
"stake-accounts",
"stake-monitor",
"sys-tuner",
"transaction-status",

View File

@@ -9,7 +9,46 @@ Blockchain Rebuilt for Scale
Solana&trade; is a new blockchain architecture built from the ground up for scale. The architecture supports
up to 710 thousand transactions per second on a gigabit network.
Read all about it at [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com/v/master).
Documentation
===
Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
Release Binaries
===
Official release binaries are available at [Github Releases](https://github.com/solana-labs/solana/releases).
Additionally we provide pre-release binaries for the latest code on the edge and
beta channels. Note that these pre-release binaries may be less stable than an
official release.
### Edge channel
#### Linux (x86_64-unknown-linux-gnu)
* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
#### mac OS (x86_64-apple-darwin)
* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-apple-darwin.tar.bz2)
* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
#### Windows (x86_64-pc-windows-msvc)
* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-pc-windows-msvc.tar.bz2)
* [solana-install-init.exe](http://release.solana.com/edge/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
#### All platforms
* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/edge/solana-metrics.tar.bz2)
### Beta channel
#### Linux (x86_64-unknown-linux-gnu)
* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
#### mac OS (x86_64-apple-darwin)
* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-apple-darwin.tar.bz2)
* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
#### Windows (x86_64-pc-windows-msvc)
* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-pc-windows-msvc.tar.bz2)
* [solana-install-init.exe](http://release.solana.com/beta/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
#### All platforms
* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/beta/solana-metrics.tar.bz2)
Developing
===

View File

@@ -1,19 +0,0 @@
[package]
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-accounts-bench"
version = "1.1.3"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
log = "0.4.6"
rayon = "1.3.0"
solana-logger = { path = "../logger", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-measure = { path = "../measure", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
rand = "0.6.5"
clap = "2.33.0"
crossbeam-channel = "0.4"

View File

@@ -1,103 +0,0 @@
use clap::{value_t, App, Arg};
use rayon::prelude::*;
use solana_measure::measure::Measure;
use solana_runtime::accounts::{create_test_accounts, update_accounts, Accounts};
use solana_sdk::pubkey::Pubkey;
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
fn main() {
solana_logger::setup();
let matches = App::new("crate")
.about("about")
.version("version")
.arg(
Arg::with_name("num_slots")
.long("num_slots")
.takes_value(true)
.value_name("SLOTS")
.help("Number of slots to store to."),
)
.arg(
Arg::with_name("num_accounts")
.long("num_accounts")
.takes_value(true)
.value_name("NUM_ACCOUNTS")
.help("Total number of accounts"),
)
.arg(
Arg::with_name("iterations")
.long("iterations")
.takes_value(true)
.value_name("ITERATIONS")
.help("Number of bench iterations"),
)
.arg(
Arg::with_name("clean")
.long("clean")
.takes_value(false)
.help("Run clean"),
)
.get_matches();
let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4);
let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
let clean = matches.is_present("clean");
println!("clean: {:?}", clean);
let path = PathBuf::from("farf/accounts-bench");
if fs::remove_dir_all(path.clone()).is_err() {
println!("Warning: Couldn't remove {:?}", path);
}
let accounts = Accounts::new(vec![path]);
println!("Creating {} accounts", num_accounts);
let mut create_time = Measure::start("create accounts");
let pubkeys: Vec<_> = (0..num_slots)
.into_par_iter()
.map(|slot| {
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(
&accounts,
&mut pubkeys,
num_accounts / num_slots,
slot as u64,
);
pubkeys
})
.collect();
let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect();
create_time.stop();
println!(
"created {} accounts in {} slots {}",
(num_accounts / num_slots) * num_slots,
num_slots,
create_time
);
let mut ancestors: HashMap<u64, usize> = vec![(0, 0)].into_iter().collect();
for i in 1..num_slots {
ancestors.insert(i as u64, i - 1);
accounts.add_root(i as u64);
}
for x in 0..iterations {
if clean {
let mut time = Measure::start("clean");
accounts.accounts_db.clean_accounts();
time.stop();
println!("{}", time);
for slot in 0..num_slots {
update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
accounts.add_root((x * num_slots + slot) as u64);
}
} else {
let mut pubkeys: Vec<Pubkey> = vec![];
let mut time = Measure::start("hash");
let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors);
time.stop();
println!("hash: {} {}", hash, time);
create_test_accounts(&accounts, &mut pubkeys, 1, 0);
}
}
}

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-archiver-lib"
version = "1.1.3"
version = "1.0.16"
description = "Solana Archiver Library"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -10,31 +10,30 @@ edition = "2018"
[dependencies]
bincode = "1.2.1"
crossbeam-channel = "0.4"
crossbeam-channel = "0.3"
ed25519-dalek = "=1.0.0-pre.1"
log = "0.4.8"
rand = "0.6.5"
rand_chacha = "0.1.1"
solana-client = { path = "../client", version = "1.1.3" }
solana-storage-program = { path = "../programs/storage", version = "1.1.3" }
solana-client = { path = "../client", version = "1.0.16" }
solana-storage-program = { path = "../programs/storage", version = "1.0.16" }
thiserror = "1.0"
serde = "1.0.105"
serde_json = "1.0.48"
serde = "1.0.104"
serde_json = "1.0.46"
serde_derive = "1.0.103"
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-chacha = { path = "../chacha", version = "1.1.3" }
solana-chacha-sys = { path = "../chacha-sys", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-core = { path = "../core", version = "1.1.3" }
solana-streamer = { path = "../streamer", version = "1.1.3" }
solana-archiver-utils = { path = "../archiver-utils", version = "1.1.3" }
solana-metrics = { path = "../metrics", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-chacha = { path = "../chacha", version = "1.0.16" }
solana-chacha-sys = { path = "../chacha-sys", version = "1.0.16" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-perf = { path = "../perf", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
solana-core = { path = "../core", version = "1.0.16" }
solana-archiver-utils = { path = "../archiver-utils", version = "1.0.16" }
solana-metrics = { path = "../metrics", version = "1.0.16" }
[dev-dependencies]
hex = "0.4.2"
hex = "0.4.0"
[lib]
name = "solana_archiver_lib"

View File

@@ -10,15 +10,16 @@ use solana_client::{
};
use solana_core::{
cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
cluster_slots::ClusterSlots,
contact_info::ContactInfo,
gossip_service::GossipService,
packet::{limited_deserialize, PACKET_DATA_SIZE},
repair_service,
repair_service::{RepairService, RepairSlotRange, RepairStats, RepairStrategy},
repair_service::{RepairService, RepairSlotRange, RepairStrategy},
serve_repair::ServeRepair,
shred_fetch_stage::ShredFetchStage,
sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
storage_stage::NUM_STORAGE_SAMPLES,
streamer::{receiver, responder, PacketReceiver},
window_service::WindowService,
};
use solana_ledger::{
@@ -26,7 +27,6 @@ use solana_ledger::{
};
use solana_net_utils::bind_in_range;
use solana_perf::packet::Packets;
use solana_perf::packet::{limited_deserialize, PACKET_DATA_SIZE};
use solana_perf::recycler::Recycler;
use solana_sdk::packet::Packet;
use solana_sdk::{
@@ -45,7 +45,6 @@ use solana_storage_program::{
storage_contract::StorageContract,
storage_instruction::{self, StorageAccountType},
};
use solana_streamer::streamer::{receiver, responder, PacketReceiver};
use std::{
io::{self, ErrorKind},
net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@@ -188,7 +187,7 @@ impl Archiver {
let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
cluster_info.set_entrypoint(cluster_entrypoint.clone());
let cluster_info = Arc::new(RwLock::new(cluster_info));
let cluster_slots = Arc::new(ClusterSlots::default());
// Note for now, this ledger will not contain any of the existing entries
// in the ledger located at ledger_path, and will only append on newly received
// entries after being passed to window_service
@@ -263,7 +262,6 @@ impl Archiver {
repair_socket,
shred_fetch_receiver,
slot_sender,
cluster_slots,
) {
Ok(window_service) => window_service,
Err(e) => {
@@ -402,7 +400,6 @@ impl Archiver {
}
// Find a segment to replicate and download it.
#[allow(clippy::too_many_arguments)]
fn setup(
meta: &mut ArchiverMeta,
cluster_info: Arc<RwLock<ClusterInfo>>,
@@ -413,7 +410,6 @@ impl Archiver {
repair_socket: Arc<UdpSocket>,
shred_fetch_receiver: PacketReceiver,
slot_sender: Sender<u64>,
cluster_slots: Arc<ClusterSlots>,
) -> Result<WindowService> {
let slots_per_segment =
match Self::get_segment_config(&cluster_info, meta.client_commitment) {
@@ -471,7 +467,6 @@ impl Archiver {
RepairStrategy::RepairRange(repair_slot_range),
&Arc::new(LeaderScheduleCache::default()),
|_, _, _, _| true,
cluster_slots,
);
info!("waiting for ledger download");
Self::wait_for_segment_download(
@@ -844,14 +839,13 @@ impl Archiver {
repair_service::MAX_REPAIR_LENGTH,
&repair_slot_range,
);
let mut repair_stats = RepairStats::default();
//iter over the repairs and send them
if let Ok(repairs) = repairs {
let reqs: Vec<_> = repairs
.into_iter()
.filter_map(|repair_request| {
serve_repair
.map_repair_request(&repair_request, &mut repair_stats)
.map_repair_request(&repair_request)
.map(|result| ((archiver_info.gossip, result), repair_request))
.ok()
})

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-archiver-utils"
version = "1.1.3"
version = "1.0.16"
description = "Solana Archiver Utils"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -11,15 +11,15 @@ edition = "2018"
[dependencies]
log = "0.4.8"
rand = "0.6.5"
solana-chacha = { path = "../chacha", version = "1.1.3" }
solana-chacha-sys = { path = "../chacha-sys", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-chacha = { path = "../chacha", version = "1.0.16" }
solana-chacha-sys = { path = "../chacha-sys", version = "1.0.16" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-perf = { path = "../perf", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
[dev-dependencies]
hex = "0.4.2"
hex = "0.4.0"
[lib]
name = "solana_archiver_utils"

View File

@@ -2,19 +2,19 @@
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-archiver"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
clap = "2.33.0"
console = "0.10.0"
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-core = { path = "../core", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-metrics = { path = "../metrics", version = "1.1.3" }
solana-archiver-lib = { path = "../archiver-lib", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
console = "0.9.2"
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-core = { path = "../core", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-metrics = { path = "../metrics", version = "1.0.16" }
solana-archiver-lib = { path = "../archiver-lib", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }

View File

@@ -2,21 +2,19 @@
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-banking-bench"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
log = "0.4.6"
rayon = "1.3.0"
solana-core = { path = "../core", version = "1.1.3" }
solana-streamer = { path = "../streamer", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-measure = { path = "../measure", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
rayon = "1.2.0"
solana-core = { path = "../core", version = "1.0.16" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-runtime = { path = "../runtime", version = "1.0.16" }
solana-measure = { path = "../measure", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
rand = "0.6.5"
crossbeam-channel = "0.4"
crossbeam-channel = "0.3"

View File

@@ -2,36 +2,29 @@ use crossbeam_channel::unbounded;
use log::*;
use rand::{thread_rng, Rng};
use rayon::prelude::*;
use solana_core::{
banking_stage::{create_test_recorder, BankingStage},
cluster_info::ClusterInfo,
cluster_info::Node,
poh_recorder::PohRecorder,
poh_recorder::WorkingBankEntry,
};
use solana_ledger::{
bank_forks::BankForks,
blockstore::Blockstore,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
};
use solana_core::banking_stage::{create_test_recorder, BankingStage};
use solana_core::cluster_info::ClusterInfo;
use solana_core::cluster_info::Node;
use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_core::packet::to_packets_chunked;
use solana_core::poh_recorder::PohRecorder;
use solana_core::poh_recorder::WorkingBankEntry;
use solana_ledger::bank_forks::BankForks;
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
use solana_measure::measure::Measure;
use solana_perf::packet::to_packets_chunked;
use solana_runtime::bank::Bank;
use solana_sdk::{
hash::Hash,
pubkey::Pubkey,
signature::Keypair,
signature::Signature,
system_transaction,
timing::{duration_as_us, timestamp},
transaction::Transaction,
};
use std::{
sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
thread::sleep,
time::{Duration, Instant},
};
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Keypair;
use solana_sdk::signature::Signature;
use solana_sdk::system_transaction;
use solana_sdk::timing::{duration_as_us, timestamp};
use solana_sdk::transaction::Transaction;
use std::sync::atomic::Ordering;
use std::sync::mpsc::Receiver;
use std::sync::{Arc, Mutex, RwLock};
use std::thread::sleep;
use std::time::{Duration, Instant};
fn check_txs(
receiver: &Arc<Receiver<WorkingBankEntry>>,

View File

@@ -2,7 +2,7 @@
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-bench-exchange"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
@@ -10,25 +10,25 @@ publish = false
[dependencies]
clap = "2.32.0"
itertools = "0.9.0"
itertools = "0.8.2"
log = "0.4.8"
num-derive = "0.3"
num-traits = "0.2"
rand = "0.6.5"
rayon = "1.3.0"
serde_json = "1.0.48"
rayon = "1.2.0"
serde_json = "1.0.46"
serde_yaml = "0.8.11"
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-core = { path = "../core", version = "1.1.3" }
solana-genesis = { path = "../genesis", version = "1.1.3" }
solana-client = { path = "../client", version = "1.1.3" }
solana-faucet = { path = "../faucet", version = "1.1.3" }
solana-exchange-program = { path = "../programs/exchange", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-metrics = { path = "../metrics", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-core = { path = "../core", version = "1.0.16" }
solana-genesis = { path = "../genesis", version = "1.0.16" }
solana-client = { path = "../client", version = "1.0.16" }
solana-faucet = { path = "../faucet", version = "1.0.16" }
solana-exchange-program = { path = "../programs/exchange", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-metrics = { path = "../metrics", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-runtime = { path = "../runtime", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
[dev-dependencies]
solana-local-cluster = { path = "../local-cluster", version = "1.1.3" }
solana-local-cluster = { path = "../local-cluster", version = "1.0.16" }

View File

@@ -2,14 +2,14 @@
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-bench-streamer"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
clap = "2.33.0"
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-streamer = { path = "../streamer", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-core = { path = "../core", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }

View File

@@ -1,6 +1,6 @@
use clap::{crate_description, crate_name, App, Arg};
use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
use solana_streamer::streamer::{receiver, PacketReceiver};
use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
use solana_core::streamer::{receiver, PacketReceiver};
use std::cmp::max;
use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

View File

@@ -2,7 +2,7 @@
authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-bench-tps"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
@@ -11,27 +11,27 @@ homepage = "https://solana.com/"
bincode = "1.2.1"
clap = "2.33.0"
log = "0.4.8"
rayon = "1.3.0"
serde_json = "1.0.48"
rayon = "1.2.0"
serde_json = "1.0.46"
serde_yaml = "0.8.11"
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-core = { path = "../core", version = "1.1.3" }
solana-genesis = { path = "../genesis", version = "1.1.3" }
solana-client = { path = "../client", version = "1.1.3" }
solana-faucet = { path = "../faucet", version = "1.1.3" }
solana-librapay = { path = "../programs/librapay", version = "1.1.3", optional = true }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-metrics = { path = "../metrics", version = "1.1.3" }
solana-measure = { path = "../measure", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.3", optional = true }
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-core = { path = "../core", version = "1.0.16" }
solana-genesis = { path = "../genesis", version = "1.0.16" }
solana-client = { path = "../client", version = "1.0.16" }
solana-faucet = { path = "../faucet", version = "1.0.16" }
solana-librapay = { path = "../programs/librapay", version = "1.0.16", optional = true }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-metrics = { path = "../metrics", version = "1.0.16" }
solana-measure = { path = "../measure", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-runtime = { path = "../runtime", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.16", optional = true }
[dev-dependencies]
serial_test = "0.4.0"
serial_test = "0.3.2"
serial_test_derive = "0.4.0"
solana-local-cluster = { path = "../local-cluster", version = "1.1.3" }
solana-local-cluster = { path = "../local-cluster", version = "1.0.16" }
[features]
move = ["solana-librapay", "solana-move-loader-program"]

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-chacha-cuda"
version = "1.1.3"
version = "1.0.16"
description = "Solana Chacha Cuda APIs"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
[dependencies]
log = "0.4.8"
solana-archiver-utils = { path = "../archiver-utils", version = "1.1.3" }
solana-chacha = { path = "../chacha", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-archiver-utils = { path = "../archiver-utils", version = "1.0.16" }
solana-chacha = { path = "../chacha", version = "1.0.16" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-perf = { path = "../perf", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
[dev-dependencies]
hex-literal = "0.2.1"

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-chacha-sys"
version = "1.1.3"
version = "1.0.16"
description = "Solana chacha-sys"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-chacha"
version = "1.1.3"
version = "1.0.16"
description = "Solana Chacha APIs"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -12,11 +12,11 @@ edition = "2018"
log = "0.4.8"
rand = "0.6.5"
rand_chacha = "0.1.1"
solana-chacha-sys = { path = "../chacha-sys", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-chacha-sys = { path = "../chacha-sys", version = "1.0.16" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-perf = { path = "../perf", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
[dev-dependencies]
hex-literal = "0.2.1"

View File

@@ -32,7 +32,6 @@ RUN set -x \
&& cargo install cargo-audit \
&& cargo install svgbob_cli \
&& cargo install mdbook \
&& cargo install mdbook-linkcheck \
&& rustc --version \
&& cargo --version \
&& curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

View File

@@ -67,9 +67,8 @@ _ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
_ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
-- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
# Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
# Run banking bench. Doesn't require nightly, but use since it is already built.
_ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
# `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
# reason

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-clap-utils"
version = "1.1.3"
version = "1.0.16"
description = "Solana utilities for the clap"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
[dependencies]
clap = "2.33.0"
rpassword = "4.0"
solana-remote-wallet = { path = "../remote-wallet", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-remote-wallet = { path = "../remote-wallet", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
thiserror = "1.0.11"
tiny-bip39 = "0.7.0"
url = "2.1.0"

View File

@@ -62,21 +62,6 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
}
}
pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>> {
matches.values_of(name).map(|values| {
values
.filter_map(|value| {
if value == ASK_KEYWORD {
let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
keypair_from_seed_phrase(name, skip_validation, true).ok()
} else {
read_keypair_file(value).ok()
}
})
.collect()
})
}
// Return a pubkey for an argument that can itself be parsed into a pubkey,
// or is a filename that can be read as a keypair
pub fn pubkey_of(matches: &ArgMatches<'_>, name: &str) -> Option<Pubkey> {

View File

@@ -47,13 +47,6 @@ pub fn parse_keypair_path(path: &str) -> KeypairUrl {
}
}
pub fn check_for_usb<S>(mut items: impl Iterator<Item = S>) -> bool
where
S: Into<String>,
{
items.any(|arg| matches!(parse_keypair_path(&arg.into()), KeypairUrl::Usb(_)))
}
pub fn presigner_from_pubkey_sigs(
pubkey: &Pubkey,
signers: &[(Pubkey, Signature)],
@@ -263,20 +256,4 @@ mod tests {
sanitize_seed_phrase(seed_phrase)
);
}
#[test]
fn test_check_for_usb() {
let args: Vec<&str> = vec![];
assert_eq!(check_for_usb(args.into_iter()), false);
let args = vec!["usb://"];
assert_eq!(check_for_usb(args.into_iter()), true);
let args = vec!["other"];
assert_eq!(check_for_usb(args.into_iter()), false);
let args = vec!["other", "usb://", "another"];
assert_eq!(check_for_usb(args.into_iter()), true);
let args = vec!["other", "another"];
assert_eq!(check_for_usb(args.into_iter()), false);
let args = vec!["usb://", "usb://"];
assert_eq!(check_for_usb(args.into_iter()), true);
}
}

View File

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-cli-config"
description = "Blockchain, Rebuilt for Scale"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
@@ -11,7 +11,7 @@ homepage = "https://solana.com/"
[dependencies]
dirs = "2.0.2"
lazy_static = "1.4.0"
serde = "1.0.105"
serde = "1.0.104"
serde_derive = "1.0.103"
serde_yaml = "0.8.11"
url = "2.1.1"

View File

@@ -26,7 +26,7 @@ impl Default for Config {
keypair_path.extend(&[".config", "solana", "id.json"]);
keypair_path.to_str().unwrap().to_string()
};
let json_rpc_url = "https://api.mainnet-beta.solana.com".to_string();
let json_rpc_url = "http://127.0.0.1:8899".to_string();
// Empty websocket_url string indicates the client should
// `Config::compute_websocket_url(&json_rpc_url)`

View File

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
edition = "2018"
name = "solana-cli"
description = "Blockchain, Rebuilt for Scale"
version = "1.1.3"
version = "1.0.16"
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
@@ -11,43 +11,43 @@ homepage = "https://solana.com/"
[dependencies]
bincode = "1.2.1"
bs58 = "0.3.0"
chrono = { version = "0.4.11", features = ["serde"] }
chrono = { version = "0.4.10", features = ["serde"] }
clap = "2.33.0"
criterion-stats = "0.3.0"
ctrlc = { version = "3.1.4", features = ["termination"] }
console = "0.10.0"
console = "0.9.2"
dirs = "2.0.2"
log = "0.4.8"
indicatif = "0.14.0"
humantime = "2.0.0"
num-traits = "0.2"
pretty-hex = "0.1.1"
reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
serde = "1.0.105"
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
serde = "1.0.104"
serde_derive = "1.0.103"
serde_json = "1.0.48"
solana-budget-program = { path = "../programs/budget", version = "1.1.3" }
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-cli-config = { path = "../cli-config", version = "1.1.3" }
solana-client = { path = "../client", version = "1.1.3" }
solana-config-program = { path = "../programs/config", version = "1.1.3" }
solana-faucet = { path = "../faucet", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-remote-wallet = { path = "../remote-wallet", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-stake-program = { path = "../programs/stake", version = "1.1.3" }
solana-storage-program = { path = "../programs/storage", version = "1.1.3" }
solana-vote-program = { path = "../programs/vote", version = "1.1.3" }
solana-vote-signer = { path = "../vote-signer", version = "1.1.3" }
serde_json = "1.0.46"
solana-budget-program = { path = "../programs/budget", version = "1.0.16" }
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-cli-config = { path = "../cli-config", version = "1.0.16" }
solana-client = { path = "../client", version = "1.0.16" }
solana-config-program = { path = "../programs/config", version = "1.0.16" }
solana-faucet = { path = "../faucet", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-remote-wallet = { path = "../remote-wallet", version = "1.0.16" }
solana-runtime = { path = "../runtime", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
solana-stake-program = { path = "../programs/stake", version = "1.0.16" }
solana-storage-program = { path = "../programs/storage", version = "1.0.16" }
solana-vote-program = { path = "../programs/vote", version = "1.0.16" }
solana-vote-signer = { path = "../vote-signer", version = "1.0.16" }
titlecase = "1.1.0"
thiserror = "1.0.13"
thiserror = "1.0.11"
url = "2.1.1"
[dev-dependencies]
solana-core = { path = "../core", version = "1.1.3" }
solana-budget-program = { path = "../programs/budget", version = "1.1.3" }
solana-core = { path = "../core", version = "1.0.16" }
solana-budget-program = { path = "../programs/budget", version = "1.0.16" }
tempfile = "3.1.0"
[[bin]]

View File

@@ -38,9 +38,9 @@ use solana_sdk::{
message::Message,
native_token::lamports_to_sol,
program_utils::DecodeError,
pubkey::{Pubkey, MAX_SEED_LEN},
pubkey::Pubkey,
signature::{Keypair, Signature, Signer, SignerError},
system_instruction::{self, SystemError},
system_instruction::{self, create_address_with_seed, SystemError, MAX_ADDRESS_SEED_LEN},
system_program,
transaction::{Transaction, TransactionError},
};
@@ -173,8 +173,6 @@ pub enum CliCommand {
Catchup {
node_pubkey: Pubkey,
node_json_rpc_url: Option<String>,
commitment_config: CommitmentConfig,
follow: bool,
},
ClusterVersion,
CreateAddressWithSeed {
@@ -196,9 +194,6 @@ pub enum CliCommand {
GetSlot {
commitment_config: CommitmentConfig,
},
TotalSupply {
commitment_config: CommitmentConfig,
},
GetTransactionCount {
commitment_config: CommitmentConfig,
},
@@ -594,7 +589,6 @@ pub fn parse_command(
}),
("epoch", Some(matches)) => parse_get_epoch(matches),
("slot", Some(matches)) => parse_get_slot(matches),
("total-supply", Some(matches)) => parse_total_supply(matches),
("transaction-count", Some(matches)) => parse_get_transaction_count(matches),
("leader-schedule", Some(_matches)) => Ok(CliCommandInfo {
command: CliCommand::LeaderSchedule,
@@ -1074,7 +1068,7 @@ pub fn parse_create_address_with_seed(
let seed = matches.value_of("seed").unwrap().to_string();
if seed.len() > MAX_SEED_LEN {
if seed.len() > MAX_ADDRESS_SEED_LEN {
return Err(CliError::BadParameter(
"Address seed must not be longer than 32 bytes".to_string(),
));
@@ -1101,7 +1095,7 @@ fn process_create_address_with_seed(
} else {
config.pubkey()?
};
let address = Pubkey::create_with_seed(&from_pubkey, seed, program_id)?;
let address = create_address_with_seed(&from_pubkey, seed, program_id)?;
Ok(address.to_string())
}
@@ -1603,15 +1597,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
CliCommand::Catchup {
node_pubkey,
node_json_rpc_url,
commitment_config,
follow,
} => process_catchup(
&rpc_client,
node_pubkey,
node_json_rpc_url,
*commitment_config,
*follow,
),
} => process_catchup(&rpc_client, node_pubkey, node_json_rpc_url),
CliCommand::ClusterVersion => process_cluster_version(&rpc_client),
CliCommand::CreateAddressWithSeed {
from_pubkey,
@@ -1630,9 +1616,6 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
CliCommand::GetSlot { commitment_config } => {
process_get_slot(&rpc_client, *commitment_config)
}
CliCommand::TotalSupply { commitment_config } => {
process_total_supply(&rpc_client, *commitment_config)
}
CliCommand::GetTransactionCount { commitment_config } => {
process_get_transaction_count(&rpc_client, *commitment_config)
}
@@ -3216,7 +3199,7 @@ mod tests {
let process_id = Pubkey::new_rand();
config.command = CliCommand::Cancel(process_id);
assert!(process_command(&config).is_ok());
assert_eq!(process_command(&config).unwrap(), SIGNATURE);
let good_signature = Signature::new(&bs58::decode(SIGNATURE).into_vec().unwrap());
config.command = CliCommand::Confirm(good_signature);
@@ -3233,8 +3216,8 @@ mod tests {
commission: 0,
};
config.signers = vec![&keypair, &bob_keypair, &identity_keypair];
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let new_authorized_pubkey = Pubkey::new_rand();
config.signers = vec![&bob_keypair];
@@ -3243,8 +3226,8 @@ mod tests {
new_authorized_pubkey,
vote_authorize: VoteAuthorize::Voter,
};
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let new_identity_keypair = Keypair::new();
config.signers = vec![&keypair, &bob_keypair, &new_identity_keypair];
@@ -3252,8 +3235,8 @@ mod tests {
vote_account_pubkey: bob_pubkey,
new_identity_account: 2,
};
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let bob_keypair = Keypair::new();
let bob_pubkey = bob_keypair.pubkey();
@@ -3277,8 +3260,8 @@ mod tests {
from: 0,
};
config.signers = vec![&keypair, &bob_keypair];
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let stake_pubkey = Pubkey::new_rand();
let to_pubkey = Pubkey::new_rand();
@@ -3294,8 +3277,8 @@ mod tests {
fee_payer: 0,
};
config.signers = vec![&keypair];
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let stake_pubkey = Pubkey::new_rand();
config.command = CliCommand::DeactivateStake {
@@ -3307,8 +3290,8 @@ mod tests {
nonce_authority: 0,
fee_payer: 0,
};
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let stake_pubkey = Pubkey::new_rand();
let split_stake_account = Keypair::new();
@@ -3325,8 +3308,8 @@ mod tests {
fee_payer: 0,
};
config.signers = vec![&keypair, &split_stake_account];
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
config.command = CliCommand::GetSlot {
commitment_config: CommitmentConfig::default(),
@@ -3344,8 +3327,8 @@ mod tests {
to: bob_pubkey,
..PayCommand::default()
});
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let date_string = "\"2018-09-19T17:30:59Z\"";
let dt: DateTime<Utc> = serde_json::from_str(&date_string).unwrap();
@@ -3357,7 +3340,16 @@ mod tests {
..PayCommand::default()
});
let result = process_command(&config);
assert!(result.is_ok());
let json: Value = serde_json::from_str(&result.unwrap()).unwrap();
assert_eq!(
json.as_object()
.unwrap()
.get("signature")
.unwrap()
.as_str()
.unwrap(),
SIGNATURE.to_string()
);
let witness = Pubkey::new_rand();
config.command = CliCommand::Pay(PayCommand {
@@ -3368,18 +3360,27 @@ mod tests {
..PayCommand::default()
});
let result = process_command(&config);
assert!(result.is_ok());
let json: Value = serde_json::from_str(&result.unwrap()).unwrap();
assert_eq!(
json.as_object()
.unwrap()
.get("signature")
.unwrap()
.as_str()
.unwrap(),
SIGNATURE.to_string()
);
let process_id = Pubkey::new_rand();
config.command = CliCommand::TimeElapsed(bob_pubkey, process_id, dt);
config.signers = vec![&keypair];
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let witness = Pubkey::new_rand();
config.command = CliCommand::Witness(bob_pubkey, witness);
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
// CreateAddressWithSeed
let from_pubkey = Pubkey::new_rand();
@@ -3391,7 +3392,7 @@ mod tests {
};
let address = process_command(&config);
let expected_address =
Pubkey::create_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
create_address_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
assert_eq!(address.unwrap(), expected_address.to_string());
// Need airdrop cases
@@ -3406,13 +3407,13 @@ mod tests {
assert!(process_command(&config).is_ok());
config.command = CliCommand::TimeElapsed(bob_pubkey, process_id, dt);
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
let witness = Pubkey::new_rand();
config.command = CliCommand::Witness(bob_pubkey, witness);
let result = process_command(&config);
assert!(result.is_ok());
let signature = process_command(&config);
assert_eq!(signature.unwrap(), SIGNATURE.to_string());
// sig_not_found case
config.rpc_client = Some(RpcClient::new_mock("sig_not_found".to_string()));

View File

@@ -5,7 +5,6 @@ use crate::{
},
display::println_name_value,
};
use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
use console::{style, Emoji};
use indicatif::{ProgressBar, ProgressStyle};
@@ -23,7 +22,6 @@ use solana_sdk::{
epoch_schedule::Epoch,
hash::Hash,
message::Message,
native_token::lamports_to_sol,
pubkey::Pubkey,
signature::{Keypair, Signer},
system_instruction,
@@ -69,20 +67,6 @@ impl ClusterQuerySubCommands for App<'_, '_> {
.takes_value(true)
.validator(is_url)
.help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
)
.arg(
Arg::with_name("confirmed")
.long("confirmed")
.takes_value(false)
.help(
"Return information at maximum-lockout commitment level",
),
)
.arg(
Arg::with_name("follow")
.long("follow")
.takes_value(false)
.help("Continue reporting progress even after the validator has caught up"),
),
)
.subcommand(
@@ -144,17 +128,6 @@ impl ClusterQuerySubCommands for App<'_, '_> {
),
),
)
.subcommand(
SubCommand::with_name("total-supply").about("Get total number of SOL")
.arg(
Arg::with_name("confirmed")
.long("confirmed")
.takes_value(false)
.help(
"Return count at maximum-lockout commitment level",
),
),
)
.subcommand(
SubCommand::with_name("transaction-count").about("Get current transaction count")
.alias("get-transaction-count")
@@ -287,18 +260,10 @@ pub fn parse_catchup(
) -> Result<CliCommandInfo, CliError> {
let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
let commitment_config = if matches.is_present("confirmed") {
CommitmentConfig::default()
} else {
CommitmentConfig::recent()
};
let follow = matches.is_present("follow");
Ok(CliCommandInfo {
command: CliCommand::Catchup {
node_pubkey,
node_json_rpc_url,
commitment_config,
follow,
},
signers: vec![],
})
@@ -383,18 +348,6 @@ pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliEr
})
}
pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
let commitment_config = if matches.is_present("confirmed") {
CommitmentConfig::default()
} else {
CommitmentConfig::recent()
};
Ok(CliCommandInfo {
command: CliCommand::TotalSupply { commitment_config },
signers: vec![],
})
}
pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
let commitment_config = if matches.is_present("confirmed") {
CommitmentConfig::default()
@@ -454,37 +407,20 @@ pub fn process_catchup(
rpc_client: &RpcClient,
node_pubkey: &Pubkey,
node_json_rpc_url: &Option<String>,
commitment_config: CommitmentConfig,
follow: bool,
) -> ProcessResult {
let sleep_interval = 5;
let progress_bar = new_spinner_progress_bar();
progress_bar.set_message("Connecting...");
let cluster_nodes = rpc_client.get_cluster_nodes()?;
let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
RpcClient::new(node_json_rpc_url.to_string())
} else {
let rpc_addr = loop {
let cluster_nodes = rpc_client.get_cluster_nodes()?;
if let Some(contact_info) = cluster_nodes
RpcClient::new_socket(
cluster_nodes
.iter()
.find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
{
if let Some(rpc_addr) = contact_info.rpc {
break rpc_addr;
}
progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
} else {
progress_bar.set_message(&format!(
"Contact information not found for {}",
node_pubkey
));
}
sleep(Duration::from_secs(sleep_interval as u64));
};
RpcClient::new_socket(rpc_addr)
.ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
.rpc
.ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?,
)
};
let reported_node_pubkey = node_client.get_identity()?;
@@ -500,12 +436,16 @@ pub fn process_catchup(
return Err("Both RPC URLs reference the same node, unable to monitor for catchup. Try a different --url".into());
}
let progress_bar = new_spinner_progress_bar();
progress_bar.set_message("Connecting...");
let mut previous_rpc_slot = std::u64::MAX;
let mut previous_slot_distance = 0;
let sleep_interval = 5;
loop {
let rpc_slot = rpc_client.get_slot_with_commitment(commitment_config)?;
let node_slot = node_client.get_slot_with_commitment(commitment_config)?;
if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
let rpc_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::recent())?;
let node_slot = node_client.get_slot_with_commitment(CommitmentConfig::recent())?;
if node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
progress_bar.finish_and_clear();
return Ok(format!(
"{} has caught up (us:{} them:{})",
@@ -519,7 +459,7 @@ pub fn process_catchup(
slot_distance,
node_slot,
rpc_slot,
if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
if previous_rpc_slot == std::u64::MAX {
"".to_string()
} else {
let slots_per_second =
@@ -594,13 +534,7 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
let timestamp = rpc_client.get_block_time(slot)?;
let result = format!(
"{} (UnixTimestamp: {})",
DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc)
.to_rfc3339_opts(SecondsFormat::Secs, true),
timestamp
);
Ok(result)
Ok(timestamp.to_string())
}
fn slot_to_human_time(slot: Slot) -> String {
@@ -853,14 +787,6 @@ pub fn process_show_block_production(
Ok("".to_string())
}
pub fn process_total_supply(
rpc_client: &RpcClient,
commitment_config: CommitmentConfig,
) -> ProcessResult {
let total_supply = rpc_client.total_supply_with_commitment(commitment_config.clone())?;
Ok(format!("{} SOL", lamports_to_sol(total_supply)))
}
pub fn process_get_transaction_count(
rpc_client: &RpcClient,
commitment_config: CommitmentConfig,
@@ -1276,9 +1202,7 @@ pub fn process_show_validators(
);
}
let mut current = vote_accounts.current;
current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
for vote_account in current.into_iter() {
for vote_account in vote_accounts.current.into_iter() {
print_vote_account(
vote_account,
epoch_info.epoch,
@@ -1287,9 +1211,7 @@ pub fn process_show_validators(
false,
);
}
let mut delinquent = vote_accounts.delinquent;
delinquent.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
for vote_account in delinquent.into_iter() {
for vote_account in vote_accounts.delinquent.into_iter() {
print_vote_account(
vote_account,
epoch_info.epoch,
@@ -1402,19 +1324,6 @@ mod tests {
}
);
let test_total_supply = test_commands
.clone()
.get_matches_from(vec!["test", "total-supply"]);
assert_eq!(
parse_command(&test_total_supply, &default_keypair_file, None).unwrap(),
CliCommandInfo {
command: CliCommand::TotalSupply {
commitment_config: CommitmentConfig::recent(),
},
signers: vec![],
}
);
let test_transaction_count = test_commands
.clone()
.get_matches_from(vec!["test", "transaction-count"]);

View File

@@ -2,9 +2,7 @@ use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches
use console::style;
use solana_clap_utils::{
input_validators::is_url,
keypair::{check_for_usb, SKIP_SEED_PHRASE_VALIDATION_ARG},
offline::SIGN_ONLY_ARG,
input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
DisplayError,
};
use solana_cli::{
@@ -235,20 +233,12 @@ fn main() -> Result<(), Box<dyn error::Error>> {
)
.get_matches();
do_main(&matches, check_for_usb(std::env::args()))
.map_err(|err| DisplayError::new_as_boxed(err).into())
do_main(&matches).map_err(|err| DisplayError::new_as_boxed(err).into())
}
fn do_main(
matches: &ArgMatches<'_>,
need_wallet_manager: bool,
) -> Result<(), Box<dyn error::Error>> {
fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
if parse_settings(&matches)? {
let wallet_manager = if need_wallet_manager {
maybe_wallet_manager()?
} else {
None
};
let wallet_manager = maybe_wallet_manager()?;
let (mut config, signers) = parse_args(&matches, wallet_manager)?;
config.signers = signers.iter().map(|s| s.as_ref()).collect();

View File

@@ -21,8 +21,9 @@ use solana_sdk::{
},
pubkey::Pubkey,
system_instruction::{
advance_nonce_account, authorize_nonce_account, create_nonce_account,
create_nonce_account_with_seed, withdraw_nonce_account, NonceError, SystemError,
advance_nonce_account, authorize_nonce_account, create_address_with_seed,
create_nonce_account, create_nonce_account_with_seed, withdraw_nonce_account, NonceError,
SystemError,
},
system_program,
transaction::Transaction,
@@ -473,7 +474,7 @@ pub fn process_create_nonce_account(
) -> ProcessResult {
let nonce_account_pubkey = config.signers[nonce_account].pubkey();
let nonce_account_address = if let Some(seed) = seed.clone() {
Pubkey::create_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
create_address_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
} else {
nonce_account_pubkey
};

View File

@@ -17,7 +17,7 @@ use solana_sdk::{
account_utils::StateMut,
message::Message,
pubkey::Pubkey,
system_instruction::SystemError,
system_instruction::{create_address_with_seed, SystemError},
sysvar::{
stake_history::{self, StakeHistory},
Sysvar,
@@ -86,7 +86,7 @@ impl StakeSubCommands for App<'_, '_> {
.takes_value(true)
.validator(is_amount)
.required(true)
.help("The amount to send to the stake account, in SOL")
.help("The amount of send to the vote account, in SOL")
)
.arg(
Arg::with_name("custodian")
@@ -790,7 +790,7 @@ pub fn process_create_stake_account(
) -> ProcessResult {
let stake_account = config.signers[stake_account];
let stake_account_address = if let Some(seed) = seed {
Pubkey::create_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
create_address_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
} else {
stake_account.pubkey()
};
@@ -1108,7 +1108,7 @@ pub fn process_split_stake(
let stake_authority = config.signers[stake_authority];
let split_stake_account_address = if let Some(seed) = split_stake_account_seed {
Pubkey::create_with_seed(
create_address_with_seed(
&split_stake_account.pubkey(),
&seed,
&solana_stake_program::id(),

View File

@@ -1,15 +1,19 @@
use crate::cli::{
build_balance_message, check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult,
SignerIndex,
log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, CliSignerInfo,
ProcessResult, SignerIndex,
};
use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
use solana_clap_utils::{input_parsers::*, input_validators::*};
use solana_client::rpc_client::RpcClient;
use solana_remote_wallet::remote_wallet::RemoteWalletManager;
use solana_sdk::{
account::Account, commitment_config::CommitmentConfig, message::Message, pubkey::Pubkey,
system_instruction::SystemError, transaction::Transaction,
account::Account,
commitment_config::CommitmentConfig,
message::Message,
pubkey::Pubkey,
system_instruction::{create_address_with_seed, SystemError},
transaction::Transaction,
};
use solana_vote_program::{
vote_instruction::{self, withdraw, VoteError},
@@ -88,17 +92,9 @@ impl VoteSubCommands for App<'_, '_> {
.validator(is_valid_pubkey)
.help("Vote account in which to set the authorized voter"),
)
.arg(
Arg::with_name("authorized")
.index(2)
.value_name("AUTHORIZED_KEYPAIR")
.required(true)
.validator(is_valid_signer)
.help("Current authorized vote signer"),
)
.arg(
Arg::with_name("new_authorized_pubkey")
.index(3)
.index(2)
.value_name("AUTHORIZED_PUBKEY")
.takes_value(true)
.required(true)
@@ -118,17 +114,9 @@ impl VoteSubCommands for App<'_, '_> {
.validator(is_valid_pubkey)
.help("Vote account in which to set the authorized withdrawer"),
)
.arg(
Arg::with_name("authorized")
.index(2)
.value_name("AUTHORIZED_KEYPAIR")
.required(true)
.validator(is_valid_signer)
.help("Current authorized withdrawer"),
)
.arg(
Arg::with_name("new_authorized_pubkey")
.index(3)
.index(2)
.value_name("AUTHORIZED_PUBKEY")
.takes_value(true)
.required(true)
@@ -280,11 +268,10 @@ pub fn parse_vote_authorize(
pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
let new_authorized_pubkey =
pubkey_of_signer(matches, "new_authorized_pubkey", wallet_manager)?.unwrap();
let (authorized, _) = signer_of(matches, "authorized", wallet_manager)?;
let payer_provided = None;
let signer_info = generate_unique_signers(
vec![payer_provided, authorized],
let authorized_voter_provided = None;
let CliSignerInfo { signers } = generate_unique_signers(
vec![authorized_voter_provided],
matches,
default_signer_path,
wallet_manager,
@@ -296,7 +283,7 @@ pub fn parse_vote_authorize(
new_authorized_pubkey,
vote_authorize,
},
signers: signer_info.signers,
signers,
})
}
@@ -394,7 +381,7 @@ pub fn process_create_vote_account(
let vote_account = config.signers[1];
let vote_account_pubkey = vote_account.pubkey();
let vote_account_address = if let Some(seed) = seed {
Pubkey::create_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
create_address_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
} else {
vote_account_pubkey
};
@@ -472,24 +459,16 @@ pub fn process_vote_authorize(
new_authorized_pubkey: &Pubkey,
vote_authorize: VoteAuthorize,
) -> ProcessResult {
// If the `authorized_account` is also the fee payer, `config.signers` will only have one
// keypair in it
let authorized = if config.signers.len() == 2 {
config.signers[1]
} else {
config.signers[0]
};
check_unique_pubkeys(
(&authorized.pubkey(), "authorized_account".to_string()),
(vote_account_pubkey, "vote_account_pubkey".to_string()),
(new_authorized_pubkey, "new_authorized_pubkey".to_string()),
)?;
let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
let ixs = vec![vote_instruction::authorize(
vote_account_pubkey, // vote account to update
&authorized.pubkey(), // current authorized
new_authorized_pubkey, // new vote signer/withdrawer
vote_authorize, // vote or withdraw
vote_account_pubkey, // vote account to update
&config.signers[0].pubkey(), // current authorized voter
new_authorized_pubkey, // new vote signer/withdrawer
vote_authorize, // vote or withdraw
)];
let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
@@ -683,7 +662,6 @@ mod tests {
"test",
"vote-authorize-voter",
&pubkey_string,
&default_keypair_file,
&pubkey2_string,
]);
assert_eq!(
@@ -698,32 +676,6 @@ mod tests {
}
);
let authorized_keypair = Keypair::new();
let (authorized_keypair_file, mut tmp_file) = make_tmp_file();
write_keypair(&authorized_keypair, tmp_file.as_file_mut()).unwrap();
let test_authorize_voter = test_commands.clone().get_matches_from(vec![
"test",
"vote-authorize-voter",
&pubkey_string,
&authorized_keypair_file,
&pubkey2_string,
]);
assert_eq!(
parse_command(&test_authorize_voter, &default_keypair_file, None).unwrap(),
CliCommandInfo {
command: CliCommand::VoteAuthorize {
vote_account_pubkey: pubkey,
new_authorized_pubkey: pubkey2,
vote_authorize: VoteAuthorize::Voter
},
signers: vec![
read_keypair_file(&default_keypair_file).unwrap().into(),
read_keypair_file(&authorized_keypair_file).unwrap().into(),
],
}
);
let (keypair_file, mut tmp_file) = make_tmp_file();
let keypair = Keypair::new();
write_keypair(&keypair, tmp_file.as_file_mut()).unwrap();

View File

@@ -13,6 +13,7 @@ use solana_sdk::{
hash::Hash,
pubkey::Pubkey,
signature::{keypair_from_seed, Keypair, Signer},
system_instruction::create_address_with_seed,
system_program,
};
use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
@@ -129,7 +130,7 @@ fn full_battery_tests(
config_nonce.signers = vec![&nonce_keypair];
let nonce_account = if let Some(seed) = seed.as_ref() {
Pubkey::create_with_seed(
create_address_with_seed(
&config_nonce.signers[0].pubkey(),
seed,
&system_program::id(),
@@ -265,7 +266,6 @@ fn test_create_account_with_seed() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: 1,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();
@@ -301,7 +301,7 @@ fn test_create_account_with_seed() {
let authority_pubkey = offline_nonce_authority_signer.pubkey();
let seed = authority_pubkey.to_string()[0..32].to_string();
let nonce_address =
Pubkey::create_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
create_address_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
check_balance(0, &rpc_client, &nonce_address);
let mut creator_config = CliConfig::default();

View File

@@ -14,6 +14,7 @@ use solana_sdk::{
nonce::State as NonceState,
pubkey::Pubkey,
signature::{keypair_from_seed, Keypair, Signer},
system_instruction::create_address_with_seed,
};
use solana_stake_program::{
stake_instruction::LockupArgs,
@@ -159,7 +160,7 @@ fn test_seed_stake_delegation_and_deactivation() {
.unwrap();
check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
let stake_address = Pubkey::create_with_seed(
let stake_address = create_address_with_seed(
&config_validator.signers[0].pubkey(),
"hi there",
&solana_stake_program::id(),
@@ -811,7 +812,6 @@ fn test_stake_authorize_with_fee_payer() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: SIG_FEE,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();
run_local_faucet(alice, sender, None);
@@ -939,7 +939,6 @@ fn test_stake_split() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: 1,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();
run_local_faucet(alice, sender, None);
@@ -1087,7 +1086,6 @@ fn test_stake_set_lockup() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: 1,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();
run_local_faucet(alice, sender, None);
@@ -1525,7 +1523,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
};
process_command(&config).unwrap();
let seed_address =
Pubkey::create_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
create_address_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
check_balance(50_000, &rpc_client, &seed_address);
server.close().unwrap();

View File

@@ -40,7 +40,6 @@ fn test_transfer() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: 1,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();
@@ -236,7 +235,6 @@ fn test_transfer_multisession_signing() {
} = TestValidator::run_with_options(TestValidatorOptions {
fees: 1,
bootstrap_validator_lamports: 42_000,
..TestValidatorOptions::default()
});
let (sender, receiver) = channel();

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-client"
version = "1.1.3"
version = "1.0.16"
description = "Solana Client"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -14,15 +14,15 @@ bs58 = "0.3.0"
indicatif = "0.14.0"
jsonrpc-core = "14.0.5"
log = "0.4.8"
rayon = "1.3.0"
reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
serde = "1.0.105"
rayon = "1.2.0"
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
serde = "1.0.104"
serde_derive = "1.0.103"
serde_json = "1.0.48"
solana-transaction-status = { path = "../transaction-status", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-vote-program = { path = "../programs/vote", version = "1.1.3" }
serde_json = "1.0.46"
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
solana-transaction-status = { path = "../transaction-status", version = "1.0.16" }
solana-vote-program = { path = "../programs/vote", version = "1.0.16" }
thiserror = "1.0"
tungstenite = "0.10.1"
url = "2.1.1"
@@ -31,4 +31,4 @@ url = "2.1.1"
assert_matches = "1.3.0"
jsonrpc-core = "14.0.5"
jsonrpc-http-server = "14.0.6"
solana-logger = { path = "../logger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.0.16" }

View File

@@ -8,8 +8,7 @@ use serde_json::{Number, Value};
use solana_sdk::{
fee_calculator::{FeeCalculator, FeeRateGovernor},
instruction::InstructionError,
signature::Signature,
transaction::{self, Transaction, TransactionError},
transaction::{self, TransactionError},
};
use solana_transaction_status::TransactionStatus;
use std::{collections::HashMap, sync::RwLock};
@@ -41,7 +40,7 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
fn send(
&self,
request: &RpcRequest,
params: serde_json::Value,
_params: serde_json::Value,
_retries: usize,
) -> Result<serde_json::Value> {
if let Some(value) = self.mocks.write().unwrap().remove(request) {
@@ -106,17 +105,7 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
}
RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
RpcRequest::GetSlot => Value::Number(Number::from(0)),
RpcRequest::SendTransaction => {
let signature = if self.url == "malicious" {
Signature::new(&[8; 64]).to_string()
} else {
let tx_str = params.as_array().unwrap()[0].as_str().unwrap().to_string();
let data = bs58::decode(tx_str).into_vec().unwrap();
let tx: Transaction = bincode::deserialize(&data).unwrap();
tx.signatures[0].to_string()
};
Value::String(signature)
}
RpcRequest::SendTransaction => Value::String(SIGNATURE.to_string()),
RpcRequest::GetMinimumBalanceForRentExemption => Value::Number(Number::from(1234)),
_ => Value::Null,
};

View File

@@ -4,7 +4,11 @@ use crate::{
mock_rpc_client_request::{MockRpcClientRequest, Mocks},
rpc_client_request::RpcClientRequest,
rpc_request::{RpcError, RpcRequest},
rpc_response::*,
rpc_response::{
Response, RpcAccount, RpcBlockhashFeeCalculator, RpcContactInfo, RpcEpochInfo,
RpcFeeCalculator, RpcFeeRateGovernor, RpcIdentity, RpcKeyedAccount, RpcLeaderSchedule,
RpcResult, RpcVersionInfo, RpcVoteAccountStatus,
},
};
use bincode::serialize;
use indicatif::{ProgressBar, ProgressStyle};
@@ -99,24 +103,9 @@ impl RpcClient {
None => {
Err(RpcError::ForUser("Received result of an unexpected type".to_string()).into())
}
Some(signature_base58_str) => {
let signature = signature_base58_str.parse::<Signature>().map_err(|err| {
Into::<ClientError>::into(RpcError::ParseError(err.to_string()))
})?;
// A mismatching RPC response signature indicates an issue with the RPC node, and
// should not be passed along to confirmation methods. The transaction may or may
// not have been submitted to the cluster, so callers should verify the success of
// the correct transaction signature independently.
if signature != transaction.signatures[0] {
Err(RpcError::RpcRequestError(format!(
"RPC node returned mismatched signature {:?}, expected {:?}",
signature, transaction.signatures[0]
))
.into())
} else {
Ok(transaction.signatures[0])
}
}
Some(signature_base58_str) => signature_base58_str
.parse::<Signature>()
.map_err(|err| RpcError::ParseError(err.to_string()).into()),
}
}
@@ -175,23 +164,6 @@ impl RpcClient {
.map_err(|err| ClientError::new_with_command(err.into(), "GetSlot"))
}
pub fn total_supply(&self) -> ClientResult<u64> {
self.total_supply_with_commitment(CommitmentConfig::default())
}
pub fn total_supply_with_commitment(
&self,
commitment_config: CommitmentConfig,
) -> ClientResult<u64> {
let response = self
.client
.send(&RpcRequest::GetTotalSupply, json!([commitment_config]), 0)
.map_err(|err| err.into_with_command("GetTotalSupply"))?;
serde_json::from_value(response)
.map_err(|err| ClientError::new_with_command(err.into(), "GetTotalSupply"))
}
pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
self.get_vote_accounts_with_commitment(CommitmentConfig::default())
}
@@ -971,7 +943,7 @@ impl RpcClient {
.send(
&RpcRequest::GetSignatureStatuses,
json!([[signature.to_string()]]),
5,
1,
)
.map_err(|err| err.into_with_command("GetSignatureStatuses"))?;
let result: Response<Vec<Option<TransactionStatus>>> = serde_json::from_value(response)
@@ -1109,7 +1081,10 @@ pub fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::{client_error::ClientErrorKind, mock_rpc_client_request::PUBKEY};
use crate::{
client_error::ClientErrorKind,
mock_rpc_client_request::{PUBKEY, SIGNATURE},
};
use assert_matches::assert_matches;
use jsonrpc_core::{Error, IoHandler, Params};
use jsonrpc_http_server::{AccessControlAllowOrigin, DomainsValidation, ServerBuilder};
@@ -1222,17 +1197,12 @@ mod tests {
let tx = system_transaction::transfer(&key, &to, 50, blockhash);
let signature = rpc_client.send_transaction(&tx);
assert_eq!(signature.unwrap(), tx.signatures[0]);
assert_eq!(signature.unwrap(), SIGNATURE.parse().unwrap());
let rpc_client = RpcClient::new_mock("fails".to_string());
let signature = rpc_client.send_transaction(&tx);
assert!(signature.is_err());
// Test bad signature returned from rpc node
let rpc_client = RpcClient::new_mock("malicious".to_string());
let signature = rpc_client.send_transaction(&tx);
assert!(signature.is_err());
}
#[test]
fn test_get_recent_blockhash() {

View File

@@ -28,7 +28,6 @@ pub enum RpcRequest {
GetStorageTurnRate,
GetSlotsPerSegment,
GetStoragePubkeysForSlot,
GetTotalSupply,
GetTransactionCount,
GetVersion,
GetVoteAccounts,
@@ -69,7 +68,6 @@ impl RpcRequest {
RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
RpcRequest::GetStoragePubkeysForSlot => "getStoragePubkeysForSlot",
RpcRequest::GetTotalSupply => "getTotalSupply",
RpcRequest::GetTransactionCount => "getTransactionCount",
RpcRequest::GetVersion => "getVersion",
RpcRequest::GetVoteAccounts => "getVoteAccounts",

View File

@@ -1,7 +1,7 @@
[package]
name = "solana-core"
description = "Blockchain, Rebuilt for Scale"
version = "1.1.3"
version = "1.0.16"
documentation = "https://docs.rs/solana"
homepage = "https://solana.com/"
readme = "../README.md"
@@ -17,65 +17,66 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
bincode = "1.2.1"
bv = { version = "0.11.1", features = ["serde"] }
bs58 = "0.3.0"
byteorder = "1.3.4"
chrono = { version = "0.4.11", features = ["serde"] }
byteorder = "1.3.2"
chrono = { version = "0.4.10", features = ["serde"] }
compression = "0.1.5"
core_affinity = "0.5.10"
crossbeam-channel = "0.4"
crossbeam-channel = "0.3"
fs_extra = "1.1.0"
flate2 = "1.0"
indexmap = "1.3"
itertools = "0.9.0"
itertools = "0.8.2"
jsonrpc-core = "14.0.5"
jsonrpc-core-client = { version = "14.0.5", features = ["ws"] }
jsonrpc-derive = "14.0.5"
jsonrpc-http-server = "14.0.6"
jsonrpc-pubsub = "14.0.6"
jsonrpc-ws-server = "14.0.6"
libc = "0.2.66"
log = "0.4.8"
nix = "0.17.0"
num_cpus = "1.0.0"
num-traits = "0.2"
rand = "0.6.5"
rand_chacha = "0.1.1"
rayon = "1.3.0"
regex = "1.3.6"
serde = "1.0.105"
rayon = "1.2.0"
regex = "1.3.4"
serde = "1.0.104"
serde_derive = "1.0.103"
serde_json = "1.0.48"
solana-budget-program = { path = "../programs/budget", version = "1.1.3" }
solana-clap-utils = { path = "../clap-utils", version = "1.1.3" }
solana-client = { path = "../client", version = "1.1.3" }
solana-transaction-status = { path = "../transaction-status", version = "1.1.3" }
solana-faucet = { path = "../faucet", version = "1.1.3" }
serde_json = "1.0.46"
solana-budget-program = { path = "../programs/budget", version = "1.0.16" }
solana-clap-utils = { path = "../clap-utils", version = "1.0.16" }
solana-client = { path = "../client", version = "1.0.16" }
solana-transaction-status = { path = "../transaction-status", version = "1.0.16" }
solana-faucet = { path = "../faucet", version = "1.0.16" }
ed25519-dalek = "=1.0.0-pre.1"
solana-ledger = { path = "../ledger", version = "1.1.3" }
solana-logger = { path = "../logger", version = "1.1.3" }
solana-merkle-tree = { path = "../merkle-tree", version = "1.1.3" }
solana-metrics = { path = "../metrics", version = "1.1.3" }
solana-measure = { path = "../measure", version = "1.1.3" }
solana-net-utils = { path = "../net-utils", version = "1.1.3" }
solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.3" }
solana-perf = { path = "../perf", version = "1.1.3" }
solana-runtime = { path = "../runtime", version = "1.1.3" }
solana-sdk = { path = "../sdk", version = "1.1.3" }
solana-stake-program = { path = "../programs/stake", version = "1.1.3" }
solana-storage-program = { path = "../programs/storage", version = "1.1.3" }
solana-streamer = { path = "../streamer", version = "1.1.3" }
solana-vote-program = { path = "../programs/vote", version = "1.1.3" }
solana-vote-signer = { path = "../vote-signer", version = "1.1.3" }
solana-sys-tuner = { path = "../sys-tuner", version = "1.1.3" }
solana-ledger = { path = "../ledger", version = "1.0.16" }
solana-logger = { path = "../logger", version = "1.0.16" }
solana-merkle-tree = { path = "../merkle-tree", version = "1.0.16" }
solana-metrics = { path = "../metrics", version = "1.0.16" }
solana-measure = { path = "../measure", version = "1.0.16" }
solana-net-utils = { path = "../net-utils", version = "1.0.16" }
solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.16" }
solana-perf = { path = "../perf", version = "1.0.16" }
solana-runtime = { path = "../runtime", version = "1.0.16" }
solana-sdk = { path = "../sdk", version = "1.0.16" }
solana-stake-program = { path = "../programs/stake", version = "1.0.16" }
solana-storage-program = { path = "../programs/storage", version = "1.0.16" }
solana-vote-program = { path = "../programs/vote", version = "1.0.16" }
solana-vote-signer = { path = "../vote-signer", version = "1.0.16" }
solana-sys-tuner = { path = "../sys-tuner", version = "1.0.16" }
tempfile = "3.1.0"
thiserror = "1.0"
tokio = "0.1"
tokio-codec = "0.1"
tokio-fs = "0.1"
tokio-io = "0.1"
solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.3" }
solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.16" }
trees = "0.2.1"
[dev-dependencies]
matches = "0.1.6"
reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
serial_test = "0.4.0"
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
serial_test = "0.3.2"
serial_test_derive = "0.4.0"
systemstat = "0.1.5"

View File

@@ -9,12 +9,12 @@ use rayon::prelude::*;
use solana_core::banking_stage::{create_test_recorder, BankingStage};
use solana_core::cluster_info::ClusterInfo;
use solana_core::cluster_info::Node;
use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_core::packet::to_packets_chunked;
use solana_core::poh_recorder::WorkingBankEntry;
use solana_ledger::blockstore_processor::process_entries;
use solana_ledger::entry::{next_hash, Entry};
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
use solana_perf::packet::to_packets_chunked;
use solana_perf::test_tx::test_tx;
use solana_runtime::bank::Bank;
use solana_sdk::genesis_config::GenesisConfig;

View File

@@ -3,14 +3,13 @@
extern crate test;
use rand::{thread_rng, Rng};
use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
use solana_core::cluster_info::{ClusterInfo, Node};
use solana_core::contact_info::ContactInfo;
use solana_ledger::shred::Shred;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::timing::timestamp;
use std::sync::RwLock;
use std::{collections::HashMap, net::UdpSocket, sync::Arc, time::Instant};
use std::collections::HashMap;
use std::net::UdpSocket;
use std::sync::Arc;
use test::Bencher;
#[bench]
@@ -21,8 +20,10 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info.clone());
let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
const SHRED_SIZE: usize = 1024;
const NUM_SHREDS: usize = 32;
let shreds = vec![Shred::new_empty_data_shred(); NUM_SHREDS];
let shreds = vec![vec![0; SHRED_SIZE]; NUM_SHREDS];
let seeds = vec![[0u8; 32]; NUM_SHREDS];
let mut stakes = HashMap::new();
const NUM_PEERS: usize = 200;
for _ in 0..NUM_PEERS {
@@ -32,19 +33,10 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);
}
let stakes = Arc::new(stakes);
let cluster_info = Arc::new(RwLock::new(cluster_info));
let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(stakes.clone()));
let shreds = Arc::new(shreds);
bencher.iter(move || {
let shreds = shreds.clone();
broadcast_shreds(
&socket,
&shreds,
&peers_and_stakes,
&peers,
&mut Instant::now(),
&mut 0,
)
.unwrap();
cluster_info
.broadcast_shreds(&socket, shreds, &seeds, Some(stakes.clone()))
.unwrap();
});
}

View File

@@ -6,12 +6,12 @@ extern crate test;
use log::*;
use solana_core::cluster_info::{ClusterInfo, Node};
use solana_core::contact_info::ContactInfo;
use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_core::packet::to_packets_chunked;
use solana_core::retransmit_stage::retransmitter;
use solana_ledger::bank_forks::BankForks;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
use solana_measure::measure::Measure;
use solana_perf::packet::to_packets_chunked;
use solana_perf::test_tx::test_tx;
use solana_runtime::bank::Bank;
use solana_sdk::pubkey::Pubkey;

View File

@@ -6,9 +6,9 @@ extern crate test;
use crossbeam_channel::unbounded;
use log::*;
use rand::{thread_rng, Rng};
use solana_core::packet::to_packets_chunked;
use solana_core::sigverify::TransactionSigVerifier;
use solana_core::sigverify_stage::SigVerifyStage;
use solana_perf::packet::to_packets_chunked;
use solana_perf::test_tx::test_tx;
use solana_sdk::hash::Hash;
use solana_sdk::signature::{Keypair, Signer};

View File

@@ -1,45 +0,0 @@
// Service to clean up dead slots in accounts_db
//
// This can be expensive since we have to walk the append vecs being cleaned up.
use solana_ledger::bank_forks::BankForks;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc, RwLock,
};
use std::thread::{self, sleep, Builder, JoinHandle};
use std::time::Duration;
pub struct AccountsBackgroundService {
t_background: JoinHandle<()>,
}
const INTERVAL_MS: u64 = 100;
impl AccountsBackgroundService {
pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
info!("AccountsBackgroundService active");
let exit = exit.clone();
let t_background = Builder::new()
.name("solana-accounts-background".to_string())
.spawn(move || loop {
if exit.load(Ordering::Relaxed) {
break;
}
let bank = bank_forks.read().unwrap().working_bank();
bank.process_dead_slots();
// Currently, given INTERVAL_MS, we process 1 slot/100 ms
bank.process_stale_slot();
sleep(Duration::from_millis(INTERVAL_MS));
})
.unwrap();
Self { t_background }
}
pub fn join(self) -> thread::Result<()> {
self.t_background.join()
}
}

View File

@@ -3,6 +3,7 @@
//! can do its processing in parallel with signature verification on the GPU.
use crate::{
cluster_info::ClusterInfo,
packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
poh_service::PohService,
};
@@ -16,11 +17,7 @@ use solana_ledger::{
};
use solana_measure::{measure::Measure, thread_mem_usage};
use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
use solana_perf::{
cuda_runtime::PinnedVec,
packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
perf_libs,
};
use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
use solana_runtime::{
accounts_db::ErrorCounters,
bank::{Bank, TransactionBalancesSet, TransactionProcessResult},
@@ -1012,7 +1009,10 @@ pub fn create_test_recorder(
mod tests {
use super::*;
use crate::{
cluster_info::Node, poh_recorder::WorkingBank,
cluster_info::Node,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
packet::to_packets,
poh_recorder::WorkingBank,
transaction_status_service::TransactionStatusService,
};
use crossbeam_channel::unbounded;
@@ -1020,10 +1020,8 @@ mod tests {
use solana_ledger::{
blockstore::entries_to_test_shreds,
entry::{next_entry, Entry, EntrySlice},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
};
use solana_perf::packet::to_packets;
use solana_runtime::bank::HashAgeKind;
use solana_sdk::{
instruction::InstructionError,

283
core/src/blockstream.rs Normal file
View File

@@ -0,0 +1,283 @@
//! The `blockstream` module provides a method for streaming entries out via a
//! local unix socket, to provide client services such as a block explorer with
//! real-time access to entries.
use bincode::serialize;
use chrono::{SecondsFormat, Utc};
use serde_json::json;
use solana_ledger::entry::Entry;
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
use std::cell::RefCell;
use std::io::Result;
use std::path::{Path, PathBuf};
pub trait EntryWriter: std::fmt::Debug {
fn write(&self, payload: String) -> Result<()>;
}
#[derive(Debug, Default)]
pub struct EntryVec {
values: RefCell<Vec<String>>,
}
impl EntryWriter for EntryVec {
fn write(&self, payload: String) -> Result<()> {
self.values.borrow_mut().push(payload);
Ok(())
}
}
impl EntryVec {
pub fn new() -> Self {
EntryVec {
values: RefCell::new(Vec::new()),
}
}
pub fn entries(&self) -> Vec<String> {
self.values.borrow().clone()
}
}
#[derive(Debug)]
pub struct EntrySocket {
unix_socket: PathBuf,
}
impl EntryWriter for EntrySocket {
#[cfg(not(windows))]
fn write(&self, payload: String) -> Result<()> {
use std::io::prelude::*;
use std::net::Shutdown;
use std::os::unix::net::UnixStream;
const MESSAGE_TERMINATOR: &str = "\n";
let mut socket = UnixStream::connect(&self.unix_socket)?;
socket.write_all(payload.as_bytes())?;
socket.write_all(MESSAGE_TERMINATOR.as_bytes())?;
socket.shutdown(Shutdown::Write)?;
Ok(())
}
#[cfg(windows)]
fn write(&self, _payload: String) -> Result<()> {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
"EntryWriter::write() not implemented for windows",
))
}
}
pub trait BlockstreamEvents {
fn emit_entry_event(
&self,
slot: Slot,
tick_height: u64,
leader_pubkey: &Pubkey,
entries: &Entry,
) -> Result<()>;
fn emit_block_event(
&self,
slot: Slot,
tick_height: u64,
leader_pubkey: &Pubkey,
blockhash: Hash,
) -> Result<()>;
}
#[derive(Debug)]
pub struct Blockstream<T: EntryWriter> {
pub output: T,
}
impl<T> BlockstreamEvents for Blockstream<T>
where
T: EntryWriter,
{
fn emit_entry_event(
&self,
slot: Slot,
tick_height: u64,
leader_pubkey: &Pubkey,
entry: &Entry,
) -> Result<()> {
let transactions: Vec<Vec<u8>> = serialize_transactions(entry);
let stream_entry = json!({
"num_hashes": entry.num_hashes,
"hash": entry.hash,
"transactions": transactions
});
let json_entry = serde_json::to_string(&stream_entry)?;
let payload = format!(
r#"{{"dt":"{}","t":"entry","s":{},"h":{},"l":"{:?}","entry":{}}}"#,
Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
slot,
tick_height,
leader_pubkey,
json_entry,
);
self.output.write(payload)?;
Ok(())
}
fn emit_block_event(
&self,
slot: Slot,
tick_height: u64,
leader_pubkey: &Pubkey,
blockhash: Hash,
) -> Result<()> {
let payload = format!(
r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":"{:?}","hash":"{:?}"}}"#,
Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
slot,
tick_height,
leader_pubkey,
blockhash,
);
self.output.write(payload)?;
Ok(())
}
}
pub type SocketBlockstream = Blockstream<EntrySocket>;
impl SocketBlockstream {
pub fn new(unix_socket: &Path) -> Self {
Blockstream {
output: EntrySocket {
unix_socket: unix_socket.to_path_buf(),
},
}
}
}
pub type MockBlockstream = Blockstream<EntryVec>;
impl MockBlockstream {
pub fn new(_: &Path) -> Self {
Blockstream {
output: EntryVec::new(),
}
}
pub fn entries(&self) -> Vec<String> {
self.output.entries()
}
}
fn serialize_transactions(entry: &Entry) -> Vec<Vec<u8>> {
entry
.transactions
.iter()
.map(|tx| serialize(&tx).unwrap())
.collect()
}
#[cfg(test)]
mod test {
use super::*;
use chrono::{DateTime, FixedOffset};
use serde_json::Value;
use solana_sdk::hash::Hash;
use solana_sdk::signature::{Keypair, Signer};
use solana_sdk::system_transaction;
use std::collections::HashSet;
use std::path::PathBuf;
#[test]
fn test_serialize_transactions() {
let entry = Entry::new(&Hash::default(), 1, vec![]);
let empty_vec: Vec<Vec<u8>> = vec![];
assert_eq!(serialize_transactions(&entry), empty_vec);
let keypair0 = Keypair::new();
let keypair1 = Keypair::new();
let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
let tx1 = system_transaction::transfer(&keypair1, &keypair0.pubkey(), 2, Hash::default());
let serialized_tx0 = serialize(&tx0).unwrap();
let serialized_tx1 = serialize(&tx1).unwrap();
let entry = Entry::new(&Hash::default(), 1, vec![tx0, tx1]);
assert_eq!(
serialize_transactions(&entry),
vec![serialized_tx0, serialized_tx1]
);
}
#[test]
fn test_blockstream() -> () {
let blockstream = MockBlockstream::new(&PathBuf::from("test_stream"));
let ticks_per_slot = 5;
let mut blockhash = Hash::default();
let mut entries = Vec::new();
let mut expected_entries = Vec::new();
let tick_height_initial = 1;
let tick_height_final = tick_height_initial + ticks_per_slot + 2;
let mut curr_slot = 0;
let leader_pubkey = Pubkey::new_rand();
for tick_height in tick_height_initial..=tick_height_final {
if tick_height == 5 {
blockstream
.emit_block_event(curr_slot, tick_height, &leader_pubkey, blockhash)
.unwrap();
curr_slot += 1;
}
let entry = Entry::new(&mut blockhash, 1, vec![]); // just ticks
blockhash = entry.hash;
blockstream
.emit_entry_event(curr_slot, tick_height, &leader_pubkey, &entry)
.unwrap();
expected_entries.push(entry.clone());
entries.push(entry);
}
assert_eq!(
blockstream.entries().len() as u64,
// one entry per tick (1..=N+2) is +3, plus one block
ticks_per_slot + 3 + 1
);
let mut j = 0;
let mut matched_entries = 0;
let mut matched_slots = HashSet::new();
let mut matched_blocks = HashSet::new();
for item in blockstream.entries() {
let json: Value = serde_json::from_str(&item).unwrap();
let dt_str = json["dt"].as_str().unwrap();
// Ensure `ts` field parses as valid DateTime
let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
let item_type = json["t"].as_str().unwrap();
match item_type {
"block" => {
let hash = json["hash"].to_string();
matched_blocks.insert(hash);
}
"entry" => {
let slot = json["s"].as_u64().unwrap();
matched_slots.insert(slot);
let entry_obj = json["entry"].clone();
let entry: Entry = serde_json::from_value(entry_obj).unwrap();
assert_eq!(entry, expected_entries[j]);
matched_entries += 1;
j += 1;
}
_ => {
assert!(false, "unknown item type {}", item);
}
}
}
assert_eq!(matched_entries, expected_entries.len());
assert_eq!(matched_slots.len(), 2);
assert_eq!(matched_blocks.len(), 1);
}
}

View File

@@ -0,0 +1,228 @@
//! The `blockstream_service` implements optional streaming of entries and block metadata
//! using the `blockstream` module, providing client services such as a block explorer with
//! real-time access to entries.
use crate::blockstream::BlockstreamEvents;
#[cfg(test)]
use crate::blockstream::MockBlockstream as Blockstream;
#[cfg(not(test))]
use crate::blockstream::SocketBlockstream as Blockstream;
use crate::result::{Error, Result};
use solana_ledger::blockstore::Blockstore;
use solana_sdk::pubkey::Pubkey;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, RecvTimeoutError};
use std::sync::Arc;
use std::thread::{self, Builder, JoinHandle};
use std::time::Duration;
pub struct BlockstreamService {
t_blockstream: JoinHandle<()>,
}
impl BlockstreamService {
#[allow(clippy::new_ret_no_self)]
pub fn new(
slot_full_receiver: Receiver<(u64, Pubkey)>,
blockstore: Arc<Blockstore>,
unix_socket: &Path,
exit: &Arc<AtomicBool>,
) -> Self {
let mut blockstream = Blockstream::new(unix_socket);
let exit = exit.clone();
let t_blockstream = Builder::new()
.name("solana-blockstream".to_string())
.spawn(move || loop {
if exit.load(Ordering::Relaxed) {
break;
}
if let Err(e) =
Self::process_entries(&slot_full_receiver, &blockstore, &mut blockstream)
{
match e {
Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
_ => info!("Error from process_entries: {:?}", e),
}
}
})
.unwrap();
Self { t_blockstream }
}
fn process_entries(
slot_full_receiver: &Receiver<(u64, Pubkey)>,
blockstore: &Arc<Blockstore>,
blockstream: &mut Blockstream,
) -> Result<()> {
let timeout = Duration::new(1, 0);
let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
// Slot might not exist due to LedgerCleanupService, check first
let blockstore_meta = blockstore.meta(slot).unwrap();
if let Some(blockstore_meta) = blockstore_meta {
// Return error to main loop. Thread won't exit, will just log the error
let entries = blockstore.get_slot_entries(slot, 0, None)?;
let _parent_slot = if slot == 0 {
None
} else {
Some(blockstore_meta.parent_slot)
};
let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
let mut tick_height = ticks_per_slot * slot;
for (i, entry) in entries.iter().enumerate() {
if entry.is_tick() {
tick_height += 1;
}
blockstream
.emit_entry_event(slot, tick_height, &slot_leader, &entry)
.unwrap_or_else(|e| {
debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
});
if i == entries.len() - 1 {
blockstream
.emit_block_event(slot, tick_height, &slot_leader, entry.hash)
.unwrap_or_else(|e| {
debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
});
}
}
}
Ok(())
}
pub fn join(self) -> thread::Result<()> {
self.t_blockstream.join()
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use bincode::{deserialize, serialize};
use chrono::{DateTime, FixedOffset};
use serde_json::Value;
use solana_ledger::create_new_tmp_ledger;
use solana_ledger::entry::{create_ticks, Entry};
use solana_sdk::hash::Hash;
use solana_sdk::signature::{Keypair, Signer};
use solana_sdk::system_transaction;
use std::path::PathBuf;
use std::sync::mpsc::channel;
#[test]
fn test_blockstream_service_process_entries() {
let ticks_per_slot = 5;
let leader_pubkey = Pubkey::new_rand();
// Set up genesis config and blockstore
let GenesisConfigInfo {
mut genesis_config, ..
} = create_genesis_config(1000);
genesis_config.ticks_per_slot = ticks_per_slot;
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
let blockstore = Blockstore::open(&ledger_path).unwrap();
// Set up blockstream
let mut blockstream = Blockstream::new(&PathBuf::from("test_stream"));
// Set up dummy channel to receive a full-slot notification
let (slot_full_sender, slot_full_receiver) = channel();
// Create entries - 4 ticks + 1 populated entry + 1 tick
let mut entries = create_ticks(4, 0, Hash::default());
let keypair = Keypair::new();
let mut blockhash = entries[3].hash;
let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default());
let entry = Entry::new(&mut blockhash, 1, vec![tx]);
blockhash = entry.hash;
entries.push(entry);
let final_tick = create_ticks(1, 0, blockhash);
entries.extend_from_slice(&final_tick);
let expected_entries = entries.clone();
let expected_tick_heights = [6, 7, 8, 9, 9, 10];
blockstore
.write_entries(
1,
0,
0,
ticks_per_slot,
None,
true,
&Arc::new(Keypair::new()),
entries,
0,
)
.unwrap();
slot_full_sender.send((1, leader_pubkey)).unwrap();
BlockstreamService::process_entries(
&slot_full_receiver,
&Arc::new(blockstore),
&mut blockstream,
)
.unwrap();
assert_eq!(blockstream.entries().len(), 7);
let (entry_events, block_events): (Vec<Value>, Vec<Value>) = blockstream
.entries()
.iter()
.map(|item| {
let json: Value = serde_json::from_str(&item).unwrap();
let dt_str = json["dt"].as_str().unwrap();
// Ensure `ts` field parses as valid DateTime
let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
json
})
.partition(|json| {
let item_type = json["t"].as_str().unwrap();
item_type == "entry"
});
for (i, json) in entry_events.iter().enumerate() {
let height = json["h"].as_u64().unwrap();
assert_eq!(height, expected_tick_heights[i]);
let entry_obj = json["entry"].clone();
let tx = entry_obj["transactions"].as_array().unwrap();
let entry: Entry;
if tx.len() == 0 {
entry = serde_json::from_value(entry_obj).unwrap();
} else {
let entry_json = entry_obj.as_object().unwrap();
entry = Entry {
num_hashes: entry_json.get("num_hashes").unwrap().as_u64().unwrap(),
hash: serde_json::from_value(entry_json.get("hash").unwrap().clone()).unwrap(),
transactions: entry_json
.get("transactions")
.unwrap()
.as_array()
.unwrap()
.into_iter()
.enumerate()
.map(|(j, tx)| {
let tx_vec: Vec<u8> = serde_json::from_value(tx.clone()).unwrap();
// Check explicitly that transaction matches bincode-serialized format
assert_eq!(
tx_vec,
serialize(&expected_entries[i].transactions[j]).unwrap()
);
deserialize(&tx_vec).unwrap()
})
.collect(),
};
}
assert_eq!(entry, expected_entries[i]);
}
for json in block_events {
let slot = json["s"].as_u64().unwrap();
assert_eq!(1, slot);
let height = json["h"].as_u64().unwrap();
assert_eq!(2 * ticks_per_slot, height);
}
}
}

View File

@@ -1,48 +1,30 @@
//! A stage to broadcast data from a leader node to validators
use self::{
broadcast_fake_shreds_run::BroadcastFakeShredsRun,
fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
standard_broadcast_run::StandardBroadcastRun,
};
use crate::contact_info::ContactInfo;
use crate::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
use crate::weighted_shuffle::weighted_best;
use crate::{
cluster_info::{ClusterInfo, ClusterInfoError},
poh_recorder::WorkingBankEntry,
result::{Error, Result},
};
use crossbeam_channel::{
Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
Sender as CrossbeamSender,
};
use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
use solana_measure::measure::Measure;
use self::broadcast_fake_shreds_run::BroadcastFakeShredsRun;
use self::fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun;
use self::standard_broadcast_run::StandardBroadcastRun;
use crate::cluster_info::{ClusterInfo, ClusterInfoError};
use crate::poh_recorder::WorkingBankEntry;
use crate::result::{Error, Result};
use solana_ledger::blockstore::Blockstore;
use solana_ledger::shred::Shred;
use solana_ledger::staking_utils;
use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
use solana_runtime::bank::Bank;
use solana_sdk::timing::duration_as_s;
use solana_sdk::timing::timestamp;
use solana_sdk::{clock::Slot, pubkey::Pubkey};
use solana_streamer::sendmmsg::send_mmsg;
use std::{
collections::HashMap,
net::UdpSocket,
sync::atomic::{AtomicBool, Ordering},
sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
sync::{Arc, Mutex, RwLock},
thread::{self, Builder, JoinHandle},
time::{Duration, Instant},
};
use solana_sdk::pubkey::Pubkey;
use std::collections::HashMap;
use std::net::UdpSocket;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
use std::sync::{Arc, Mutex, RwLock};
use std::thread::{self, Builder, JoinHandle};
use std::time::Instant;
pub const NUM_INSERT_THREADS: usize = 2;
mod broadcast_fake_shreds_run;
pub(crate) mod broadcast_utils;
mod fail_entry_verification_broadcast_run;
mod standard_broadcast_run;
pub const NUM_INSERT_THREADS: usize = 2;
pub type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
pub type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum BroadcastStageReturnType {
ChannelDisconnected,
@@ -61,7 +43,6 @@ impl BroadcastStageType {
sock: Vec<UdpSocket>,
cluster_info: Arc<RwLock<ClusterInfo>>,
receiver: Receiver<WorkingBankEntry>,
retransmit_slots_receiver: RetransmitSlotsReceiver,
exit_sender: &Arc<AtomicBool>,
blockstore: &Arc<Blockstore>,
shred_version: u16,
@@ -72,7 +53,6 @@ impl BroadcastStageType {
sock,
cluster_info,
receiver,
retransmit_slots_receiver,
exit_sender,
blockstore,
StandardBroadcastRun::new(keypair, shred_version),
@@ -82,7 +62,6 @@ impl BroadcastStageType {
sock,
cluster_info,
receiver,
retransmit_slots_receiver,
exit_sender,
blockstore,
FailEntryVerificationBroadcastRun::new(keypair, shred_version),
@@ -92,7 +71,6 @@ impl BroadcastStageType {
sock,
cluster_info,
receiver,
retransmit_slots_receiver,
exit_sender,
blockstore,
BroadcastFakeShredsRun::new(keypair, 0, shred_version),
@@ -101,7 +79,7 @@ impl BroadcastStageType {
}
}
pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
trait BroadcastRun {
fn run(
&mut self,
@@ -111,7 +89,7 @@ trait BroadcastRun {
blockstore_sender: &Sender<Arc<Vec<Shred>>>,
) -> Result<()>;
fn transmit(
&mut self,
&self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,
@@ -157,27 +135,25 @@ impl BroadcastStage {
loop {
let res =
broadcast_stage_run.run(blockstore, receiver, socket_sender, blockstore_sender);
let res = Self::handle_error(res, "run");
let res = Self::handle_error(res);
if let Some(res) = res {
return res;
}
}
}
fn handle_error(r: Result<()>, name: &str) -> Option<BroadcastStageReturnType> {
fn handle_error(r: Result<()>) -> Option<BroadcastStageReturnType> {
if let Err(e) = r {
match e {
Error::RecvTimeoutError(RecvTimeoutError::Disconnected)
| Error::SendError
| Error::RecvError(RecvError)
| Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Disconnected) => {
| Error::RecvError(RecvError) => {
return Some(BroadcastStageReturnType::ChannelDisconnected);
}
Error::RecvTimeoutError(RecvTimeoutError::Timeout)
| Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Timeout) => (),
Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
Error::ClusterInfoError(ClusterInfoError::NoPeers) => (), // TODO: Why are the unit-tests throwing hundreds of these?
_ => {
inc_new_counter_error!("streamer-broadcaster-error", 1, 1);
error!("{} broadcaster error: {:?}", name, e);
error!("broadcaster error: {:?}", e);
}
}
}
@@ -204,7 +180,6 @@ impl BroadcastStage {
socks: Vec<UdpSocket>,
cluster_info: Arc<RwLock<ClusterInfo>>,
receiver: Receiver<WorkingBankEntry>,
retransmit_slots_receiver: RetransmitSlotsReceiver,
exit_sender: &Arc<AtomicBool>,
blockstore: &Arc<Blockstore>,
broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone,
@@ -214,8 +189,6 @@ impl BroadcastStage {
let (socket_sender, socket_receiver) = channel();
let (blockstore_sender, blockstore_receiver) = channel();
let bs_run = broadcast_stage_run.clone();
let socket_sender_ = socket_sender.clone();
let thread_hdl = Builder::new()
.name("solana-broadcaster".to_string())
.spawn(move || {
@@ -223,7 +196,7 @@ impl BroadcastStage {
Self::run(
&btree,
&receiver,
&socket_sender_,
&socket_sender,
&blockstore_sender,
bs_run,
)
@@ -233,13 +206,13 @@ impl BroadcastStage {
let socket_receiver = Arc::new(Mutex::new(socket_receiver));
for sock in socks.into_iter() {
let socket_receiver = socket_receiver.clone();
let mut bs_transmit = broadcast_stage_run.clone();
let bs_transmit = broadcast_stage_run.clone();
let cluster_info = cluster_info.clone();
let t = Builder::new()
.name("solana-broadcaster-transmit".to_string())
.spawn(move || loop {
let res = bs_transmit.transmit(&socket_receiver, &cluster_info, &sock);
let res = Self::handle_error(res, "solana-broadcaster-transmit");
let res = Self::handle_error(res);
if let Some(res) = res {
return res;
}
@@ -256,7 +229,7 @@ impl BroadcastStage {
.name("solana-broadcaster-record".to_string())
.spawn(move || loop {
let res = bs_record.record(&blockstore_receiver, &btree);
let res = Self::handle_error(res, "solana-broadcaster-record");
let res = Self::handle_error(res);
if let Some(res) = res {
return res;
}
@@ -265,68 +238,9 @@ impl BroadcastStage {
thread_hdls.push(t);
}
let blockstore = blockstore.clone();
let retransmit_thread = Builder::new()
.name("solana-broadcaster-retransmit".to_string())
.spawn(move || loop {
if let Some(res) = Self::handle_error(
Self::check_retransmit_signals(
&blockstore,
&retransmit_slots_receiver,
&socket_sender,
),
"solana-broadcaster-retransmit-check_retransmit_signals",
) {
return res;
}
})
.unwrap();
thread_hdls.push(retransmit_thread);
Self { thread_hdls }
}
fn check_retransmit_signals(
blockstore: &Blockstore,
retransmit_slots_receiver: &RetransmitSlotsReceiver,
socket_sender: &Sender<TransmitShreds>,
) -> Result<()> {
let timer = Duration::from_millis(100);
// Check for a retransmit signal
let mut retransmit_slots = retransmit_slots_receiver.recv_timeout(timer)?;
while let Ok(new_retransmit_slots) = retransmit_slots_receiver.try_recv() {
retransmit_slots.extend(new_retransmit_slots);
}
for (_, bank) in retransmit_slots.iter() {
let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
let stakes = stakes.map(Arc::new);
let data_shreds = Arc::new(
blockstore
.get_data_shreds_for_slot(bank.slot(), 0)
.expect("My own shreds must be reconstructable"),
);
if !data_shreds.is_empty() {
socket_sender.send((stakes.clone(), data_shreds))?;
}
let coding_shreds = Arc::new(
blockstore
.get_coding_shreds_for_slot(bank.slot(), 0)
.expect("My own shreds must be reconstructable"),
);
if !coding_shreds.is_empty() {
socket_sender.send((stakes.clone(), coding_shreds))?;
}
}
Ok(())
}
pub fn join(self) -> thread::Result<BroadcastStageReturnType> {
for thread_hdl in self.thread_hdls.into_iter() {
let _ = thread_hdl.join();
@@ -335,216 +249,23 @@ impl BroadcastStage {
}
}
fn update_peer_stats(num_live_peers: i64, broadcast_len: i64, last_datapoint_submit: &mut Instant) {
if duration_as_s(&Instant::now().duration_since(*last_datapoint_submit)) >= 1.0 {
datapoint_info!(
"cluster_info-num_nodes",
("live_count", num_live_peers, i64),
("broadcast_count", broadcast_len, i64)
);
*last_datapoint_submit = Instant::now();
}
}
pub fn get_broadcast_peers<S: std::hash::BuildHasher>(
cluster_info: &Arc<RwLock<ClusterInfo>>,
stakes: Option<Arc<HashMap<Pubkey, u64, S>>>,
) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
use crate::cluster_info;
let mut peers = cluster_info.read().unwrap().tvu_peers();
let peers_and_stakes = cluster_info::stake_weight_peers(&mut peers, stakes);
(peers, peers_and_stakes)
}
/// broadcast messages from the leader to layer 1 nodes
/// # Remarks
pub fn broadcast_shreds(
s: &UdpSocket,
shreds: &Arc<Vec<Shred>>,
peers_and_stakes: &[(u64, usize)],
peers: &[ContactInfo],
last_datapoint_submit: &mut Instant,
send_mmsg_total: &mut u64,
) -> Result<()> {
let broadcast_len = peers_and_stakes.len();
if broadcast_len == 0 {
update_peer_stats(1, 1, last_datapoint_submit);
return Ok(());
}
let packets: Vec<_> = shreds
.iter()
.map(|shred| {
let broadcast_index = weighted_best(&peers_and_stakes, shred.seed());
(&shred.payload, &peers[broadcast_index].tvu)
})
.collect();
let mut sent = 0;
let mut send_mmsg_time = Measure::start("send_mmsg");
while sent < packets.len() {
match send_mmsg(s, &packets[sent..]) {
Ok(n) => sent += n,
Err(e) => {
return Err(Error::IO(e));
}
}
}
send_mmsg_time.stop();
*send_mmsg_total += send_mmsg_time.as_us();
let num_live_peers = num_live_peers(&peers);
update_peer_stats(
num_live_peers,
broadcast_len as i64 + 1,
last_datapoint_submit,
);
Ok(())
}
fn num_live_peers(peers: &[ContactInfo]) -> i64 {
let mut num_live_peers = 1i64;
peers.iter().for_each(|p| {
// A peer is considered live if they generated their contact info recently
if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
num_live_peers += 1;
}
});
num_live_peers
}
#[cfg(test)]
pub mod test {
mod test {
use super::*;
use crate::cluster_info::{ClusterInfo, Node};
use crossbeam_channel::unbounded;
use solana_ledger::{
blockstore::{make_slot_entries, Blockstore},
entry::create_ticks,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
shred::{max_ticks_per_n_shreds, Shredder, RECOMMENDED_FEC_RATE},
};
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::entry::create_ticks;
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
use solana_runtime::bank::Bank;
use solana_sdk::{
hash::Hash,
pubkey::Pubkey,
signature::{Keypair, Signer},
};
use std::{
path::Path,
sync::atomic::AtomicBool,
sync::mpsc::channel,
sync::{Arc, RwLock},
thread::sleep,
};
pub fn make_transmit_shreds(
slot: Slot,
num: u64,
stakes: Option<Arc<HashMap<Pubkey, u64>>>,
) -> (
Vec<Shred>,
Vec<Shred>,
Vec<TransmitShreds>,
Vec<TransmitShreds>,
) {
let num_entries = max_ticks_per_n_shreds(num);
let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
let keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
.expect("Expected to create a new shredder");
let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..]);
(
data_shreds.clone(),
coding_shreds.clone(),
data_shreds
.into_iter()
.map(|s| (stakes.clone(), Arc::new(vec![s])))
.collect(),
coding_shreds
.into_iter()
.map(|s| (stakes.clone(), Arc::new(vec![s])))
.collect(),
)
}
fn check_all_shreds_received(
transmit_receiver: &Receiver<TransmitShreds>,
mut data_index: u64,
mut coding_index: u64,
num_expected_data_shreds: u64,
num_expected_coding_shreds: u64,
) {
while let Ok(new_retransmit_slots) = transmit_receiver.try_recv() {
if new_retransmit_slots.1[0].is_data() {
for data_shred in new_retransmit_slots.1.iter() {
assert_eq!(data_shred.index() as u64, data_index);
data_index += 1;
}
} else {
assert_eq!(new_retransmit_slots.1[0].index() as u64, coding_index);
for coding_shred in new_retransmit_slots.1.iter() {
assert_eq!(coding_shred.index() as u64, coding_index);
coding_index += 1;
}
}
}
assert_eq!(num_expected_data_shreds, data_index);
assert_eq!(num_expected_coding_shreds, coding_index);
}
#[test]
fn test_duplicate_retransmit_signal() {
// Setup
let ledger_path = get_tmp_ledger_path!();
let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
let (transmit_sender, transmit_receiver) = channel();
let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
let bank0 = Arc::new(Bank::new(&genesis_config));
// Make some shreds
let updated_slot = 0;
let (all_data_shreds, all_coding_shreds, _, _all_coding_transmit_shreds) =
make_transmit_shreds(updated_slot, 10, None);
let num_data_shreds = all_data_shreds.len();
let num_coding_shreds = all_coding_shreds.len();
assert!(num_data_shreds >= 10);
// Insert all the shreds
blockstore
.insert_shreds(all_data_shreds, None, true)
.unwrap();
blockstore
.insert_shreds(all_coding_shreds, None, true)
.unwrap();
// Insert duplicate retransmit signal, blocks should
// only be retransmitted once
retransmit_slots_sender
.send(vec![(updated_slot, bank0.clone())].into_iter().collect())
.unwrap();
retransmit_slots_sender
.send(vec![(updated_slot, bank0.clone())].into_iter().collect())
.unwrap();
BroadcastStage::check_retransmit_signals(
&blockstore,
&retransmit_slots_receiver,
&transmit_sender,
)
.unwrap();
// Check all the data shreds were received only once
check_all_shreds_received(
&transmit_receiver,
0,
0,
num_data_shreds as u64,
num_coding_shreds as u64,
);
}
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, Signer};
use std::path::Path;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::channel;
use std::sync::{Arc, RwLock};
use std::thread::sleep;
use std::time::Duration;
struct MockBroadcastStage {
blockstore: Arc<Blockstore>,
@@ -556,7 +277,6 @@ pub mod test {
leader_pubkey: &Pubkey,
ledger_path: &Path,
entry_receiver: Receiver<WorkingBankEntry>,
retransmit_slots_receiver: RetransmitSlotsReceiver,
) -> MockBroadcastStage {
// Make the database ledger
let blockstore = Arc::new(Blockstore::open(ledger_path).unwrap());
@@ -584,7 +304,6 @@ pub mod test {
leader_info.sockets.broadcast,
cluster_info,
entry_receiver,
retransmit_slots_receiver,
&exit_sender,
&blockstore,
StandardBroadcastRun::new(leader_keypair, 0),
@@ -607,12 +326,10 @@ pub mod test {
let leader_keypair = Keypair::new();
let (entry_sender, entry_receiver) = channel();
let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
let broadcast_service = setup_dummy_broadcast_service(
&leader_keypair.pubkey(),
&ledger_path,
entry_receiver,
retransmit_slots_receiver,
);
let start_tick_height;
let max_tick_height;
@@ -631,7 +348,6 @@ pub mod test {
.expect("Expect successful send to broadcast service");
}
}
sleep(Duration::from_millis(2000));
trace!(
@@ -648,7 +364,6 @@ pub mod test {
assert_eq!(entries.len(), max_tick_height as usize);
drop(entry_sender);
drop(retransmit_slots_sender);
broadcast_service
.broadcast_service
.join()

View File

@@ -96,7 +96,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
Ok(())
}
fn transmit(
&mut self,
&self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,

View File

@@ -76,7 +76,7 @@ pub(super) fn recv_slot_entries(receiver: &Receiver<WorkingBankEntry>) -> Result
#[cfg(test)]
mod tests {
use super::*;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_sdk::genesis_config::GenesisConfig;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::system_transaction;

View File

@@ -72,25 +72,19 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
Ok(())
}
fn transmit(
&mut self,
&self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,
) -> Result<()> {
let (stakes, shreds) = receiver.lock().unwrap().recv()?;
let all_seeds: Vec<[u8; 32]> = shreds.iter().map(|s| s.seed()).collect();
// Broadcast data
let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
let mut send_mmsg_total = 0;
broadcast_shreds(
sock,
&shreds,
&peers_and_stakes,
&peers,
&mut Instant::now(),
&mut send_mmsg_total,
)?;
let all_shred_bufs: Vec<Vec<u8>> = shreds.to_vec().into_iter().map(|s| s.payload).collect();
cluster_info
.write()
.unwrap()
.broadcast_shreds(sock, all_shred_bufs, &all_seeds, stakes)?;
Ok(())
}
fn record(

View File

@@ -1,11 +1,11 @@
use super::broadcast_utils::{self, ReceiveResults};
use super::*;
use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
use solana_ledger::{
entry::Entry,
shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK},
};
use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
use solana_ledger::entry::Entry;
use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Keypair;
use solana_sdk::timing::duration_as_us;
use std::collections::HashMap;
use std::time::Duration;
@@ -17,24 +17,26 @@ struct BroadcastStats {
broadcast_elapsed: u64,
receive_elapsed: u64,
seed_elapsed: u64,
send_mmsg_elapsed: u64,
}
impl BroadcastStats {
fn reset(&mut self) {
*self = Self::default();
self.insert_shreds_elapsed = 0;
self.shredding_elapsed = 0;
self.broadcast_elapsed = 0;
self.receive_elapsed = 0;
self.seed_elapsed = 0;
}
}
#[derive(Clone)]
pub struct StandardBroadcastRun {
pub(super) struct StandardBroadcastRun {
stats: Arc<RwLock<BroadcastStats>>,
unfinished_slot: Option<UnfinishedSlotInfo>,
current_slot_and_parent: Option<(u64, u64)>,
slot_broadcast_start: Option<Instant>,
keypair: Arc<Keypair>,
shred_version: u16,
last_datapoint_submit: Instant,
}
impl StandardBroadcastRun {
@@ -46,7 +48,6 @@ impl StandardBroadcastRun {
slot_broadcast_start: None,
keypair,
shred_version,
last_datapoint_submit: Instant::now(),
}
}
@@ -211,8 +212,7 @@ impl StandardBroadcastRun {
blockstore_sender.send(data_shreds.clone())?;
let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred]);
let coding_shreds = Arc::new(coding_shreds);
socket_sender.send((stakes, coding_shreds.clone()))?;
blockstore_sender.send(coding_shreds)?;
socket_sender.send((stakes, coding_shreds))?;
self.update_broadcast_stats(BroadcastStats {
shredding_elapsed: duration_as_us(&to_shreds_elapsed),
receive_elapsed: duration_as_us(&receive_elapsed),
@@ -248,37 +248,31 @@ impl StandardBroadcastRun {
}
fn broadcast(
&mut self,
&self,
sock: &UdpSocket,
cluster_info: &Arc<RwLock<ClusterInfo>>,
stakes: Option<Arc<HashMap<Pubkey, u64>>>,
shreds: Arc<Vec<Shred>>,
) -> Result<()> {
let seed_start = Instant::now();
let seeds: Vec<[u8; 32]> = shreds.iter().map(|s| s.seed()).collect();
let seed_elapsed = seed_start.elapsed();
// Broadcast the shreds
let broadcast_start = Instant::now();
trace!("Broadcasting {:?} shreds", shreds.len());
let shred_bufs: Vec<Vec<u8>> = shreds.to_vec().into_iter().map(|s| s.payload).collect();
trace!("Broadcasting {:?} shreds", shred_bufs.len());
let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
let mut send_mmsg_total = 0;
broadcast_shreds(
sock,
&shreds,
&peers_and_stakes,
&peers,
&mut self.last_datapoint_submit,
&mut send_mmsg_total,
)?;
cluster_info
.write()
.unwrap()
.broadcast_shreds(sock, shred_bufs, &seeds, stakes)?;
let broadcast_elapsed = broadcast_start.elapsed();
self.update_broadcast_stats(BroadcastStats {
broadcast_elapsed: duration_as_us(&broadcast_elapsed),
seed_elapsed: duration_as_us(&seed_elapsed),
send_mmsg_elapsed: send_mmsg_total,
..BroadcastStats::default()
});
Ok(())
@@ -291,7 +285,6 @@ impl StandardBroadcastRun {
wstats.insert_shreds_elapsed += stats.insert_shreds_elapsed;
wstats.broadcast_elapsed += stats.broadcast_elapsed;
wstats.seed_elapsed += stats.seed_elapsed;
wstats.send_mmsg_elapsed += stats.send_mmsg_elapsed;
}
fn report_and_reset_stats(&mut self) {
@@ -304,7 +297,6 @@ impl StandardBroadcastRun {
("insertion_time", stats.insert_shreds_elapsed as i64, i64),
("broadcast_time", stats.broadcast_elapsed as i64, i64),
("receive_time", stats.receive_elapsed as i64, i64),
("send_mmsg", stats.send_mmsg_elapsed as i64, i64),
("seed", stats.seed_elapsed as i64, i64),
(
"num_shreds",
@@ -339,7 +331,7 @@ impl BroadcastRun for StandardBroadcastRun {
)
}
fn transmit(
&mut self,
&self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,
@@ -361,13 +353,14 @@ impl BroadcastRun for StandardBroadcastRun {
mod test {
use super::*;
use crate::cluster_info::{ClusterInfo, Node};
use solana_ledger::genesis_utils::create_genesis_config;
use crate::genesis_utils::create_genesis_config;
use solana_ledger::{
blockstore::Blockstore, entry::create_ticks, get_tmp_ledger_path,
shred::max_ticks_per_n_shreds,
};
use solana_runtime::bank::Bank;
use solana_sdk::{
clock::Slot,
genesis_config::GenesisConfig,
signature::{Keypair, Signer},
};
@@ -476,9 +469,11 @@ mod test {
.receive_elapsed = 10;
// Try to fetch ticks from blockstore, nothing should break
assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), ticks0);
assert_eq!(blockstore.get_slot_entries(0, 0, None).unwrap(), ticks0);
assert_eq!(
blockstore.get_slot_entries(0, num_shreds_per_slot).unwrap(),
blockstore
.get_slot_entries(0, num_shreds_per_slot, None)
.unwrap(),
vec![],
);
@@ -514,9 +509,11 @@ mod test {
);
// Try to fetch the incomplete ticks from blockstore, should succeed
assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), ticks0);
assert_eq!(blockstore.get_slot_entries(0, 0, None).unwrap(), ticks0);
assert_eq!(
blockstore.get_slot_entries(0, num_shreds_per_slot).unwrap(),
blockstore
.get_slot_entries(0, num_shreds_per_slot, None)
.unwrap(),
vec![],
);
}

View File

@@ -12,19 +12,23 @@
//! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
//!
//! Bank needs to provide an interface for us to query the stake weight
use crate::crds_value::CompressionType::*;
use crate::crds_value::EpochIncompleteSlots;
use crate::packet::limited_deserialize;
use crate::streamer::{PacketReceiver, PacketSender};
use crate::{
contact_info::ContactInfo,
crds_gossip::CrdsGossip,
crds_gossip_error::CrdsGossipError,
crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
crds_value::{
self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlotsIndex, LowestSlot, SnapshotHash, Vote,
},
epoch_slots::EpochSlots,
crds_value::{self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote},
packet::{Packet, PACKET_DATA_SIZE},
result::{Error, Result},
weighted_shuffle::weighted_shuffle,
sendmmsg::{multicast, send_mmsg},
weighted_shuffle::{weighted_best, weighted_shuffle},
};
use bincode::{serialize, serialized_size};
use compression::prelude::*;
use core::cmp;
use itertools::Itertools;
use rayon::iter::IntoParallelIterator;
@@ -37,25 +41,21 @@ use solana_net_utils::{
bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range,
multi_bind_in_range, PortRange,
};
use solana_perf::packet::{
limited_deserialize, to_packets_with_destination, Packet, Packets, PacketsRecycler,
PACKET_DATA_SIZE,
};
use solana_perf::packet::{to_packets_with_destination, Packets, PacketsRecycler};
use solana_rayon_threadlimit::get_thread_count;
use solana_sdk::hash::Hash;
use solana_sdk::timing::duration_as_s;
use solana_sdk::{
clock::{Slot, DEFAULT_MS_PER_SLOT, DEFAULT_SLOTS_PER_EPOCH},
clock::{Slot, DEFAULT_MS_PER_SLOT},
pubkey::Pubkey,
signature::{Keypair, Signable, Signature, Signer},
timing::{duration_as_ms, timestamp},
transaction::Transaction,
};
use solana_streamer::sendmmsg::multicast;
use solana_streamer::streamer::{PacketReceiver, PacketSender};
use std::{
borrow::Cow,
cmp::min,
collections::{HashMap, HashSet},
collections::{BTreeSet, HashMap, HashSet},
fmt,
net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket},
sync::atomic::{AtomicBool, Ordering},
@@ -71,8 +71,7 @@ pub const DATA_PLANE_FANOUT: usize = 200;
/// milliseconds we sleep for between gossip requests
pub const GOSSIP_SLEEP_MILLIS: u64 = 100;
/// The maximum size of a bloom filter
pub const MAX_BLOOM_SIZE: usize = MAX_CRDS_OBJECT_SIZE;
pub const MAX_CRDS_OBJECT_SIZE: usize = 928;
pub const MAX_BLOOM_SIZE: usize = 1018;
/// The maximum size of a protocol payload
const MAX_PROTOCOL_PAYLOAD_SIZE: u64 = PACKET_DATA_SIZE as u64 - MAX_PROTOCOL_HEADER_SIZE;
/// The largest protocol header size
@@ -82,6 +81,9 @@ const MAX_PROTOCOL_HEADER_SIZE: u64 = 214;
/// 128MB/PACKET_DATA_SIZE
const MAX_GOSSIP_TRAFFIC: usize = 128_000_000 / PACKET_DATA_SIZE;
const NUM_BITS_PER_BYTE: u64 = 8;
const MIN_SIZE_TO_COMPRESS_GZIP: u64 = 64;
/// Keep the number of snapshot hashes a node publishes under MAX_PROTOCOL_PAYLOAD_SIZE
pub const MAX_SNAPSHOT_HASHES: usize = 16;
@@ -100,6 +102,7 @@ pub struct ClusterInfo {
pub(crate) keypair: Arc<Keypair>,
/// The network entrypoint
entrypoint: Option<ContactInfo>,
last_datapoint_submit: Instant,
}
#[derive(Default, Clone)]
@@ -207,6 +210,7 @@ impl ClusterInfo {
gossip: CrdsGossip::default(),
keypair,
entrypoint: None,
last_datapoint_submit: Instant::now(),
};
let id = contact_info.id;
me.gossip.set_self(&id);
@@ -258,16 +262,6 @@ impl ClusterInfo {
self.lookup(&self.id()).cloned().unwrap()
}
pub fn lookup_epoch_slots(&self, ix: EpochSlotsIndex) -> EpochSlots {
let entry = CrdsValueLabel::EpochSlots(ix, self.id());
self.gossip
.crds
.lookup(&entry)
.and_then(CrdsValue::epoch_slots)
.cloned()
.unwrap_or_else(|| EpochSlots::new(self.id(), timestamp()))
}
pub fn contact_info_trace(&self) -> String {
let now = timestamp();
let mut spy_nodes = 0;
@@ -342,80 +336,119 @@ impl ClusterInfo {
)
}
pub fn push_lowest_slot(&mut self, id: Pubkey, min: Slot) {
let now = timestamp();
let last = self
.gossip
.crds
.lookup(&CrdsValueLabel::LowestSlot(self.id()))
.and_then(|x| x.lowest_slot())
.map(|x| x.lowest)
.unwrap_or(0);
if min > last {
let entry = CrdsValue::new_signed(
CrdsData::LowestSlot(0, LowestSlot::new(id, min, now)),
&self.keypair,
);
self.gossip
.process_push_message(&self.id(), vec![entry], now);
pub fn compress_incomplete_slots(incomplete_slots: &BTreeSet<Slot>) -> EpochIncompleteSlots {
if !incomplete_slots.is_empty() {
let first_slot = incomplete_slots
.iter()
.next()
.expect("expected to find at least one slot");
let last_slot = incomplete_slots
.iter()
.next_back()
.expect("expected to find last slot");
let num_uncompressed_bits = last_slot.saturating_sub(*first_slot) + 1;
let num_uncompressed_bytes = if num_uncompressed_bits % NUM_BITS_PER_BYTE > 0 {
1
} else {
0
} + num_uncompressed_bits / NUM_BITS_PER_BYTE;
let mut uncompressed = vec![0u8; num_uncompressed_bytes as usize];
incomplete_slots.iter().for_each(|slot| {
let offset_from_first_slot = slot.saturating_sub(*first_slot);
let index = offset_from_first_slot / NUM_BITS_PER_BYTE;
let bit_index = offset_from_first_slot % NUM_BITS_PER_BYTE;
uncompressed[index as usize] |= 1 << bit_index;
});
if num_uncompressed_bytes >= MIN_SIZE_TO_COMPRESS_GZIP {
if let Ok(compressed) = uncompressed
.iter()
.cloned()
.encode(&mut GZipEncoder::new(), Action::Finish)
.collect::<std::result::Result<Vec<u8>, _>>()
{
return EpochIncompleteSlots {
first: *first_slot,
compression: GZip,
compressed_list: compressed,
};
}
} else {
return EpochIncompleteSlots {
first: *first_slot,
compression: Uncompressed,
compressed_list: uncompressed,
};
}
}
EpochIncompleteSlots::default()
}
fn bitmap_to_slot_list(first: Slot, bitmap: &[u8]) -> BTreeSet<Slot> {
let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
bitmap.iter().enumerate().for_each(|(i, val)| {
if *val != 0 {
(0..8).for_each(|bit_index| {
if (1 << bit_index & *val) != 0 {
let slot = first + i as u64 * NUM_BITS_PER_BYTE + bit_index as u64;
old_incomplete_slots.insert(slot);
}
})
}
});
old_incomplete_slots
}
pub fn decompress_incomplete_slots(slots: &EpochIncompleteSlots) -> BTreeSet<Slot> {
match slots.compression {
Uncompressed => Self::bitmap_to_slot_list(slots.first, &slots.compressed_list),
GZip => {
if let Ok(decompressed) = slots
.compressed_list
.iter()
.cloned()
.decode(&mut GZipDecoder::new())
.collect::<std::result::Result<Vec<u8>, _>>()
{
Self::bitmap_to_slot_list(slots.first, &decompressed)
} else {
BTreeSet::new()
}
}
BZip2 => {
if let Ok(decompressed) = slots
.compressed_list
.iter()
.cloned()
.decode(&mut BZip2Decoder::new())
.collect::<std::result::Result<Vec<u8>, _>>()
{
Self::bitmap_to_slot_list(slots.first, &decompressed)
} else {
BTreeSet::new()
}
}
}
}
pub fn push_epoch_slots(&mut self, update: &[Slot]) {
let mut num = 0;
let mut current_slots: Vec<_> = (0..crds_value::MAX_EPOCH_SLOTS)
.filter_map(|ix| {
Some((
self.gossip
.crds
.lookup(&CrdsValueLabel::EpochSlots(ix, self.id()))
.and_then(CrdsValue::epoch_slots)
.and_then(|x| Some((x.wallclock, x.first_slot()?)))?,
ix,
))
})
.collect();
current_slots.sort();
let min_slot: Slot = current_slots
.iter()
.map(|((_, s), _)| *s)
.min()
.unwrap_or(0);
let max_slot: Slot = update.iter().max().cloned().unwrap_or(0);
let total_slots = max_slot as isize - min_slot as isize;
// WARN if CRDS is not storing at least a full epoch worth of slots
if DEFAULT_SLOTS_PER_EPOCH as isize > total_slots
&& crds_value::MAX_EPOCH_SLOTS as usize <= current_slots.len()
{
inc_new_counter_warn!("cluster_info-epoch_slots-filled", 1);
warn!(
"EPOCH_SLOTS are filling up FAST {}/{}",
total_slots,
current_slots.len()
);
}
let mut reset = false;
let mut epoch_slot_index = current_slots.last().map(|(_, x)| *x).unwrap_or(0);
while num < update.len() {
let ix = (epoch_slot_index % crds_value::MAX_EPOCH_SLOTS) as u8;
let now = timestamp();
let mut slots = if !reset {
self.lookup_epoch_slots(ix)
} else {
EpochSlots::new(self.id(), now)
};
let n = slots.fill(&update[num..], now);
if n > 0 {
let entry = CrdsValue::new_signed(CrdsData::EpochSlots(ix, slots), &self.keypair);
self.gossip
.process_push_message(&self.id(), vec![entry], now);
}
num += n;
if num < update.len() {
epoch_slot_index += 1;
reset = true;
}
}
pub fn push_epoch_slots(
&mut self,
id: Pubkey,
root: Slot,
min: Slot,
slots: BTreeSet<Slot>,
incomplete_slots: &BTreeSet<Slot>,
) {
let compressed = Self::compress_incomplete_slots(incomplete_slots);
let now = timestamp();
let entry = CrdsValue::new_signed(
CrdsData::EpochSlots(
0,
EpochSlots::new(id, root, min, slots, vec![compressed], now),
),
&self.keypair,
);
self.gossip
.process_push_message(&self.id(), vec![entry], now);
}
pub fn push_message(&mut self, message: CrdsValue) {
@@ -471,23 +504,23 @@ impl ClusterInfo {
/// since. This allows the bank to query for new votes only.
///
/// * return - The votes, and the max timestamp from the new set.
pub fn get_votes(&self, since: u64) -> (Vec<CrdsValueLabel>, Vec<Transaction>, u64) {
let mut max_ts = since;
let (labels, txs): (Vec<CrdsValueLabel>, Vec<Transaction>) = self
pub fn get_votes(&self, since: u64) -> (Vec<Transaction>, u64) {
let votes: Vec<_> = self
.gossip
.crds
.table
.iter()
.filter(|(_, x)| x.insert_timestamp > since)
.filter_map(|(label, x)| {
max_ts = std::cmp::max(x.insert_timestamp, max_ts);
.values()
.filter(|x| x.insert_timestamp > since)
.filter_map(|x| {
x.value
.vote()
.map(|v| (label.clone(), v.transaction.clone()))
.map(|v| (x.insert_timestamp, v.transaction.clone()))
})
.unzip();
.collect();
let max_ts = votes.iter().map(|x| x.0).max().unwrap_or(since);
let txs: Vec<Transaction> = votes.into_iter().map(|x| x.1).collect();
inc_new_counter_info!("cluster_info-get_votes-count", txs.len());
(labels, txs, max_ts)
(txs, max_ts)
}
pub fn get_snapshot_hash(&self, slot: Slot) -> Vec<(Pubkey, Hash)> {
@@ -523,39 +556,21 @@ impl ClusterInfo {
.map(|x| &x.value.snapshot_hash().unwrap().hashes)
}
pub fn get_lowest_slot_for_node(
pub fn get_epoch_state_for_node(
&self,
pubkey: &Pubkey,
since: Option<u64>,
) -> Option<(&LowestSlot, u64)> {
) -> Option<(&EpochSlots, u64)> {
self.gossip
.crds
.table
.get(&CrdsValueLabel::LowestSlot(*pubkey))
.get(&CrdsValueLabel::EpochSlots(*pubkey))
.filter(|x| {
since
.map(|since| x.insert_timestamp > since)
.unwrap_or(true)
})
.map(|x| (x.value.lowest_slot().unwrap(), x.insert_timestamp))
}
pub fn get_epoch_slots_since(&self, since: Option<u64>) -> (Vec<EpochSlots>, Option<u64>) {
let vals: Vec<_> = self
.gossip
.crds
.table
.values()
.filter(|x| {
since
.map(|since| x.insert_timestamp > since)
.unwrap_or(true)
})
.filter_map(|x| Some((x.value.epoch_slots()?, x.insert_timestamp)))
.collect();
let max = vals.iter().map(|x| x.1).max().or(since);
let vec = vals.into_iter().map(|x| x.0).cloned().collect();
(vec, max)
.map(|x| (x.value.epoch_slots().unwrap(), x.insert_timestamp))
}
pub fn get_contact_info_for_node(&self, pubkey: &Pubkey) -> Option<&ContactInfo> {
@@ -699,8 +714,8 @@ impl ClusterInfo {
&& x.shred_version == me.shred_version
&& ContactInfo::is_valid_address(&x.serve_repair)
&& {
self.get_lowest_slot_for_node(&x.id, None)
.map(|(lowest_slot, _)| lowest_slot.lowest <= slot)
self.get_epoch_state_for_node(&x.id, None)
.map(|(epoch_slots, _)| epoch_slots.lowest <= slot)
.unwrap_or_else(|| /* fallback to legacy behavior */ true)
}
})
@@ -933,6 +948,76 @@ impl ClusterInfo {
.collect()
}
fn sorted_tvu_peers_and_stakes(
&self,
stakes: Option<Arc<HashMap<Pubkey, u64>>>,
) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
let mut peers = self.tvu_peers();
peers.dedup();
let stakes_and_index = ClusterInfo::sorted_stakes_with_index(&peers, stakes);
(peers, stakes_and_index)
}
/// broadcast messages from the leader to layer 1 nodes
/// # Remarks
pub fn broadcast_shreds(
&mut self,
s: &UdpSocket,
shreds: Vec<Vec<u8>>,
seeds: &[[u8; 32]],
stakes: Option<Arc<HashMap<Pubkey, u64>>>,
) -> Result<()> {
let (peers, peers_and_stakes) = self.sorted_tvu_peers_and_stakes(stakes);
let broadcast_len = peers_and_stakes.len();
if broadcast_len == 0 {
if duration_as_s(&Instant::now().duration_since(self.last_datapoint_submit)) >= 1.0 {
datapoint_info!(
"cluster_info-num_nodes",
("live_count", 1, i64),
("broadcast_count", 1, i64)
);
self.last_datapoint_submit = Instant::now();
}
return Ok(());
}
let mut packets: Vec<_> = shreds
.into_iter()
.zip(seeds)
.map(|(shred, seed)| {
let broadcast_index = weighted_best(&peers_and_stakes, *seed);
(shred, &peers[broadcast_index].tvu)
})
.collect();
let mut sent = 0;
while sent < packets.len() {
match send_mmsg(s, &mut packets[sent..]) {
Ok(n) => sent += n,
Err(e) => {
return Err(Error::IO(e));
}
}
}
let mut num_live_peers = 1i64;
peers.iter().for_each(|p| {
// A peer is considered live if they generated their contact info recently
if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
num_live_peers += 1;
}
});
if duration_as_s(&Instant::now().duration_since(self.last_datapoint_submit)) >= 1.0 {
datapoint_info!(
"cluster_info-num_nodes",
("live_count", num_live_peers, i64),
("broadcast_count", broadcast_len + 1, i64)
);
self.last_datapoint_submit = Instant::now();
}
Ok(())
}
/// retransmit messages to a list of nodes
/// # Remarks
/// We need to avoid having obj locked while doing a io, such as the `send_to`
@@ -1240,7 +1325,7 @@ impl ClusterInfo {
Protocol::PullRequest(filter, caller) => {
let start = allocated.get();
if !caller.verify() {
inc_new_counter_info!(
inc_new_counter_error!(
"cluster_info-gossip_pull_request_verify_fail",
1
);
@@ -1266,7 +1351,7 @@ impl ClusterInfo {
data.retain(|v| {
let ret = v.verify();
if !ret {
inc_new_counter_info!(
inc_new_counter_error!(
"cluster_info-gossip_pull_response_verify_fail",
1
);
@@ -1284,7 +1369,7 @@ impl ClusterInfo {
data.retain(|v| {
let ret = v.verify();
if !ret {
inc_new_counter_info!(
inc_new_counter_error!(
"cluster_info-gossip_push_msg_verify_fail",
1
);
@@ -1864,20 +1949,12 @@ impl Node {
}
fn report_time_spent(label: &str, time: &Duration, extra: &str) {
let time_ms = duration_as_ms(time);
if time_ms > 50 {
info!("{} took: {} ms {}", label, time_ms, extra);
let count = duration_as_ms(time);
if count > 5 {
info!("{} took: {} ms {}", label, count, extra);
}
}
pub fn stake_weight_peers<S: std::hash::BuildHasher>(
peers: &mut Vec<ContactInfo>,
stakes: Option<Arc<HashMap<Pubkey, u64, S>>>,
) -> Vec<(u64, usize)> {
peers.dedup();
ClusterInfo::sorted_stakes_with_index(peers, stakes)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1998,39 +2075,40 @@ mod tests {
#[test]
fn new_with_external_ip_test_gossip() {
// Can't use VALIDATOR_PORT_RANGE because if this test runs in parallel with others, the
// port returned by `bind_in_range()` might be snatched up before `Node::new_with_external_ip()` runs
let port_range = (VALIDATOR_PORT_RANGE.1 + 10, VALIDATOR_PORT_RANGE.1 + 20);
let ip = IpAddr::V4(Ipv4Addr::from(0));
let port = bind_in_range(ip, port_range).expect("Failed to bind").0;
let node =
Node::new_with_external_ip(&Pubkey::new_rand(), &socketaddr!(0, port), port_range, ip);
let port = {
bind_in_range(ip, VALIDATOR_PORT_RANGE)
.expect("Failed to bind")
.0
};
let node = Node::new_with_external_ip(
&Pubkey::new_rand(),
&socketaddr!(0, port),
VALIDATOR_PORT_RANGE,
ip,
);
check_node_sockets(&node, ip, port_range);
check_node_sockets(&node, ip, VALIDATOR_PORT_RANGE);
assert_eq!(node.sockets.gossip.local_addr().unwrap().port(), port);
}
#[test]
fn new_archiver_external_ip_test() {
// Can't use VALIDATOR_PORT_RANGE because if this test runs in parallel with others, the
// port returned by `bind_in_range()` might be snatched up before `Node::new_with_external_ip()` runs
let port_range = (VALIDATOR_PORT_RANGE.1 + 20, VALIDATOR_PORT_RANGE.1 + 30);
let ip = Ipv4Addr::from(0);
let node = Node::new_archiver_with_external_ip(
&Pubkey::new_rand(),
&socketaddr!(ip, 0),
port_range,
VALIDATOR_PORT_RANGE,
IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
);
let ip = IpAddr::V4(ip);
check_socket(&node.sockets.storage.unwrap(), ip, port_range);
check_socket(&node.sockets.gossip, ip, port_range);
check_socket(&node.sockets.repair, ip, port_range);
check_socket(&node.sockets.storage.unwrap(), ip, VALIDATOR_PORT_RANGE);
check_socket(&node.sockets.gossip, ip, VALIDATOR_PORT_RANGE);
check_socket(&node.sockets.repair, ip, VALIDATOR_PORT_RANGE);
check_sockets(&node.sockets.tvu, ip, port_range);
check_sockets(&node.sockets.tvu, ip, VALIDATOR_PORT_RANGE);
}
//test that all cluster_info objects only generate signed messages
@@ -2208,62 +2286,30 @@ mod tests {
#[test]
fn test_push_vote() {
let keys = Keypair::new();
let now = timestamp();
let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
// make sure empty crds is handled correctly
let now = timestamp();
let (_, votes, max_ts) = cluster_info.get_votes(now);
let (votes, max_ts) = cluster_info.get_votes(now);
assert_eq!(votes, vec![]);
assert_eq!(max_ts, now);
// add a vote
let tx = test_tx();
let index = 1;
cluster_info.push_vote(index, tx.clone());
cluster_info.push_vote(0, tx.clone());
// -1 to make sure that the clock is strictly lower then when insert occurred
let (labels, votes, max_ts) = cluster_info.get_votes(now - 1);
let (votes, max_ts) = cluster_info.get_votes(now - 1);
assert_eq!(votes, vec![tx]);
assert_eq!(labels.len(), 1);
match labels[0] {
CrdsValueLabel::Vote(_, pubkey) => {
assert_eq!(pubkey, keys.pubkey());
}
_ => panic!("Bad match"),
}
assert!(max_ts >= now - 1);
// make sure timestamp filter works
let (_, votes, new_max_ts) = cluster_info.get_votes(max_ts);
let (votes, new_max_ts) = cluster_info.get_votes(max_ts);
assert_eq!(votes, vec![]);
assert_eq!(max_ts, new_max_ts);
}
#[test]
fn test_push_epoch_slots() {
let keys = Keypair::new();
let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
let (slots, since) = cluster_info.get_epoch_slots_since(None);
assert!(slots.is_empty());
assert!(since.is_none());
cluster_info.push_epoch_slots(&[0]);
let (slots, since) = cluster_info.get_epoch_slots_since(Some(std::u64::MAX));
assert!(slots.is_empty());
assert_eq!(since, Some(std::u64::MAX));
let (slots, since) = cluster_info.get_epoch_slots_since(None);
assert_eq!(slots.len(), 1);
assert!(since.is_some());
let (slots, since2) = cluster_info.get_epoch_slots_since(since.clone());
assert!(slots.is_empty());
assert_eq!(since2, since);
}
#[test]
fn test_add_entrypoint() {
let node_keypair = Arc::new(Keypair::new());
@@ -2317,9 +2363,20 @@ mod tests {
#[test]
fn test_split_messages_large() {
let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
let mut btree_slots = BTreeSet::new();
for i in 0..128 {
btree_slots.insert(i);
}
let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
0,
LowestSlot::new(Pubkey::default(), 0, 0),
EpochSlots {
from: Pubkey::default(),
root: 0,
lowest: 0,
slots: btree_slots,
stash: vec![],
wallclock: 0,
},
));
test_split_messages(value);
}
@@ -2331,19 +2388,39 @@ mod tests {
let payload: Vec<CrdsValue> = vec![];
let vec_size = serialized_size(&payload).unwrap();
let desired_size = MAX_PROTOCOL_PAYLOAD_SIZE - vec_size;
let mut value = CrdsValue::new_unsigned(CrdsData::SnapshotHashes(SnapshotHash {
from: Pubkey::default(),
hashes: vec![],
wallclock: 0,
}));
let mut value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
0,
EpochSlots {
from: Pubkey::default(),
root: 0,
lowest: 0,
slots: BTreeSet::new(),
stash: vec![],
wallclock: 0,
},
));
let mut i = 0;
while value.size() <= desired_size {
value.data = CrdsData::SnapshotHashes(SnapshotHash {
from: Pubkey::default(),
hashes: vec![(0, Hash::default()); i],
wallclock: 0,
});
let slots = (0..i).collect::<BTreeSet<_>>();
if slots.len() > 200 {
panic!(
"impossible to match size: last {:?} vs desired {:?}",
serialized_size(&value).unwrap(),
desired_size
);
}
value.data = CrdsData::EpochSlots(
0,
EpochSlots {
from: Pubkey::default(),
root: 0,
lowest: 0,
slots,
stash: vec![],
wallclock: 0,
},
);
i += 1;
}
let split = ClusterInfo::split_gossip_messages(vec![value.clone()]);
@@ -2415,8 +2492,7 @@ mod tests {
stakes.insert(id4, 10);
let stakes = Arc::new(stakes);
let mut peers = cluster_info.tvu_peers();
let peers_and_stakes = stake_weight_peers(&mut peers, Some(stakes));
let (peers, peers_and_stakes) = cluster_info.sorted_tvu_peers_and_stakes(Some(stakes));
assert_eq!(peers.len(), 2);
assert_eq!(peers[0].id, id);
assert_eq!(peers[1].id, id2);
@@ -2474,17 +2550,26 @@ mod tests {
node_keypair,
);
for i in 0..10 {
let mut peer_root = 5;
let mut peer_lowest = 0;
if i >= 5 {
// make these invalid for the upcoming repair request
peer_root = 15;
peer_lowest = 10;
}
let other_node_pubkey = Pubkey::new_rand();
let other_node = ContactInfo::new_localhost(&other_node_pubkey, timestamp());
cluster_info.insert_info(other_node.clone());
let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
0,
LowestSlot::new(other_node_pubkey, peer_lowest, timestamp()),
EpochSlots::new(
other_node_pubkey,
peer_root,
peer_lowest,
BTreeSet::new(),
vec![],
timestamp(),
),
));
let _ = cluster_info.gossip.crds.insert(value, timestamp());
}
@@ -2494,8 +2579,7 @@ mod tests {
#[test]
fn test_max_bloom_size() {
// check that the constant fits into the dynamic size
assert!(MAX_BLOOM_SIZE <= max_bloom_size());
assert_eq!(MAX_BLOOM_SIZE, max_bloom_size());
}
#[test]
@@ -2548,24 +2632,36 @@ mod tests {
}
#[test]
fn test_push_epoch_slots_large() {
use rand::Rng;
let node_keypair = Arc::new(Keypair::new());
let mut cluster_info = ClusterInfo::new(
ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
node_keypair,
fn test_compress_incomplete_slots() {
let mut incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
assert_eq!(
EpochIncompleteSlots::default(),
ClusterInfo::compress_incomplete_slots(&incomplete_slots)
);
let mut range: Vec<Slot> = vec![];
//random should be hard to compress
for _ in 0..32000 {
let last = *range.last().unwrap_or(&0);
range.push(last + rand::thread_rng().gen_range(1, 32));
}
cluster_info.push_epoch_slots(&range[..16000]);
cluster_info.push_epoch_slots(&range[16000..]);
let (slots, since) = cluster_info.get_epoch_slots_since(None);
let slots: Vec<_> = slots.iter().flat_map(|x| x.to_slots(0)).collect();
assert_eq!(slots, range);
assert!(since.is_some());
incomplete_slots.insert(100);
let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
assert_eq!(100, compressed.first);
let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
assert_eq!(incomplete_slots, decompressed);
incomplete_slots.insert(104);
let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
assert_eq!(100, compressed.first);
let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
assert_eq!(incomplete_slots, decompressed);
incomplete_slots.insert(80);
let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
assert_eq!(80, compressed.first);
let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
assert_eq!(incomplete_slots, decompressed);
incomplete_slots.insert(10000);
let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
assert_eq!(80, compressed.first);
let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
assert_eq!(incomplete_slots, decompressed);
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,348 +0,0 @@
use crate::{
cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
serve_repair::RepairType,
};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::epoch_stakes::NodeIdToVoteAccounts;
use solana_sdk::{clock::Slot, pubkey::Pubkey};
use std::{
collections::{HashMap, HashSet},
sync::{Arc, RwLock},
};
pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
#[derive(Default)]
pub struct ClusterSlots {
cluster_slots: ClusterSlotsMap,
keys: RwLock<HashSet<Arc<Pubkey>>>,
since: RwLock<Option<u64>>,
validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
epoch: RwLock<Option<u64>>,
self_id: RwLock<Pubkey>,
}
impl ClusterSlots {
pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
self.cluster_slots.read().unwrap().get(&slot).cloned()
}
pub fn update(
&self,
root: Slot,
cluster_info: &RwLock<ClusterInfo>,
bank_forks: &RwLock<BankForks>,
) {
self.update_peers(cluster_info, bank_forks);
let since = *self.since.read().unwrap();
let epoch_slots = cluster_info.read().unwrap().get_epoch_slots_since(since);
self.update_internal(root, epoch_slots);
}
fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
let (epoch_slots_list, since) = epoch_slots;
for epoch_slots in epoch_slots_list {
let slots = epoch_slots.to_slots(root);
for slot in &slots {
if *slot <= root {
continue;
}
let pubkey = Arc::new(epoch_slots.from);
let exists = self.keys.read().unwrap().get(&pubkey).is_some();
if !exists {
self.keys.write().unwrap().insert(pubkey.clone());
}
let from = self.keys.read().unwrap().get(&pubkey).unwrap().clone();
let balance = self
.validator_stakes
.read()
.unwrap()
.get(&from)
.map(|v| v.total_stake)
.unwrap_or(0);
let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(slot).cloned();
if slot_pubkeys.is_none() {
let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
self.cluster_slots
.write()
.unwrap()
.insert(*slot, new_slot_pubkeys.clone());
slot_pubkeys = Some(new_slot_pubkeys);
}
slot_pubkeys
.unwrap()
.write()
.unwrap()
.insert(from.clone(), balance);
}
}
self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
self.keys
.write()
.unwrap()
.retain(|x| Arc::strong_count(x) > 1);
*self.since.write().unwrap() = since;
}
pub fn collect(&self, id: &Pubkey) -> HashSet<Slot> {
self.cluster_slots
.read()
.unwrap()
.iter()
.filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
.map(|(slot, _)| slot)
.cloned()
.collect()
}
fn update_peers(&self, cluster_info: &RwLock<ClusterInfo>, bank_forks: &RwLock<BankForks>) {
let root_bank = bank_forks.read().unwrap().root_bank().clone();
let root_epoch = root_bank.epoch();
let my_epoch = *self.epoch.read().unwrap();
if Some(root_epoch) != my_epoch {
let validator_stakes = root_bank
.epoch_stakes(root_epoch)
.expect(
"Bank must have epoch stakes
for its own epoch",
)
.node_id_to_vote_accounts()
.clone();
*self.validator_stakes.write().unwrap() = validator_stakes;
let id = cluster_info.read().unwrap().id();
*self.self_id.write().unwrap() = id;
*self.epoch.write().unwrap() = Some(root_epoch);
}
}
pub fn compute_weights(&self, slot: Slot, repair_peers: &[ContactInfo]) -> Vec<(u64, usize)> {
let slot_peers = self.lookup(slot);
repair_peers
.iter()
.enumerate()
.map(|(i, x)| {
let peer_stake = slot_peers
.as_ref()
.and_then(|v| v.read().unwrap().get(&x.id).cloned())
.unwrap_or(0);
(
1 + peer_stake
+ self
.validator_stakes
.read()
.unwrap()
.get(&x.id)
.map(|v| v.total_stake)
.unwrap_or(0),
i,
)
})
.collect()
}
pub fn generate_repairs_for_missing_slots(
&self,
self_id: &Pubkey,
root: Slot,
) -> Vec<RepairType> {
let my_slots = self.collect(self_id);
self.cluster_slots
.read()
.unwrap()
.keys()
.filter(|x| **x > root)
.filter(|x| !my_slots.contains(*x))
.map(|x| RepairType::HighestShred(*x, 0))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use solana_runtime::epoch_stakes::NodeVoteAccounts;
#[test]
fn test_default() {
let cs = ClusterSlots::default();
assert!(cs.cluster_slots.read().unwrap().is_empty());
assert!(cs.since.read().unwrap().is_none());
}
#[test]
fn test_update_noop() {
let cs = ClusterSlots::default();
cs.update_internal(0, (vec![], None));
assert!(cs.cluster_slots.read().unwrap().is_empty());
assert!(cs.since.read().unwrap().is_none());
}
#[test]
fn test_update_empty() {
let cs = ClusterSlots::default();
let epoch_slot = EpochSlots::default();
cs.update_internal(0, (vec![epoch_slot], Some(0)));
assert_eq!(*cs.since.read().unwrap(), Some(0));
assert!(cs.lookup(0).is_none());
}
#[test]
fn test_update_rooted() {
//root is 0, so it should clear out the slot
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[0], 0);
cs.update_internal(0, (vec![epoch_slot], Some(0)));
assert_eq!(*cs.since.read().unwrap(), Some(0));
assert!(cs.lookup(0).is_none());
}
#[test]
fn test_update_new_slot() {
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[1], 0);
cs.update_internal(0, (vec![epoch_slot], Some(0)));
assert_eq!(*cs.since.read().unwrap(), Some(0));
assert!(cs.lookup(0).is_none());
assert!(cs.lookup(1).is_some());
assert_eq!(
cs.lookup(1)
.unwrap()
.read()
.unwrap()
.get(&Pubkey::default()),
Some(&0)
);
}
#[test]
fn test_compute_weights() {
let cs = ClusterSlots::default();
let ci = ContactInfo::default();
assert_eq!(cs.compute_weights(0, &[ci]), vec![(1, 0)]);
}
#[test]
fn test_best_peer_2() {
let cs = ClusterSlots::default();
let mut c1 = ContactInfo::default();
let mut c2 = ContactInfo::default();
let mut map = HashMap::new();
let k1 = Pubkey::new_rand();
let k2 = Pubkey::new_rand();
map.insert(Arc::new(k1.clone()), std::u64::MAX / 2);
map.insert(Arc::new(k2.clone()), 0);
cs.cluster_slots
.write()
.unwrap()
.insert(0, Arc::new(RwLock::new(map)));
c1.id = k1;
c2.id = k2;
assert_eq!(
cs.compute_weights(0, &[c1, c2]),
vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
);
}
#[test]
fn test_best_peer_3() {
let cs = ClusterSlots::default();
let mut c1 = ContactInfo::default();
let mut c2 = ContactInfo::default();
let mut map = HashMap::new();
let k1 = Pubkey::new_rand();
let k2 = Pubkey::new_rand();
map.insert(Arc::new(k2.clone()), 0);
cs.cluster_slots
.write()
.unwrap()
.insert(0, Arc::new(RwLock::new(map)));
//make sure default weights are used as well
let validator_stakes: HashMap<_, _> = vec![(
*Arc::new(k1.clone()),
NodeVoteAccounts {
total_stake: std::u64::MAX / 2,
vote_accounts: vec![Pubkey::default()],
},
)]
.into_iter()
.collect();
*cs.validator_stakes.write().unwrap() = Arc::new(validator_stakes);
c1.id = k1;
c2.id = k2;
assert_eq!(
cs.compute_weights(0, &[c1, c2]),
vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
);
}
#[test]
fn test_update_new_staked_slot() {
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[1], 0);
let map = Arc::new(
vec![(
Pubkey::default(),
NodeVoteAccounts {
total_stake: 1,
vote_accounts: vec![Pubkey::default()],
},
)]
.into_iter()
.collect(),
);
*cs.validator_stakes.write().unwrap() = map;
cs.update_internal(0, (vec![epoch_slot], None));
assert!(cs.lookup(1).is_some());
assert_eq!(
cs.lookup(1)
.unwrap()
.read()
.unwrap()
.get(&Pubkey::default()),
Some(&1)
);
}
#[test]
fn test_generate_repairs() {
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[1], 0);
cs.update_internal(0, (vec![epoch_slot], None));
let self_id = Pubkey::new_rand();
assert_eq!(
cs.generate_repairs_for_missing_slots(&self_id, 0),
vec![RepairType::HighestShred(1, 0)]
)
}
#[test]
fn test_collect_my_slots() {
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[1], 0);
let self_id = epoch_slot.from;
cs.update_internal(0, (vec![epoch_slot], None));
let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
assert_eq!(slots, vec![1]);
}
#[test]
fn test_generate_repairs_existing() {
let cs = ClusterSlots::default();
let mut epoch_slot = EpochSlots::default();
epoch_slot.fill(&[1], 0);
let self_id = epoch_slot.from;
cs.update_internal(0, (vec![epoch_slot], None));
assert!(cs
.generate_repairs_for_missing_slots(&self_id, 0)
.is_empty());
}
}

View File

@@ -290,7 +290,7 @@ impl AggregateCommitmentService {
#[cfg(test)]
mod tests {
use super::*;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_sdk::pubkey::Pubkey;
use solana_stake_program::stake_state;
use solana_vote_program::vote_state::{self, VoteStateVersions};

View File

@@ -1,4 +1,3 @@
use crate::progress_map::ProgressMap;
use chrono::prelude::*;
use solana_ledger::bank_forks::BankForks;
use solana_runtime::bank::Bank;
@@ -321,7 +320,7 @@ impl Tower {
}
pub fn check_vote_stake_threshold(
&self,
slot: Slot,
slot: u64,
stake_lockouts: &HashMap<u64, StakeLockout>,
total_staked: u64,
) -> bool {
@@ -332,8 +331,11 @@ impl Tower {
if let Some(fork_stake) = stake_lockouts.get(&vote.slot) {
let lockout = fork_stake.stake as f64 / total_staked as f64;
trace!(
"fork_stake slot: {}, vote slot: {}, lockout: {} fork_stake: {} total_stake: {}",
slot, vote.slot, lockout, fork_stake.stake, total_staked
"fork_stake slot: {} lockout: {} fork_stake: {} total_stake: {}",
slot,
lockout,
fork_stake.stake,
total_staked
);
if vote.confirmation_count as usize > self.threshold_depth {
for old_vote in &self.lockouts.votes {
@@ -353,18 +355,6 @@ impl Tower {
}
}
pub(crate) fn check_switch_threshold(
&self,
_slot: Slot,
_ancestors: &HashMap<Slot, HashSet<u64>>,
_descendants: &HashMap<Slot, HashSet<u64>>,
_progress: &ProgressMap,
_total_epoch_stake: u64,
_epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
) -> bool {
true
}
/// Update lockouts for all the ancestors
fn update_ancestor_lockouts(
stake_lockouts: &mut HashMap<Slot, StakeLockout>,
@@ -478,12 +468,7 @@ impl Tower {
#[cfg(test)]
pub mod test {
use super::*;
use crate::{
cluster_info_vote_listener::VoteTracker,
cluster_slots::ClusterSlots,
progress_map::ForkProgress,
replay_stage::{HeaviestForkFailures, ReplayStage},
};
use crate::replay_stage::{ForkProgress, ReplayStage};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::{
bank::Bank,
@@ -496,96 +481,107 @@ pub mod test {
hash::Hash,
pubkey::Pubkey,
signature::{Keypair, Signer},
transaction::Transaction,
};
use solana_vote_program::{
vote_instruction,
vote_state::{Vote, VoteStateVersions},
vote_transaction,
};
use std::collections::HashMap;
use std::collections::{HashMap, VecDeque};
use std::sync::RwLock;
use std::{thread::sleep, time::Duration};
use trees::{tr, Tree, TreeWalk};
use trees::{tr, Node, Tree};
pub(crate) struct VoteSimulator {
pub validator_keypairs: HashMap<Pubkey, ValidatorVoteKeypairs>,
pub node_pubkeys: Vec<Pubkey>,
pub vote_pubkeys: Vec<Pubkey>,
pub bank_forks: RwLock<BankForks>,
pub progress: ProgressMap,
pub(crate) struct VoteSimulator<'a> {
searchable_nodes: HashMap<u64, &'a Node<u64>>,
}
impl VoteSimulator {
pub(crate) fn new(num_keypairs: usize) -> Self {
let (validator_keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress) =
Self::init_state(num_keypairs);
Self {
validator_keypairs,
node_pubkeys,
vote_pubkeys,
bank_forks: RwLock::new(bank_forks),
progress,
}
}
pub(crate) fn fill_bank_forks(
&mut self,
forks: Tree<u64>,
cluster_votes: &HashMap<Pubkey, Vec<u64>>,
) {
let root = forks.root().data;
assert!(self.bank_forks.read().unwrap().get(root).is_some());
let mut walk = TreeWalk::from(forks);
loop {
if let Some(visit) = walk.get() {
let slot = visit.node().data;
self.progress
.entry(slot)
.or_insert_with(|| ForkProgress::new(Hash::default(), None, None, 0, 0));
if self.bank_forks.read().unwrap().get(slot).is_some() {
walk.forward();
continue;
}
let parent = walk.get_parent().unwrap().data;
let parent_bank = self.bank_forks.read().unwrap().get(parent).unwrap().clone();
let new_bank = Bank::new_from_parent(&parent_bank, &Pubkey::default(), slot);
for (pubkey, vote) in cluster_votes.iter() {
if vote.contains(&parent) {
let keypairs = self.validator_keypairs.get(pubkey).unwrap();
let last_blockhash = parent_bank.last_blockhash();
let vote_tx = vote_transaction::new_vote_transaction(
// Must vote > root to be processed
vec![parent],
parent_bank.hash(),
last_blockhash,
&keypairs.node_keypair,
&keypairs.vote_keypair,
&keypairs.vote_keypair,
);
info!("voting {} {}", parent_bank.slot(), parent_bank.hash());
new_bank.process_transaction(&vote_tx).unwrap();
}
}
new_bank.freeze();
self.bank_forks.write().unwrap().insert(new_bank);
walk.forward();
} else {
break;
}
}
impl<'a> VoteSimulator<'a> {
pub(crate) fn new(forks: &'a Tree<u64>) -> Self {
let mut searchable_nodes = HashMap::new();
let root = forks.root();
searchable_nodes.insert(root.data, root);
Self { searchable_nodes }
}
pub(crate) fn simulate_vote(
&mut self,
vote_slot: Slot,
my_pubkey: &Pubkey,
bank_forks: &RwLock<BankForks>,
cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
validator_keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
my_keypairs: &ValidatorVoteKeypairs,
progress: &mut HashMap<u64, ForkProgress>,
tower: &mut Tower,
) -> Vec<HeaviestForkFailures> {
// Try to simulate the vote
let my_keypairs = self.validator_keypairs.get(&my_pubkey).unwrap();
) -> Vec<VoteFailures> {
let node = self
.find_node_and_update_simulation(vote_slot)
.expect("Vote to simulate must be for a slot in the tree");
let mut missing_nodes = VecDeque::new();
let mut current = node;
loop {
let current_slot = current.data;
if bank_forks.read().unwrap().get(current_slot).is_some()
|| tower.root().map(|r| current_slot < r).unwrap_or(false)
{
break;
} else {
missing_nodes.push_front(current);
}
if let Some(parent) = current.parent() {
current = parent;
} else {
break;
}
}
// Create any missing banks along the path
for missing_node in missing_nodes {
let missing_slot = missing_node.data;
let parent = missing_node.parent().unwrap().data;
let parent_bank = bank_forks
.read()
.unwrap()
.get(parent)
.expect("parent bank must exist")
.clone();
info!("parent of {} is {}", missing_slot, parent_bank.slot(),);
progress
.entry(missing_slot)
.or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash()));
// Create the missing bank
let new_bank =
Bank::new_from_parent(&parent_bank, &Pubkey::default(), missing_slot);
// Simulate ingesting the cluster's votes for the parent into this bank
for (pubkey, vote) in cluster_votes.iter() {
if vote.contains(&parent_bank.slot()) {
let keypairs = validator_keypairs.get(pubkey).unwrap();
let node_pubkey = keypairs.node_keypair.pubkey();
let vote_pubkey = keypairs.vote_keypair.pubkey();
let last_blockhash = parent_bank.last_blockhash();
let votes = Vote::new(vec![parent_bank.slot()], parent_bank.hash());
info!("voting {} {}", parent_bank.slot(), parent_bank.hash());
let vote_ix = vote_instruction::vote(&vote_pubkey, &vote_pubkey, votes);
let mut vote_tx =
Transaction::new_with_payer(vec![vote_ix], Some(&node_pubkey));
vote_tx.partial_sign(&[&keypairs.node_keypair], last_blockhash);
vote_tx.partial_sign(&[&keypairs.vote_keypair], last_blockhash);
new_bank.process_transaction(&vote_tx).unwrap();
}
}
new_bank.freeze();
bank_forks.write().unwrap().insert(new_bank);
}
// Now try to simulate the vote
let my_pubkey = my_keypairs.node_keypair.pubkey();
let my_vote_pubkey = my_keypairs.vote_keypair.pubkey();
let ancestors = self.bank_forks.read().unwrap().ancestors();
let mut frozen_banks: Vec<_> = self
.bank_forks
let ancestors = bank_forks.read().unwrap().ancestors();
let mut frozen_banks: Vec<_> = bank_forks
.read()
.unwrap()
.frozen_banks()
@@ -598,132 +594,99 @@ pub mod test {
&ancestors,
&mut frozen_banks,
tower,
&mut self.progress,
&VoteTracker::default(),
&ClusterSlots::default(),
&self.bank_forks,
&mut HashSet::new(),
progress,
);
let vote_bank = self
.bank_forks
let bank = bank_forks
.read()
.unwrap()
.get(vote_slot)
.expect("Bank must have been created before vote simulation")
.clone();
// Try to vote on the given slot
let descendants = self.bank_forks.read().unwrap().descendants();
let (_, _, failure_reasons) = ReplayStage::select_vote_and_reset_forks(
&Some(vote_bank.clone()),
&None,
&ancestors,
&descendants,
&self.progress,
&tower,
);
// Make sure this slot isn't locked out or failing threshold
info!("Checking vote: {}", vote_bank.slot());
if !failure_reasons.is_empty() {
return failure_reasons;
let fork_progress = progress
.get(&vote_slot)
.expect("Slot for vote must exist in progress map");
info!("Checking vote: {}", vote_slot);
info!("lockouts: {:?}", fork_progress.fork_stats.stake_lockouts);
let mut failures = vec![];
if fork_progress.fork_stats.is_locked_out {
failures.push(VoteFailures::LockedOut(vote_slot));
}
let vote = tower.new_vote_from_bank(&vote_bank, &my_vote_pubkey).0;
if !fork_progress.fork_stats.vote_threshold {
failures.push(VoteFailures::FailedThreshold(vote_slot));
}
if !failures.is_empty() {
return failures;
}
let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
if let Some(new_root) = tower.record_bank_vote(vote) {
ReplayStage::handle_new_root(
new_root,
&self.bank_forks,
&mut self.progress,
&None,
&mut HashSet::new(),
);
ReplayStage::handle_new_root(new_root, bank_forks, progress, &None);
}
// Mark the vote for this bank under this node's pubkey so it will be
// integrated into any future child banks
cluster_votes.entry(my_pubkey).or_default().push(vote_slot);
vec![]
}
fn can_progress_on_fork(
&mut self,
my_pubkey: &Pubkey,
tower: &mut Tower,
start_slot: u64,
num_slots: u64,
cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
) -> bool {
// Check that within some reasonable time, validator can make a new
// root on this fork
let old_root = tower.root();
for i in 1..num_slots {
// The parent of the tip of the fork
let mut fork_tip_parent = tr(start_slot + i - 1);
// The tip of the fork
fork_tip_parent.push_front(tr(start_slot + i));
self.fill_bank_forks(fork_tip_parent, cluster_votes);
if self
.simulate_vote(i + start_slot, &my_pubkey, tower)
.is_empty()
{
cluster_votes
.entry(*my_pubkey)
.or_default()
.push(start_slot + i);
}
if old_root != tower.root() {
return true;
// Find a node representing the given slot
fn find_node_and_update_simulation(&mut self, slot: u64) -> Option<&'a Node<u64>> {
let mut successful_search_node: Option<&'a Node<u64>> = None;
let mut found_node = None;
for search_node in self.searchable_nodes.values() {
if let Some((target, new_searchable_nodes)) = Self::find_node(search_node, slot) {
successful_search_node = Some(search_node);
found_node = Some(target);
for node in new_searchable_nodes {
self.searchable_nodes.insert(node.data, node);
}
break;
}
}
false
successful_search_node.map(|node| {
self.searchable_nodes.remove(&node.data);
});
found_node
}
fn init_state(
num_keypairs: usize,
) -> (
HashMap<Pubkey, ValidatorVoteKeypairs>,
Vec<Pubkey>,
Vec<Pubkey>,
BankForks,
ProgressMap,
) {
let keypairs: HashMap<_, _> = std::iter::repeat_with(|| {
let node_keypair = Keypair::new();
let vote_keypair = Keypair::new();
let stake_keypair = Keypair::new();
let node_pubkey = node_keypair.pubkey();
(
node_pubkey,
ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
)
})
.take(num_keypairs)
.collect();
let node_pubkeys: Vec<_> = keypairs
.values()
.map(|keys| keys.node_keypair.pubkey())
.collect();
let vote_pubkeys: Vec<_> = keypairs
.values()
.map(|keys| keys.vote_keypair.pubkey())
.collect();
fn find_node(
node: &'a Node<u64>,
slot: u64,
) -> Option<(&'a Node<u64>, Vec<&'a Node<u64>>)> {
if node.data == slot {
Some((node, node.iter().collect()))
} else {
let mut search_result: Option<(&'a Node<u64>, Vec<&'a Node<u64>>)> = None;
for child in node.iter() {
if let Some((_, ref mut new_searchable_nodes)) = search_result {
new_searchable_nodes.push(child);
continue;
}
search_result = Self::find_node(child, slot);
}
let (bank_forks, progress) = initialize_state(&keypairs, 10_000);
(keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress)
search_result
}
}
}
#[derive(PartialEq, Debug)]
pub(crate) enum VoteFailures {
LockedOut(u64),
FailedThreshold(u64),
}
// Setup BankForks with bank 0 and all the validator accounts
pub(crate) fn initialize_state(
validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
stake: u64,
) -> (BankForks, ProgressMap) {
) -> (BankForks, HashMap<u64, ForkProgress>) {
let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
let GenesisConfigInfo {
genesis_config,
mint_keypair,
voting_keypair: _,
} = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs, stake);
} = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs);
let bank0 = Bank::new(&genesis_config);
@@ -732,11 +695,8 @@ pub mod test {
}
bank0.freeze();
let mut progress = ProgressMap::default();
progress.insert(
0,
ForkProgress::new(bank0.last_blockhash(), None, None, 0, 0),
);
let mut progress = HashMap::new();
progress.insert(0, ForkProgress::new(bank0.last_blockhash()));
(BankForks::new(0, bank0), progress)
}
@@ -760,26 +720,84 @@ pub mod test {
stakes
}
fn can_progress_on_fork(
my_pubkey: &Pubkey,
tower: &mut Tower,
start_slot: u64,
num_slots: u64,
bank_forks: &RwLock<BankForks>,
cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
progress: &mut HashMap<u64, ForkProgress>,
) -> bool {
// Check that within some reasonable time, validator can make a new
// root on this fork
let old_root = tower.root();
let mut main_fork = tr(start_slot);
let mut tip = main_fork.root_mut();
for i in 1..num_slots {
tip.push_front(tr(start_slot + i));
tip = tip.first_mut().unwrap();
}
let mut voting_simulator = VoteSimulator::new(&main_fork);
for i in 1..num_slots {
voting_simulator.simulate_vote(
i + start_slot,
&bank_forks,
cluster_votes,
&keypairs,
keypairs.get(&my_pubkey).unwrap(),
progress,
tower,
);
if old_root != tower.root() {
return true;
}
}
false
}
#[test]
fn test_simple_votes() {
// Init state
let mut vote_simulator = VoteSimulator::new(1);
let node_pubkey = vote_simulator.node_pubkeys[0];
let mut tower = Tower::new_with_key(&node_pubkey);
let node_keypair = Keypair::new();
let vote_keypair = Keypair::new();
let stake_keypair = Keypair::new();
let node_pubkey = node_keypair.pubkey();
let mut keypairs = HashMap::new();
keypairs.insert(
node_pubkey,
ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
);
// Initialize BankForks
let (bank_forks, mut progress) = initialize_state(&keypairs);
let bank_forks = RwLock::new(bank_forks);
// Create the tree of banks
let forks = tr(0) / (tr(1) / (tr(2) / (tr(3) / (tr(4) / tr(5)))));
// Set the voting behavior
let mut cluster_votes = HashMap::new();
let mut voting_simulator = VoteSimulator::new(&forks);
let votes = vec![0, 1, 2, 3, 4, 5];
cluster_votes.insert(node_pubkey, votes.clone());
vote_simulator.fill_bank_forks(forks, &cluster_votes);
// Simulate the votes
let mut tower = Tower::new_with_key(&node_pubkey);
let mut cluster_votes = HashMap::new();
for vote in votes {
assert!(vote_simulator
.simulate_vote(vote, &node_pubkey, &mut tower,)
assert!(voting_simulator
.simulate_vote(
vote,
&bank_forks,
&mut cluster_votes,
&keypairs,
keypairs.get(&node_pubkey).unwrap(),
&mut progress,
&mut tower,
)
.is_empty());
}
@@ -791,14 +809,21 @@ pub mod test {
#[test]
fn test_double_partition() {
// Init state
let mut vote_simulator = VoteSimulator::new(2);
let node_pubkey = vote_simulator.node_pubkeys[0];
let vote_pubkey = vote_simulator.vote_pubkeys[0];
let mut tower = Tower::new_with_key(&node_pubkey);
solana_logger::setup();
let node_keypair = Keypair::new();
let vote_keypair = Keypair::new();
let stake_keypair = Keypair::new();
let node_pubkey = node_keypair.pubkey();
let vote_pubkey = vote_keypair.pubkey();
let num_slots_to_try = 200;
// Create the tree of banks
let mut keypairs = HashMap::new();
info!("my_pubkey: {}", node_pubkey);
keypairs.insert(
node_pubkey,
ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
);
// Create the tree of banks in a BankForks object
let forks = tr(0)
/ (tr(1)
/ (tr(2)
@@ -815,37 +840,56 @@ pub mod test {
/ (tr(44)
// Minor fork 2
/ (tr(45) / (tr(46) / (tr(47) / (tr(48) / (tr(49) / (tr(50)))))))
/ (tr(110) / (tr(110 + 2 * num_slots_to_try))))))))))))));
/ (tr(110)))))))))))));
// Set the successful voting behavior
let mut cluster_votes = HashMap::new();
let mut my_votes: Vec<Slot> = vec![];
let next_unlocked_slot = 110;
// Set the voting behavior
let mut voting_simulator = VoteSimulator::new(&forks);
let mut votes: Vec<Slot> = vec![];
// Vote on the first minor fork
my_votes.extend((0..=14).into_iter());
votes.extend((0..=14).into_iter());
// Come back to the main fork
my_votes.extend((43..=44).into_iter());
votes.extend((43..=44).into_iter());
// Vote on the second minor fork
my_votes.extend((45..=50).into_iter());
// Vote to come back to main fork
my_votes.push(next_unlocked_slot);
cluster_votes.insert(node_pubkey, my_votes.clone());
// Make the other validator vote fork to pass the threshold checks
let other_votes = my_votes.clone();
cluster_votes.insert(vote_simulator.node_pubkeys[1], other_votes);
vote_simulator.fill_bank_forks(forks, &cluster_votes);
votes.extend((45..=50).into_iter());
// Simulate the votes.
for vote in &my_votes {
let mut cluster_votes: HashMap<Pubkey, Vec<Slot>> = HashMap::new();
let (bank_forks, mut progress) = initialize_state(&keypairs);
let bank_forks = RwLock::new(bank_forks);
// Simulate the votes. Should fail on trying to come back to the main fork
// at 106 exclusively due to threshold failure
let mut tower = Tower::new_with_key(&node_pubkey);
for vote in &votes {
// All these votes should be ok
assert!(vote_simulator
.simulate_vote(*vote, &node_pubkey, &mut tower,)
assert!(voting_simulator
.simulate_vote(
*vote,
&bank_forks,
&mut cluster_votes,
&keypairs,
keypairs.get(&node_pubkey).unwrap(),
&mut progress,
&mut tower,
)
.is_empty());
}
// Try to come back to main fork
let next_unlocked_slot = 110;
assert!(voting_simulator
.simulate_vote(
next_unlocked_slot,
&bank_forks,
&mut cluster_votes,
&keypairs,
keypairs.get(&node_pubkey).unwrap(),
&mut progress,
&mut tower,
)
.is_empty());
info!("local tower: {:#?}", tower.lockouts.votes);
let vote_accounts = vote_simulator
.bank_forks
let vote_accounts = bank_forks
.read()
.unwrap()
.get(next_unlocked_slot)
@@ -855,17 +899,15 @@ pub mod test {
let state = VoteState::from(&observed.1).unwrap();
info!("observed tower: {:#?}", state.votes);
let num_slots_to_try = 200;
cluster_votes
.get_mut(&vote_simulator.node_pubkeys[1])
.unwrap()
.extend(next_unlocked_slot + 1..next_unlocked_slot + num_slots_to_try);
assert!(vote_simulator.can_progress_on_fork(
assert!(can_progress_on_fork(
&node_pubkey,
&mut tower,
next_unlocked_slot,
num_slots_to_try,
200,
&bank_forks,
&mut cluster_votes,
&keypairs,
&mut progress
));
}

View File

@@ -1,6 +1,4 @@
use crate::contact_info::ContactInfo;
use crate::deprecated;
use crate::epoch_slots::EpochSlots;
use bincode::{serialize, serialized_size};
use solana_sdk::timing::timestamp;
use solana_sdk::{
@@ -19,8 +17,7 @@ use std::{
pub type VoteIndex = u8;
pub const MAX_VOTES: VoteIndex = 32;
pub type EpochSlotsIndex = u8;
pub const MAX_EPOCH_SLOTS: EpochSlotsIndex = 255;
pub type EpochSlotIndex = u8;
/// CrdsValue that is replicated across the cluster
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -52,7 +49,6 @@ impl Signable for CrdsValue {
.verify(&self.pubkey().as_ref(), self.signable_data().borrow());
let data_check = match &self.data {
CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
CrdsData::EpochSlots(ix, _) => *ix < MAX_EPOCH_SLOTS,
_ => true,
};
sig_check && data_check
@@ -61,16 +57,34 @@ impl Signable for CrdsValue {
/// CrdsData that defines the different types of items CrdsValues can hold
/// * Merge Strategy - Latest wallclock is picked
/// * LowestSlot index is deprecated
#[allow(clippy::large_enum_variant)]
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub enum CrdsData {
ContactInfo(ContactInfo),
Vote(VoteIndex, Vote),
LowestSlot(u8, LowestSlot),
EpochSlots(EpochSlotIndex, EpochSlots),
SnapshotHashes(SnapshotHash),
AccountsHashes(SnapshotHash),
EpochSlots(EpochSlotsIndex, EpochSlots),
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub enum CompressionType {
Uncompressed,
GZip,
BZip2,
}
impl Default for CompressionType {
fn default() -> Self {
Self::Uncompressed
}
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct EpochIncompleteSlots {
pub first: Slot,
pub compression: CompressionType,
pub compressed_list: Vec<u8>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -89,24 +103,32 @@ impl SnapshotHash {
}
}
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub struct LowestSlot {
pub struct EpochSlots {
pub from: Pubkey,
root: Slot, //deprecated
pub root: Slot,
pub lowest: Slot,
slots: BTreeSet<Slot>, //deprecated
stash: Vec<deprecated::EpochIncompleteSlots>, //deprecated
pub slots: BTreeSet<Slot>,
pub stash: Vec<EpochIncompleteSlots>,
pub wallclock: u64,
}
impl LowestSlot {
pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
impl EpochSlots {
pub fn new(
from: Pubkey,
root: Slot,
lowest: Slot,
slots: BTreeSet<Slot>,
stash: Vec<EpochIncompleteSlots>,
wallclock: u64,
) -> Self {
Self {
from,
root: 0,
root,
lowest,
slots: BTreeSet::new(),
stash: vec![],
slots,
stash,
wallclock,
}
}
@@ -135,9 +157,8 @@ impl Vote {
pub enum CrdsValueLabel {
ContactInfo(Pubkey),
Vote(VoteIndex, Pubkey),
LowestSlot(Pubkey),
EpochSlots(Pubkey),
SnapshotHashes(Pubkey),
EpochSlots(EpochSlotsIndex, Pubkey),
AccountsHashes(Pubkey),
}
@@ -146,9 +167,8 @@ impl fmt::Display for CrdsValueLabel {
match self {
CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
CrdsValueLabel::Vote(ix, _) => write!(f, "Vote({}, {})", ix, self.pubkey()),
CrdsValueLabel::LowestSlot(_) => write!(f, "LowestSlot({})", self.pubkey()),
CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHash({})", self.pubkey()),
CrdsValueLabel::EpochSlots(ix, _) => write!(f, "EpochSlots({}, {})", ix, self.pubkey()),
CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHashes({})", self.pubkey()),
CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
}
}
@@ -159,9 +179,8 @@ impl CrdsValueLabel {
match self {
CrdsValueLabel::ContactInfo(p) => *p,
CrdsValueLabel::Vote(_, p) => *p,
CrdsValueLabel::LowestSlot(p) => *p,
CrdsValueLabel::EpochSlots(p) => *p,
CrdsValueLabel::SnapshotHashes(p) => *p,
CrdsValueLabel::EpochSlots(_, p) => *p,
CrdsValueLabel::AccountsHashes(p) => *p,
}
}
@@ -187,30 +206,27 @@ impl CrdsValue {
match &self.data {
CrdsData::ContactInfo(contact_info) => contact_info.wallclock,
CrdsData::Vote(_, vote) => vote.wallclock,
CrdsData::LowestSlot(_, obj) => obj.wallclock,
CrdsData::EpochSlots(_, vote) => vote.wallclock,
CrdsData::SnapshotHashes(hash) => hash.wallclock,
CrdsData::AccountsHashes(hash) => hash.wallclock,
CrdsData::EpochSlots(_, p) => p.wallclock,
}
}
pub fn pubkey(&self) -> Pubkey {
match &self.data {
CrdsData::ContactInfo(contact_info) => contact_info.id,
CrdsData::Vote(_, vote) => vote.from,
CrdsData::LowestSlot(_, slots) => slots.from,
CrdsData::EpochSlots(_, slots) => slots.from,
CrdsData::SnapshotHashes(hash) => hash.from,
CrdsData::AccountsHashes(hash) => hash.from,
CrdsData::EpochSlots(_, p) => p.from,
}
}
pub fn label(&self) -> CrdsValueLabel {
match &self.data {
CrdsData::ContactInfo(_) => CrdsValueLabel::ContactInfo(self.pubkey()),
CrdsData::Vote(ix, _) => CrdsValueLabel::Vote(*ix, self.pubkey()),
CrdsData::LowestSlot(_, _) => CrdsValueLabel::LowestSlot(self.pubkey()),
CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
}
}
pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -233,9 +249,9 @@ impl CrdsValue {
}
}
pub fn lowest_slot(&self) -> Option<&LowestSlot> {
pub fn epoch_slots(&self) -> Option<&EpochSlots> {
match &self.data {
CrdsData::LowestSlot(_, slots) => Some(slots),
CrdsData::EpochSlots(_, slots) => Some(slots),
_ => None,
}
}
@@ -254,23 +270,15 @@ impl CrdsValue {
}
}
pub fn epoch_slots(&self) -> Option<&EpochSlots> {
match &self.data {
CrdsData::EpochSlots(_, slots) => Some(slots),
_ => None,
}
}
/// Return all the possible labels for a record identified by Pubkey.
pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
let mut labels = vec![
CrdsValueLabel::ContactInfo(*key),
CrdsValueLabel::LowestSlot(*key),
CrdsValueLabel::EpochSlots(*key),
CrdsValueLabel::SnapshotHashes(*key),
CrdsValueLabel::AccountsHashes(*key),
];
labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
labels.extend((0..MAX_EPOCH_SLOTS).map(|ix| CrdsValueLabel::EpochSlots(ix, *key)));
labels
}
@@ -318,18 +326,15 @@ mod test {
#[test]
fn test_labels() {
let mut hits = [false; 4 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
let mut hits = [false; 4 + MAX_VOTES as usize];
// this method should cover all the possible labels
for v in &CrdsValue::record_labels(&Pubkey::default()) {
match v {
CrdsValueLabel::ContactInfo(_) => hits[0] = true,
CrdsValueLabel::LowestSlot(_) => hits[1] = true,
CrdsValueLabel::EpochSlots(_) => hits[1] = true,
CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
CrdsValueLabel::EpochSlots(ix, _) => {
hits[*ix as usize + MAX_VOTES as usize + 4] = true
}
}
}
assert!(hits.iter().all(|x| *x));
@@ -349,13 +354,13 @@ mod test {
let key = v.clone().vote().unwrap().from;
assert_eq!(v.label(), CrdsValueLabel::Vote(0, key));
let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
0,
LowestSlot::new(Pubkey::default(), 0, 0),
EpochSlots::new(Pubkey::default(), 0, 0, BTreeSet::new(), vec![], 0),
));
assert_eq!(v.wallclock(), 0);
let key = v.clone().lowest_slot().unwrap().from;
assert_eq!(v.label(), CrdsValueLabel::LowestSlot(key));
let key = v.clone().epoch_slots().unwrap().from;
assert_eq!(v.label(), CrdsValueLabel::EpochSlots(key));
}
#[test]
@@ -372,9 +377,10 @@ mod test {
Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
));
verify_signatures(&mut v, &keypair, &wrong_keypair);
v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
let btreeset: BTreeSet<Slot> = vec![1, 2, 3, 6, 8].into_iter().collect();
v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
0,
LowestSlot::new(keypair.pubkey(), 0, timestamp()),
EpochSlots::new(keypair.pubkey(), 0, 0, btreeset, vec![], timestamp()),
));
verify_signatures(&mut v, &keypair, &wrong_keypair);
}
@@ -392,18 +398,6 @@ mod test {
assert!(!vote.verify());
}
#[test]
fn test_max_epoch_slots_index() {
let keypair = Keypair::new();
let item = CrdsValue::new_signed(
CrdsData::EpochSlots(
MAX_EPOCH_SLOTS,
EpochSlots::new(keypair.pubkey(), timestamp()),
),
&keypair,
);
assert!(!item.verify());
}
#[test]
fn test_compute_vote_index_empty() {
for i in 0..MAX_VOTES {

View File

@@ -1,21 +0,0 @@
use solana_sdk::clock::Slot;
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
enum CompressionType {
Uncompressed,
GZip,
BZip2,
}
impl Default for CompressionType {
fn default() -> Self {
Self::Uncompressed
}
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub(crate) struct EpochIncompleteSlots {
first: Slot,
compression: CompressionType,
compressed_list: Vec<u8>,
}

View File

@@ -1,401 +0,0 @@
use crate::cluster_info::MAX_CRDS_OBJECT_SIZE;
use bincode::serialized_size;
use bv::BitVec;
use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress};
use solana_sdk::clock::Slot;
use solana_sdk::pubkey::Pubkey;
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub struct Uncompressed {
pub first_slot: Slot,
pub num: usize,
pub slots: BitVec<u8>,
}
#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
pub struct Flate2 {
pub first_slot: Slot,
pub num: usize,
pub compressed: Vec<u8>,
}
#[derive(Debug, PartialEq)]
pub enum Error {
CompressError,
DecompressError,
}
pub type Result<T> = std::result::Result<T, Error>;
impl std::convert::From<flate2::CompressError> for Error {
fn from(_e: flate2::CompressError) -> Error {
Error::CompressError
}
}
impl std::convert::From<flate2::DecompressError> for Error {
fn from(_e: flate2::DecompressError) -> Error {
Error::DecompressError
}
}
impl Flate2 {
fn deflate(mut unc: Uncompressed) -> Result<Self> {
let mut compressed = Vec::with_capacity(unc.slots.block_capacity());
let mut compressor = Compress::new(Compression::best(), false);
let first_slot = unc.first_slot;
let num = unc.num;
unc.slots.shrink_to_fit();
let bits = unc.slots.into_boxed_slice();
compressor.compress_vec(&bits, &mut compressed, FlushCompress::Finish)?;
let rv = Self {
first_slot,
num,
compressed,
};
let _ = rv.inflate()?;
Ok(rv)
}
pub fn inflate(&self) -> Result<Uncompressed> {
//add some head room for the decompressor which might spill more bits
let mut uncompressed = Vec::with_capacity(32 + (self.num + 4) / 8);
let mut decompress = Decompress::new(false);
decompress.decompress_vec(&self.compressed, &mut uncompressed, FlushDecompress::Finish)?;
Ok(Uncompressed {
first_slot: self.first_slot,
num: self.num,
slots: BitVec::from_bits(&uncompressed),
})
}
}
impl Uncompressed {
pub fn new(max_size: usize) -> Self {
Self {
num: 0,
first_slot: 0,
slots: BitVec::new_fill(false, 8 * max_size as u64),
}
}
pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
let mut rv = vec![];
let start = if min_slot < self.first_slot {
0 as usize
} else {
(min_slot - self.first_slot) as usize
};
for i in start..self.num {
if self.slots.get(i as u64) {
rv.push(self.first_slot + i as Slot);
}
}
rv
}
pub fn add(&mut self, slots: &[Slot]) -> usize {
for (i, s) in slots.iter().enumerate() {
if self.num == 0 {
self.first_slot = *s;
}
if *s < self.first_slot {
return i;
}
if *s - self.first_slot >= self.slots.capacity() {
return i;
}
self.slots.set(*s - self.first_slot, true);
self.num = std::cmp::max(self.num, 1 + (*s - self.first_slot) as usize);
}
slots.len()
}
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub enum CompressedSlots {
Flate2(Flate2),
Uncompressed(Uncompressed),
}
impl Default for CompressedSlots {
fn default() -> Self {
CompressedSlots::new(0)
}
}
impl CompressedSlots {
fn new(max_size: usize) -> Self {
CompressedSlots::Uncompressed(Uncompressed::new(max_size))
}
pub fn first_slot(&self) -> Slot {
match self {
CompressedSlots::Uncompressed(a) => a.first_slot,
CompressedSlots::Flate2(b) => b.first_slot,
}
}
pub fn num_slots(&self) -> usize {
match self {
CompressedSlots::Uncompressed(a) => a.num,
CompressedSlots::Flate2(b) => b.num,
}
}
pub fn add(&mut self, slots: &[Slot]) -> usize {
match self {
CompressedSlots::Uncompressed(vals) => vals.add(slots),
CompressedSlots::Flate2(_) => 0,
}
}
pub fn to_slots(&self, min_slot: Slot) -> Result<Vec<Slot>> {
match self {
CompressedSlots::Uncompressed(vals) => Ok(vals.to_slots(min_slot)),
CompressedSlots::Flate2(vals) => {
let unc = vals.inflate()?;
Ok(unc.to_slots(min_slot))
}
}
}
pub fn deflate(&mut self) -> Result<()> {
match self {
CompressedSlots::Uncompressed(vals) => {
let unc = vals.clone();
let compressed = Flate2::deflate(unc)?;
let mut new = CompressedSlots::Flate2(compressed);
std::mem::swap(self, &mut new);
Ok(())
}
CompressedSlots::Flate2(_) => Ok(()),
}
}
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct EpochSlots {
pub from: Pubkey,
pub slots: Vec<CompressedSlots>,
pub wallclock: u64,
}
impl EpochSlots {
pub fn new(from: Pubkey, now: u64) -> Self {
Self {
from,
wallclock: now,
slots: vec![],
}
}
pub fn fill(&mut self, slots: &[Slot], now: u64) -> usize {
let mut num = 0;
self.wallclock = std::cmp::max(now, self.wallclock + 1);
while num < slots.len() {
num += self.add(&slots[num..]);
if num < slots.len() {
if self.deflate().is_err() {
return num;
}
let space = self.max_compressed_slot_size();
if space > 0 {
let cslot = CompressedSlots::new(space as usize);
self.slots.push(cslot);
} else {
return num;
}
}
}
num
}
pub fn add(&mut self, slots: &[Slot]) -> usize {
let mut num = 0;
for s in &mut self.slots {
num += s.add(&slots[num..]);
if num >= slots.len() {
break;
}
}
num
}
pub fn deflate(&mut self) -> Result<()> {
for s in self.slots.iter_mut() {
s.deflate()?;
}
Ok(())
}
pub fn max_compressed_slot_size(&self) -> isize {
let len_header = serialized_size(self).unwrap();
let len_slot = serialized_size(&CompressedSlots::default()).unwrap();
MAX_CRDS_OBJECT_SIZE as isize - (len_header + len_slot) as isize
}
pub fn first_slot(&self) -> Option<Slot> {
self.slots.iter().map(|s| s.first_slot()).min()
}
pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
self.slots
.iter()
.filter(|s| min_slot < s.first_slot() + s.num_slots() as u64)
.filter_map(|s| s.to_slots(min_slot).ok())
.flatten()
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_epoch_slots_max_size() {
let epoch_slots = EpochSlots::default();
assert!(epoch_slots.max_compressed_slot_size() > 0);
}
#[test]
fn test_epoch_slots_uncompressed_add_1() {
let mut slots = Uncompressed::new(1);
assert_eq!(slots.slots.capacity(), 8);
assert_eq!(slots.add(&[1]), 1);
assert_eq!(slots.to_slots(1), vec![1]);
assert!(slots.to_slots(2).is_empty());
}
#[test]
fn test_epoch_slots_uncompressed_add_2() {
let mut slots = Uncompressed::new(1);
assert_eq!(slots.add(&[1, 2]), 2);
assert_eq!(slots.to_slots(1), vec![1, 2]);
}
#[test]
fn test_epoch_slots_uncompressed_add_3a() {
let mut slots = Uncompressed::new(1);
assert_eq!(slots.add(&[1, 3, 2]), 3);
assert_eq!(slots.to_slots(1), vec![1, 2, 3]);
}
#[test]
fn test_epoch_slots_uncompressed_add_3b() {
let mut slots = Uncompressed::new(1);
assert_eq!(slots.add(&[1, 10, 2]), 1);
assert_eq!(slots.to_slots(1), vec![1]);
}
#[test]
fn test_epoch_slots_uncompressed_add_3c() {
let mut slots = Uncompressed::new(2);
assert_eq!(slots.add(&[1, 10, 2]), 3);
assert_eq!(slots.to_slots(1), vec![1, 2, 10]);
assert_eq!(slots.to_slots(2), vec![2, 10]);
assert_eq!(slots.to_slots(3), vec![10]);
assert_eq!(slots.to_slots(11).is_empty(), true);
}
#[test]
fn test_epoch_slots_compressed() {
let mut slots = Uncompressed::new(100);
slots.add(&[1, 701, 2]);
assert_eq!(slots.num, 701);
let compressed = Flate2::deflate(slots).unwrap();
assert_eq!(compressed.first_slot, 1);
assert_eq!(compressed.num, 701);
assert!(compressed.compressed.len() < 32);
let slots = compressed.inflate().unwrap();
assert_eq!(slots.first_slot, 1);
assert_eq!(slots.num, 701);
assert_eq!(slots.to_slots(1), vec![1, 2, 701]);
}
#[test]
fn test_epoch_slots_fill_range() {
let range: Vec<Slot> = (0..5000).into_iter().collect();
let mut slots = EpochSlots::default();
assert_eq!(slots.fill(&range, 1), 5000);
assert_eq!(slots.wallclock, 1);
assert_eq!(slots.to_slots(0), range);
assert_eq!(slots.to_slots(4999), vec![4999]);
assert_eq!(slots.to_slots(5000).is_empty(), true);
}
#[test]
fn test_epoch_slots_fill_sparce_range() {
let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 3).collect();
let mut slots = EpochSlots::default();
assert_eq!(slots.fill(&range, 2), 5000);
assert_eq!(slots.wallclock, 2);
assert_eq!(slots.slots.len(), 3);
assert_eq!(slots.slots[0].first_slot(), 0);
assert_ne!(slots.slots[0].num_slots(), 0);
let next = slots.slots[0].num_slots() as u64 + slots.slots[0].first_slot();
assert!(slots.slots[1].first_slot() >= next);
assert_ne!(slots.slots[1].num_slots(), 0);
assert_ne!(slots.slots[2].num_slots(), 0);
assert_eq!(slots.to_slots(0), range);
assert_eq!(slots.to_slots(4999 * 3), vec![4999 * 3]);
}
#[test]
fn test_epoch_slots_fill_large_sparce_range() {
let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 7).collect();
let mut slots = EpochSlots::default();
assert_eq!(slots.fill(&range, 2), 5000);
assert_eq!(slots.to_slots(0), range);
}
#[test]
fn test_epoch_slots_fill_uncompressed_random_range() {
use rand::Rng;
for _ in 0..10 {
let mut range: Vec<Slot> = vec![];
for _ in 0..5000 {
let last = *range.last().unwrap_or(&0);
range.push(last + rand::thread_rng().gen_range(1, 5));
}
let sz = EpochSlots::default().max_compressed_slot_size();
let mut slots = Uncompressed::new(sz as usize);
let sz = slots.add(&range);
let slots = slots.to_slots(0);
assert_eq!(slots.len(), sz);
assert_eq!(slots[..], range[..sz]);
}
}
#[test]
fn test_epoch_slots_fill_compressed_random_range() {
use rand::Rng;
for _ in 0..10 {
let mut range: Vec<Slot> = vec![];
for _ in 0..5000 {
let last = *range.last().unwrap_or(&0);
range.push(last + rand::thread_rng().gen_range(1, 5));
}
let sz = EpochSlots::default().max_compressed_slot_size();
let mut slots = Uncompressed::new(sz as usize);
let sz = slots.add(&range);
let mut slots = CompressedSlots::Uncompressed(slots);
slots.deflate().unwrap();
let slots = slots.to_slots(0).unwrap();
assert_eq!(slots.len(), sz);
assert_eq!(slots[..], range[..sz]);
}
}
#[test]
fn test_epoch_slots_fill_random_range() {
use rand::Rng;
for _ in 0..10 {
let mut range: Vec<Slot> = vec![];
for _ in 0..5000 {
let last = *range.last().unwrap_or(&0);
range.push(last + rand::thread_rng().gen_range(1, 5));
}
let mut slots = EpochSlots::default();
let sz = slots.fill(&range, 1);
let last = range[sz - 1];
assert_eq!(
last,
slots.slots.last().unwrap().first_slot()
+ slots.slots.last().unwrap().num_slots() as u64
- 1
);
for s in &slots.slots {
assert!(s.to_slots(0).is_ok());
}
let slots = slots.to_slots(0);
assert_eq!(slots[..], range[..slots.len()]);
assert_eq!(sz, slots.len())
}
}
}

View File

@@ -1,14 +1,14 @@
//! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
use crate::banking_stage::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET;
use crate::packet::PacketsRecycler;
use crate::poh_recorder::PohRecorder;
use crate::result::{Error, Result};
use crate::streamer::{self, PacketReceiver, PacketSender};
use solana_measure::thread_mem_usage;
use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
use solana_perf::packet::PacketsRecycler;
use solana_perf::recycler::Recycler;
use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
use std::net::UdpSocket;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::{channel, RecvTimeoutError};

View File

@@ -0,0 +1 @@
pub use solana_ledger::genesis_utils::*;

View File

@@ -2,13 +2,13 @@
use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
use crate::contact_info::ContactInfo;
use crate::streamer;
use rand::{thread_rng, Rng};
use solana_client::thin_client::{create_client, ThinClient};
use solana_ledger::bank_forks::BankForks;
use solana_perf::recycler::Recycler;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, Signer};
use solana_streamer::streamer;
use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::channel;

View File

@@ -5,18 +5,17 @@
//! command-line tools to spin up validators and a Rust library
//!
pub mod accounts_background_service;
pub mod accounts_hash_verifier;
pub mod banking_stage;
pub mod broadcast_stage;
pub mod cluster_info_vote_listener;
pub mod commitment;
mod deprecated;
pub mod shred_fetch_stage;
#[macro_use]
pub mod contact_info;
pub mod blockstream;
pub mod blockstream_service;
pub mod cluster_info;
pub mod cluster_slots;
pub mod consensus;
pub mod crds;
pub mod crds_gossip;
@@ -24,15 +23,16 @@ pub mod crds_gossip_error;
pub mod crds_gossip_pull;
pub mod crds_gossip_push;
pub mod crds_value;
pub mod epoch_slots;
pub mod fetch_stage;
pub mod gen_keys;
pub mod genesis_utils;
pub mod gossip_service;
pub mod ledger_cleanup_service;
pub mod local_vote_signer_service;
pub mod packet;
pub mod poh_recorder;
pub mod poh_service;
pub mod progress_map;
pub mod recvmmsg;
pub mod repair_service;
pub mod replay_stage;
mod result;
@@ -43,6 +43,7 @@ pub mod rpc_pubsub;
pub mod rpc_pubsub_service;
pub mod rpc_service;
pub mod rpc_subscriptions;
pub mod sendmmsg;
pub mod serve_repair;
pub mod serve_repair_service;
pub mod sigverify;
@@ -50,11 +51,11 @@ pub mod sigverify_shreds;
pub mod sigverify_stage;
pub mod snapshot_packager_service;
pub mod storage_stage;
pub mod streamer;
pub mod tpu;
pub mod transaction_status_service;
pub mod tvu;
pub mod validator;
pub mod verified_vote_packets;
pub mod weighted_shuffle;
pub mod window_service;

View File

@@ -72,7 +72,7 @@ mod tests {
#[test]
fn test_packets_set_addr() {
// test that the address is actually being updated
let send_addr: SocketAddr = "127.0.0.1:123".parse().unwrap();
let send_addr = socketaddr!([127, 0, 0, 1], 123);
let packets = vec![Packet::default()];
let mut msgs = Packets::new(packets);
msgs.set_addr(&send_addr);

View File

@@ -511,8 +511,8 @@ impl PohRecorder {
#[cfg(test)]
mod tests {
use super::*;
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use bincode::serialize;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta, get_tmp_ledger_path};
use solana_perf::test_tx::test_tx;
use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;

View File

@@ -120,8 +120,8 @@ impl PohService {
#[cfg(test)]
mod tests {
use super::*;
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use crate::poh_recorder::WorkingBank;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
use solana_perf::test_tx::test_tx;

View File

@@ -1,622 +0,0 @@
use crate::{
cluster_info_vote_listener::SlotVoteTracker, cluster_slots::SlotPubkeys,
consensus::StakeLockout, replay_stage::SUPERMINORITY_THRESHOLD,
};
use solana_ledger::{
bank_forks::BankForks,
blockstore_processor::{ConfirmationProgress, ConfirmationTiming},
};
use solana_runtime::bank::Bank;
use solana_sdk::{account::Account, clock::Slot, hash::Hash, pubkey::Pubkey};
use std::{
collections::{HashMap, HashSet},
rc::Rc,
sync::{Arc, RwLock},
};
#[derive(Default)]
pub(crate) struct ReplaySlotStats(ConfirmationTiming);
impl std::ops::Deref for ReplaySlotStats {
type Target = ConfirmationTiming;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for ReplaySlotStats {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl ReplaySlotStats {
pub fn report_stats(&self, slot: Slot, num_entries: usize, num_shreds: u64) {
datapoint_info!(
"replay-slot-stats",
("slot", slot as i64, i64),
("fetch_entries_time", self.fetch_elapsed as i64, i64),
(
"fetch_entries_fail_time",
self.fetch_fail_elapsed as i64,
i64
),
("entry_verification_time", self.verify_elapsed as i64, i64),
("replay_time", self.replay_elapsed as i64, i64),
(
"replay_total_elapsed",
self.started.elapsed().as_micros() as i64,
i64
),
("total_entries", num_entries as i64, i64),
("total_shreds", num_shreds as i64, i64),
);
}
}
#[derive(Debug)]
pub(crate) struct ValidatorStakeInfo {
pub validator_vote_pubkey: Pubkey,
pub stake: u64,
pub total_epoch_stake: u64,
}
impl Default for ValidatorStakeInfo {
fn default() -> Self {
Self {
stake: 0,
validator_vote_pubkey: Pubkey::default(),
total_epoch_stake: 1,
}
}
}
impl ValidatorStakeInfo {
pub fn new(validator_vote_pubkey: Pubkey, stake: u64, total_epoch_stake: u64) -> Self {
Self {
validator_vote_pubkey,
stake,
total_epoch_stake,
}
}
}
pub(crate) struct ForkProgress {
pub(crate) is_dead: bool,
pub(crate) fork_stats: ForkStats,
pub(crate) propagated_stats: PropagatedStats,
pub(crate) replay_stats: ReplaySlotStats,
pub(crate) replay_progress: ConfirmationProgress,
// Note `num_blocks_on_fork` and `num_dropped_blocks_on_fork` only
// count new blocks replayed since last restart, which won't include
// blocks already existing in the ledger/before snapshot at start,
// so these stats do not span all of time
pub(crate) num_blocks_on_fork: u64,
pub(crate) num_dropped_blocks_on_fork: u64,
}
impl ForkProgress {
pub fn new(
last_entry: Hash,
prev_leader_slot: Option<Slot>,
validator_stake_info: Option<ValidatorStakeInfo>,
num_blocks_on_fork: u64,
num_dropped_blocks_on_fork: u64,
) -> Self {
let (
is_leader_slot,
propagated_validators_stake,
propagated_validators,
is_propagated,
total_epoch_stake,
) = validator_stake_info
.map(|info| {
(
true,
info.stake,
vec![Rc::new(info.validator_vote_pubkey)]
.into_iter()
.collect(),
{
if info.total_epoch_stake == 0 {
true
} else {
info.stake as f64 / info.total_epoch_stake as f64
> SUPERMINORITY_THRESHOLD
}
},
info.total_epoch_stake,
)
})
.unwrap_or((false, 0, HashSet::new(), false, 0));
Self {
is_dead: false,
fork_stats: ForkStats::default(),
replay_stats: ReplaySlotStats::default(),
replay_progress: ConfirmationProgress::new(last_entry),
num_blocks_on_fork,
num_dropped_blocks_on_fork,
propagated_stats: PropagatedStats {
prev_leader_slot,
is_leader_slot,
propagated_validators_stake,
propagated_validators,
is_propagated,
total_epoch_stake,
..PropagatedStats::default()
},
}
}
pub fn new_from_bank(
bank: &Bank,
my_pubkey: &Pubkey,
voting_pubkey: &Pubkey,
prev_leader_slot: Option<Slot>,
num_blocks_on_fork: u64,
num_dropped_blocks_on_fork: u64,
) -> Self {
let validator_fork_info = {
if bank.collector_id() == my_pubkey {
let stake = bank.epoch_vote_account_stake(voting_pubkey);
Some(ValidatorStakeInfo::new(
*voting_pubkey,
stake,
bank.total_epoch_stake(),
))
} else {
None
}
};
Self::new(
bank.last_blockhash(),
prev_leader_slot,
validator_fork_info,
num_blocks_on_fork,
num_dropped_blocks_on_fork,
)
}
}
#[derive(Debug, Clone, Default)]
pub(crate) struct ForkStats {
pub(crate) weight: u128,
pub(crate) fork_weight: u128,
pub(crate) total_staked: u64,
pub(crate) block_height: u64,
pub(crate) has_voted: bool,
pub(crate) is_recent: bool,
pub(crate) is_empty: bool,
pub(crate) vote_threshold: bool,
pub(crate) is_locked_out: bool,
pub(crate) stake_lockouts: HashMap<u64, StakeLockout>,
pub(crate) confirmation_reported: bool,
pub(crate) computed: bool,
}
#[derive(Clone, Default)]
pub(crate) struct PropagatedStats {
pub(crate) propagated_validators: HashSet<Rc<Pubkey>>,
pub(crate) propagated_node_ids: HashSet<Rc<Pubkey>>,
pub(crate) propagated_validators_stake: u64,
pub(crate) is_propagated: bool,
pub(crate) is_leader_slot: bool,
pub(crate) prev_leader_slot: Option<Slot>,
pub(crate) slot_vote_tracker: Option<Arc<RwLock<SlotVoteTracker>>>,
pub(crate) cluster_slot_pubkeys: Option<Arc<RwLock<SlotPubkeys>>>,
pub(crate) total_epoch_stake: u64,
}
impl PropagatedStats {
pub fn add_vote_pubkey(
&mut self,
vote_pubkey: &Pubkey,
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
stake: u64,
) {
if !self.propagated_validators.contains(vote_pubkey) {
let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(vote_pubkey).cloned();
if cached_pubkey.is_none() {
let new_pubkey = Rc::new(*vote_pubkey);
all_pubkeys.insert(new_pubkey.clone());
cached_pubkey = Some(new_pubkey);
}
let vote_pubkey = cached_pubkey.unwrap();
self.propagated_validators.insert(vote_pubkey);
self.propagated_validators_stake += stake;
}
}
pub fn add_node_pubkey(
&mut self,
node_pubkey: &Pubkey,
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
bank: &Bank,
) {
if !self.propagated_node_ids.contains(node_pubkey) {
let node_vote_accounts = bank
.epoch_vote_accounts_for_node_id(&node_pubkey)
.map(|v| &v.vote_accounts);
if let Some(node_vote_accounts) = node_vote_accounts {
self.add_node_pubkey_internal(
node_pubkey,
all_pubkeys,
node_vote_accounts,
bank.epoch_vote_accounts(bank.epoch())
.expect("Epoch stakes for bank's own epoch must exist"),
);
}
}
}
fn add_node_pubkey_internal(
&mut self,
node_pubkey: &Pubkey,
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
vote_account_pubkeys: &[Pubkey],
epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
) {
let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(node_pubkey).cloned();
if cached_pubkey.is_none() {
let new_pubkey = Rc::new(*node_pubkey);
all_pubkeys.insert(new_pubkey.clone());
cached_pubkey = Some(new_pubkey);
}
let node_pubkey = cached_pubkey.unwrap();
self.propagated_node_ids.insert(node_pubkey);
for vote_account_pubkey in vote_account_pubkeys.iter() {
let stake = epoch_vote_accounts
.get(vote_account_pubkey)
.map(|(stake, _)| *stake)
.unwrap_or(0);
self.add_vote_pubkey(vote_account_pubkey, all_pubkeys, stake);
}
}
}
#[derive(Default)]
pub(crate) struct ProgressMap {
progress_map: HashMap<Slot, ForkProgress>,
}
impl std::ops::Deref for ProgressMap {
type Target = HashMap<Slot, ForkProgress>;
fn deref(&self) -> &Self::Target {
&self.progress_map
}
}
impl std::ops::DerefMut for ProgressMap {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.progress_map
}
}
impl ProgressMap {
pub fn insert(&mut self, slot: Slot, fork_progress: ForkProgress) {
self.progress_map.insert(slot, fork_progress);
}
pub fn get_propagated_stats(&self, slot: Slot) -> Option<&PropagatedStats> {
self.progress_map
.get(&slot)
.map(|fork_progress| &fork_progress.propagated_stats)
}
pub fn get_propagated_stats_mut(&mut self, slot: Slot) -> Option<&mut PropagatedStats> {
self.progress_map
.get_mut(&slot)
.map(|fork_progress| &mut fork_progress.propagated_stats)
}
pub fn get_fork_stats(&self, slot: Slot) -> Option<&ForkStats> {
self.progress_map
.get(&slot)
.map(|fork_progress| &fork_progress.fork_stats)
}
pub fn get_fork_stats_mut(&mut self, slot: Slot) -> Option<&mut ForkStats> {
self.progress_map
.get_mut(&slot)
.map(|fork_progress| &mut fork_progress.fork_stats)
}
pub fn is_propagated(&self, slot: Slot) -> bool {
let leader_slot_to_check = self.get_latest_leader_slot(slot);
// prev_leader_slot doesn't exist because already rooted
// or this validator hasn't been scheduled as a leader
// yet. In both cases the latest leader is vacuously
// confirmed
leader_slot_to_check
.map(|leader_slot_to_check| {
// If the leader's stats are None (isn't in the
// progress map), this means that prev_leader slot is
// rooted, so return true
self.get_propagated_stats(leader_slot_to_check)
.map(|stats| stats.is_propagated)
.unwrap_or(true)
})
.unwrap_or(true)
}
pub fn get_latest_leader_slot(&self, slot: Slot) -> Option<Slot> {
let propagated_stats = self
.get_propagated_stats(slot)
.expect("All frozen banks must exist in the Progress map");
if propagated_stats.is_leader_slot {
Some(slot)
} else {
propagated_stats.prev_leader_slot
}
}
pub fn get_bank_prev_leader_slot(&self, bank: &Bank) -> Option<Slot> {
let parent_slot = bank.parent_slot();
self.get_propagated_stats(parent_slot)
.map(|stats| {
if stats.is_leader_slot {
Some(parent_slot)
} else {
stats.prev_leader_slot
}
})
.unwrap_or(None)
}
pub fn handle_new_root(&mut self, bank_forks: &BankForks) {
self.progress_map
.retain(|k, _| bank_forks.get(*k).is_some());
}
pub fn log_propagated_stats(&self, slot: Slot, bank_forks: &RwLock<BankForks>) {
if let Some(stats) = self.get_propagated_stats(slot) {
info!(
"Propagated stats:
total staked: {},
observed staked: {},
vote pubkeys: {:?},
node_pubkeys: {:?},
slot: {},
epoch: {:?}",
stats.total_epoch_stake,
stats.propagated_validators_stake,
stats.propagated_validators,
stats.propagated_node_ids,
slot,
bank_forks.read().unwrap().get(slot).map(|x| x.epoch()),
);
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_add_vote_pubkey() {
let mut stats = PropagatedStats::default();
let mut all_pubkeys = HashSet::new();
let mut vote_pubkey = Pubkey::new_rand();
all_pubkeys.insert(Rc::new(vote_pubkey.clone()));
// Add a vote pubkey, the number of references in all_pubkeys
// should be 2
stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
assert!(stats.propagated_validators.contains(&vote_pubkey));
assert_eq!(stats.propagated_validators_stake, 1);
assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
// Adding it again should change no state since the key already existed
stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
assert!(stats.propagated_validators.contains(&vote_pubkey));
assert_eq!(stats.propagated_validators_stake, 1);
// Addding another pubkey should succeed
vote_pubkey = Pubkey::new_rand();
stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 2);
assert!(stats.propagated_validators.contains(&vote_pubkey));
assert_eq!(stats.propagated_validators_stake, 3);
assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
}
#[test]
fn test_add_node_pubkey_internal() {
let num_vote_accounts = 10;
let staked_vote_accounts = 5;
let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
.take(num_vote_accounts)
.collect();
let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
.iter()
.skip(num_vote_accounts - staked_vote_accounts)
.map(|pubkey| (*pubkey, (1, Account::default())))
.collect();
let mut stats = PropagatedStats::default();
let mut all_pubkeys = HashSet::new();
let mut node_pubkey = Pubkey::new_rand();
all_pubkeys.insert(Rc::new(node_pubkey.clone()));
// Add a vote pubkey, the number of references in all_pubkeys
// should be 2
stats.add_node_pubkey_internal(
&node_pubkey,
&mut all_pubkeys,
&vote_account_pubkeys,
&epoch_vote_accounts,
);
assert!(stats.propagated_node_ids.contains(&node_pubkey));
assert_eq!(
stats.propagated_validators_stake,
staked_vote_accounts as u64
);
assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
// Adding it again should not change any state
stats.add_node_pubkey_internal(
&node_pubkey,
&mut all_pubkeys,
&vote_account_pubkeys,
&epoch_vote_accounts,
);
assert!(stats.propagated_node_ids.contains(&node_pubkey));
assert_eq!(
stats.propagated_validators_stake,
staked_vote_accounts as u64
);
// Addding another pubkey with same vote accounts should succeed, but stake
// shouldn't increase
node_pubkey = Pubkey::new_rand();
stats.add_node_pubkey_internal(
&node_pubkey,
&mut all_pubkeys,
&vote_account_pubkeys,
&epoch_vote_accounts,
);
assert!(stats.propagated_node_ids.contains(&node_pubkey));
assert_eq!(
stats.propagated_validators_stake,
staked_vote_accounts as u64
);
assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
// Addding another pubkey with different vote accounts should succeed
// and increase stake
node_pubkey = Pubkey::new_rand();
let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
.take(num_vote_accounts)
.collect();
let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
.iter()
.skip(num_vote_accounts - staked_vote_accounts)
.map(|pubkey| (*pubkey, (1, Account::default())))
.collect();
stats.add_node_pubkey_internal(
&node_pubkey,
&mut all_pubkeys,
&vote_account_pubkeys,
&epoch_vote_accounts,
);
assert!(stats.propagated_node_ids.contains(&node_pubkey));
assert_eq!(
stats.propagated_validators_stake,
2 * staked_vote_accounts as u64
);
assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
}
#[test]
fn test_is_propagated_status_on_construction() {
// If the given ValidatorStakeInfo == None, then this is not
// a leader slot and is_propagated == false
let progress = ForkProgress::new(Hash::default(), Some(9), None, 0, 0);
assert!(!progress.propagated_stats.is_propagated);
// If the stake is zero, then threshold is always achieved
let progress = ForkProgress::new(
Hash::default(),
Some(9),
Some(ValidatorStakeInfo {
total_epoch_stake: 0,
..ValidatorStakeInfo::default()
}),
0,
0,
);
assert!(progress.propagated_stats.is_propagated);
// If the stake is non zero, then threshold is not achieved unless
// validator has enough stake by itself to pass threshold
let progress = ForkProgress::new(
Hash::default(),
Some(9),
Some(ValidatorStakeInfo {
total_epoch_stake: 2,
..ValidatorStakeInfo::default()
}),
0,
0,
);
assert!(!progress.propagated_stats.is_propagated);
// Give the validator enough stake by itself to pass threshold
let progress = ForkProgress::new(
Hash::default(),
Some(9),
Some(ValidatorStakeInfo {
stake: 1,
total_epoch_stake: 2,
..ValidatorStakeInfo::default()
}),
0,
0,
);
assert!(progress.propagated_stats.is_propagated);
// Check that the default ValidatorStakeInfo::default() constructs a ForkProgress
// with is_propagated == false, otherwise propagation tests will fail to run
// the proper checks (most will auto-pass without checking anything)
let progress = ForkProgress::new(
Hash::default(),
Some(9),
Some(ValidatorStakeInfo::default()),
0,
0,
);
assert!(!progress.propagated_stats.is_propagated);
}
#[test]
fn test_is_propagated() {
let mut progress_map = ProgressMap::default();
// Insert new ForkProgress for slot 10 (not a leader slot) and its
// previous leader slot 9 (leader slot)
progress_map.insert(10, ForkProgress::new(Hash::default(), Some(9), None, 0, 0));
progress_map.insert(
9,
ForkProgress::new(
Hash::default(),
None,
Some(ValidatorStakeInfo::default()),
0,
0,
),
);
// None of these slot have parents which are confirmed
assert!(!progress_map.is_propagated(9));
assert!(!progress_map.is_propagated(10));
// Insert new ForkProgress for slot 8 with no previous leader.
// The previous leader before 8, slot 7, does not exist in
// progress map, so is_propagated(8) should return true as
// this implies the parent is rooted
progress_map.insert(8, ForkProgress::new(Hash::default(), Some(7), None, 0, 0));
assert!(progress_map.is_propagated(8));
// If we set the is_propagated = true, is_propagated should return true
progress_map
.get_propagated_stats_mut(9)
.unwrap()
.is_propagated = true;
assert!(progress_map.is_propagated(9));
assert!(progress_map.get(&9).unwrap().propagated_stats.is_propagated);
// Because slot 9 is now confirmed, then slot 10 is also confirmed b/c 9
// is the last leader slot before 10
assert!(progress_map.is_propagated(10));
// If we make slot 10 a leader slot though, even though its previous
// leader slot 9 has been confirmed, slot 10 itself is not confirmed
progress_map
.get_propagated_stats_mut(10)
.unwrap()
.is_leader_slot = true;
assert!(!progress_map.is_propagated(10));
}
}

View File

@@ -213,4 +213,49 @@ mod tests {
assert_eq!(packets[i].meta.addr(), saddr2);
}
}
#[cfg(target_os = "linux")]
#[test]
pub fn test_recv_mmsg_batch_size() {
let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
let addr = reader.local_addr().unwrap();
let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
const TEST_BATCH_SIZE: usize = 64;
let sent = TEST_BATCH_SIZE;
let mut elapsed_in_max_batch = 0;
(0..1000).for_each(|_| {
for _ in 0..sent {
let data = [0; PACKET_DATA_SIZE];
sender.send_to(&data[..], &addr).unwrap();
}
let mut packets = vec![Packet::default(); TEST_BATCH_SIZE];
let now = Instant::now();
let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
elapsed_in_max_batch += now.elapsed().as_nanos();
assert_eq!(TEST_BATCH_SIZE, recv);
});
let mut elapsed_in_small_batch = 0;
(0..1000).for_each(|_| {
for _ in 0..sent {
let data = [0; PACKET_DATA_SIZE];
sender.send_to(&data[..], &addr).unwrap();
}
let mut packets = vec![Packet::default(); 4];
let mut recv = 0;
let now = Instant::now();
while let Ok(num) = recv_mmsg(&reader, &mut packets[..]) {
recv += num.1;
if recv >= TEST_BATCH_SIZE {
break;
}
}
elapsed_in_small_batch += now.elapsed().as_nanos();
assert_eq!(TEST_BATCH_SIZE, recv);
});
assert!(elapsed_in_max_batch <= elapsed_in_small_batch);
}
}

View File

@@ -2,7 +2,6 @@
//! regularly finds missing shreds in the ledger and sends repair requests for those shreds
use crate::{
cluster_info::ClusterInfo,
cluster_slots::ClusterSlots,
result::Result,
serve_repair::{RepairType, ServeRepair},
};
@@ -10,45 +9,27 @@ use solana_ledger::{
bank_forks::BankForks,
blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
};
use solana_sdk::clock::DEFAULT_SLOTS_PER_EPOCH;
use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey};
use std::{
collections::HashMap,
collections::{BTreeSet, HashSet},
iter::Iterator,
net::SocketAddr,
net::UdpSocket,
ops::Bound::{Included, Unbounded},
sync::atomic::{AtomicBool, Ordering},
sync::{Arc, RwLock},
thread::sleep,
thread::{self, Builder, JoinHandle},
time::{Duration, Instant},
time::Duration,
};
#[derive(Default)]
pub struct RepairStatsGroup {
pub count: u64,
pub min: u64,
pub max: u64,
}
impl RepairStatsGroup {
pub fn update(&mut self, slot: u64) {
self.count += 1;
self.min = std::cmp::min(self.min, slot);
self.max = std::cmp::max(self.max, slot);
}
}
#[derive(Default)]
pub struct RepairStats {
pub shred: RepairStatsGroup,
pub highest_shred: RepairStatsGroup,
pub orphan: RepairStatsGroup,
}
pub const MAX_REPAIR_LENGTH: usize = 512;
pub const REPAIR_MS: u64 = 100;
pub const MAX_ORPHANS: usize = 5;
const MAX_COMPLETED_SLOT_CACHE_LEN: usize = 256;
const COMPLETED_SLOT_CACHE_FLUSH_TRIGGER: usize = 512;
pub enum RepairStrategy {
RepairRange(RepairSlotRange),
RepairAll {
@@ -83,7 +64,6 @@ impl RepairService {
repair_socket: Arc<UdpSocket>,
cluster_info: Arc<RwLock<ClusterInfo>>,
repair_strategy: RepairStrategy,
cluster_slots: Arc<ClusterSlots>,
) -> Self {
let t_repair = Builder::new()
.name("solana-repair-service".to_string())
@@ -94,7 +74,6 @@ impl RepairService {
&repair_socket,
&cluster_info,
repair_strategy,
&cluster_slots,
)
})
.unwrap();
@@ -103,20 +82,31 @@ impl RepairService {
}
fn run(
blockstore: &Blockstore,
exit: &AtomicBool,
repair_socket: &UdpSocket,
blockstore: &Arc<Blockstore>,
exit: &Arc<AtomicBool>,
repair_socket: &Arc<UdpSocket>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
repair_strategy: RepairStrategy,
cluster_slots: &Arc<ClusterSlots>,
) {
let serve_repair = ServeRepair::new(cluster_info.clone());
let mut epoch_slots: BTreeSet<Slot> = BTreeSet::new();
let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
let id = cluster_info.read().unwrap().id();
if let RepairStrategy::RepairAll { .. } = repair_strategy {
Self::initialize_lowest_slot(id, blockstore, cluster_info);
if let RepairStrategy::RepairAll {
ref epoch_schedule, ..
} = repair_strategy
{
let current_root = blockstore.last_root();
Self::initialize_epoch_slots(
id,
blockstore,
&mut epoch_slots,
&old_incomplete_slots,
current_root,
epoch_schedule,
cluster_info,
);
}
let mut repair_stats = RepairStats::default();
let mut last_stats = Instant::now();
loop {
if exit.load(Ordering::Relaxed) {
break;
@@ -135,38 +125,30 @@ impl RepairService {
RepairStrategy::RepairAll {
ref completed_slots_receiver,
ref bank_forks,
..
} => {
let new_root = blockstore.last_root();
let lowest_slot = blockstore.lowest_slot();
Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
Self::update_completed_slots(
&id,
Self::update_epoch_slots(
id,
new_root,
&cluster_slots,
blockstore,
completed_slots_receiver,
lowest_slot,
&mut epoch_slots,
&mut old_incomplete_slots,
&cluster_info,
completed_slots_receiver,
);
cluster_slots.update(new_root, cluster_info, bank_forks);
Self::generate_repairs(blockstore, new_root, MAX_REPAIR_LENGTH)
}
}
};
if let Ok(repairs) = repairs {
let mut cache = HashMap::new();
let reqs: Vec<((SocketAddr, Vec<u8>), RepairType)> = repairs
let reqs: Vec<_> = repairs
.into_iter()
.filter_map(|repair_request| {
serve_repair
.repair_request(
&cluster_slots,
&repair_request,
&mut cache,
&mut repair_stats,
)
.repair_request(&repair_request)
.map(|result| (result, repair_request))
.ok()
})
@@ -179,24 +161,6 @@ impl RepairService {
});
}
}
if last_stats.elapsed().as_secs() > 1 {
let repair_total = repair_stats.shred.count
+ repair_stats.highest_shred.count
+ repair_stats.orphan.count;
if repair_total > 0 {
datapoint_info!(
"serve_repair-repair",
("repair-total", repair_total, i64),
("shred-count", repair_stats.shred.count, i64),
("highest-shred-count", repair_stats.highest_shred.count, i64),
("orphan-count", repair_stats.orphan.count, i64),
("repair-highest-slot", repair_stats.highest_shred.max, i64),
("repair-orphan", repair_stats.orphan.max, i64),
);
}
repair_stats = RepairStats::default();
last_stats = Instant::now();
}
sleep(Duration::from_millis(REPAIR_MS));
}
}
@@ -308,54 +272,150 @@ impl RepairService {
}
}
fn initialize_lowest_slot(
fn get_completed_slots_past_root(
blockstore: &Blockstore,
slots_in_gossip: &mut BTreeSet<Slot>,
root: Slot,
epoch_schedule: &EpochSchedule,
) {
let last_confirmed_epoch = epoch_schedule.get_leader_schedule_epoch(root);
let last_epoch_slot = epoch_schedule.get_last_slot_in_epoch(last_confirmed_epoch);
let meta_iter = blockstore
.slot_meta_iterator(root + 1)
.expect("Couldn't get db iterator");
for (current_slot, meta) in meta_iter {
if current_slot > last_epoch_slot {
break;
}
if meta.is_full() {
slots_in_gossip.insert(current_slot);
}
}
}
fn initialize_epoch_slots(
id: Pubkey,
blockstore: &Blockstore,
slots_in_gossip: &mut BTreeSet<Slot>,
old_incomplete_slots: &BTreeSet<Slot>,
root: Slot,
epoch_schedule: &EpochSchedule,
cluster_info: &RwLock<ClusterInfo>,
) {
Self::get_completed_slots_past_root(blockstore, slots_in_gossip, root, epoch_schedule);
// Safe to set into gossip because by this time, the leader schedule cache should
// also be updated with the latest root (done in blockstore_processor) and thus
// will provide a schedule to window_service for any incoming shreds up to the
// last_confirmed_epoch.
cluster_info
.write()
.unwrap()
.push_lowest_slot(id, blockstore.lowest_slot());
cluster_info.write().unwrap().push_epoch_slots(
id,
root,
blockstore.lowest_slot(),
slots_in_gossip.clone(),
old_incomplete_slots,
);
}
fn update_completed_slots(
id: &Pubkey,
root: Slot,
cluster_slots: &ClusterSlots,
blockstore: &Blockstore,
completed_slots_receiver: &CompletedSlotsReceiver,
// Update the gossiped structure used for the "Repairmen" repair protocol. See docs
// for details.
fn update_epoch_slots(
id: Pubkey,
latest_known_root: Slot,
lowest_slot: Slot,
completed_slot_cache: &mut BTreeSet<Slot>,
incomplete_slot_stash: &mut BTreeSet<Slot>,
cluster_info: &RwLock<ClusterInfo>,
completed_slots_receiver: &CompletedSlotsReceiver,
) {
let mine = cluster_slots.collect(id);
let mut slots: Vec<Slot> = vec![];
while let Ok(mut more) = completed_slots_receiver.try_recv() {
more.retain(|x| !mine.contains(x));
slots.append(&mut more);
let mut should_update = false;
while let Ok(completed_slots) = completed_slots_receiver.try_recv() {
for slot in completed_slots {
let last_slot_in_stash = *incomplete_slot_stash.iter().next_back().unwrap_or(&0);
let removed_from_stash = incomplete_slot_stash.remove(&slot);
// If the newly completed slot was not being tracked in stash, and is > last
// slot being tracked in stash, add it to cache. Also, update gossip
if !removed_from_stash && slot >= last_slot_in_stash {
should_update |= completed_slot_cache.insert(slot);
}
// If the slot was removed from stash, update gossip
should_update |= removed_from_stash;
}
}
if should_update {
if completed_slot_cache.len() >= COMPLETED_SLOT_CACHE_FLUSH_TRIGGER {
Self::stash_old_incomplete_slots(completed_slot_cache, incomplete_slot_stash);
let lowest_completed_slot_in_cache =
*completed_slot_cache.iter().next().unwrap_or(&0);
Self::prune_incomplete_slot_stash(
incomplete_slot_stash,
lowest_completed_slot_in_cache,
);
}
cluster_info.write().unwrap().push_epoch_slots(
id,
latest_known_root,
lowest_slot,
completed_slot_cache.clone(),
incomplete_slot_stash,
);
}
}
fn stash_old_incomplete_slots(cache: &mut BTreeSet<Slot>, stash: &mut BTreeSet<Slot>) {
if cache.len() > MAX_COMPLETED_SLOT_CACHE_LEN {
let mut prev = *cache.iter().next().expect("Expected to find some slot");
cache.remove(&prev);
while cache.len() >= MAX_COMPLETED_SLOT_CACHE_LEN {
let next = *cache.iter().next().expect("Expected to find some slot");
cache.remove(&next);
// Prev slot and next slot are not included in incomplete slot list.
(prev + 1..next).for_each(|slot| {
stash.insert(slot);
});
prev = next;
}
}
}
fn prune_incomplete_slot_stash(
stash: &mut BTreeSet<Slot>,
lowest_completed_slot_in_cache: Slot,
) {
if let Some(oldest_incomplete_slot) = stash.iter().next() {
// Prune old slots
// Prune in batches to reduce overhead. Pruning starts when oldest slot is 1.5 epochs
// earlier than the new root. But, we prune all the slots that are older than 1 epoch.
// So slots in a batch of half epoch are getting pruned
if oldest_incomplete_slot + DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2
< lowest_completed_slot_in_cache
{
let oldest_slot_to_retain =
lowest_completed_slot_in_cache.saturating_sub(DEFAULT_SLOTS_PER_EPOCH);
*stash = stash
.range((Included(&oldest_slot_to_retain), Unbounded))
.cloned()
.collect();
}
}
}
#[allow(dead_code)]
fn find_incomplete_slots(blockstore: &Blockstore, root: Slot) -> HashSet<Slot> {
blockstore
.live_slots_iterator(root)
.for_each(|(slot, slot_meta)| {
if slot_meta.is_full() && !mine.contains(&slot) {
slots.push(slot)
.filter_map(|(slot, slot_meta)| {
if !slot_meta.is_full() {
Some(slot)
} else {
None
}
});
slots.sort();
slots.dedup();
if !slots.is_empty() {
cluster_info.write().unwrap().push_epoch_slots(&slots);
}
}
fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &RwLock<ClusterInfo>) {
cluster_info
.write()
.unwrap()
.push_lowest_slot(*id, lowest_slot);
})
.collect()
}
pub fn join(self) -> thread::Result<()> {
@@ -367,11 +427,15 @@ impl RepairService {
mod test {
use super::*;
use crate::cluster_info::Node;
use itertools::Itertools;
use rand::seq::SliceRandom;
use rand::{thread_rng, Rng};
use solana_ledger::blockstore::{
make_chaining_slot_entries, make_many_slot_entries, make_slot_entries,
};
use solana_ledger::shred::max_ticks_per_n_shreds;
use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
use std::thread::Builder;
#[test]
pub fn test_repair_orphan() {
@@ -587,19 +651,342 @@ mod test {
}
#[test]
pub fn test_update_lowest_slot() {
let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
node_info.info.clone(),
));
RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
let lowest = cluster_info
.read()
.unwrap()
.get_lowest_slot_for_node(&Pubkey::default(), None)
.unwrap()
.0
.clone();
assert_eq!(lowest.lowest, 5);
pub fn test_get_completed_slots_past_root() {
let blockstore_path = get_tmp_ledger_path!();
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let num_entries_per_slot = 10;
let root = 10;
let fork1 = vec![5, 7, root, 15, 20, 21];
let fork1_shreds: Vec<_> = make_chaining_slot_entries(&fork1, num_entries_per_slot)
.into_iter()
.flat_map(|(shreds, _)| shreds)
.collect();
let fork2 = vec![8, 12];
let fork2_shreds = make_chaining_slot_entries(&fork2, num_entries_per_slot);
// Remove the last shred from each slot to make an incomplete slot
let fork2_incomplete_shreds: Vec<_> = fork2_shreds
.into_iter()
.flat_map(|(mut shreds, _)| {
shreds.pop();
shreds
})
.collect();
let mut full_slots = BTreeSet::new();
blockstore.insert_shreds(fork1_shreds, None, false).unwrap();
blockstore
.insert_shreds(fork2_incomplete_shreds, None, false)
.unwrap();
// Test that only slots > root from fork1 were included
let epoch_schedule = EpochSchedule::custom(32, 32, false);
RepairService::get_completed_slots_past_root(
&blockstore,
&mut full_slots,
root,
&epoch_schedule,
);
let mut expected: BTreeSet<_> = fork1.into_iter().filter(|x| *x > root).collect();
assert_eq!(full_slots, expected);
// Test that slots past the last confirmed epoch boundary don't get included
let last_epoch = epoch_schedule.get_leader_schedule_epoch(root);
let last_slot = epoch_schedule.get_last_slot_in_epoch(last_epoch);
let fork3 = vec![last_slot, last_slot + 1];
let fork3_shreds: Vec<_> = make_chaining_slot_entries(&fork3, num_entries_per_slot)
.into_iter()
.flat_map(|(shreds, _)| shreds)
.collect();
blockstore.insert_shreds(fork3_shreds, None, false).unwrap();
RepairService::get_completed_slots_past_root(
&blockstore,
&mut full_slots,
root,
&epoch_schedule,
);
expected.insert(last_slot);
assert_eq!(full_slots, expected);
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}
#[test]
pub fn test_update_epoch_slots() {
let blockstore_path = get_tmp_ledger_path!();
{
// Create blockstore
let (blockstore, _, completed_slots_receiver) =
Blockstore::open_with_signal(&blockstore_path).unwrap();
let blockstore = Arc::new(blockstore);
let mut root = 0;
let num_slots = 100;
let entries_per_slot = 5;
let blockstore_ = blockstore.clone();
// Spin up thread to write to blockstore
let writer = Builder::new()
.name("writer".to_string())
.spawn(move || {
let slots: Vec<_> = (1..num_slots + 1).collect();
let mut shreds: Vec<_> = make_chaining_slot_entries(&slots, entries_per_slot)
.into_iter()
.flat_map(|(shreds, _)| shreds)
.collect();
shreds.shuffle(&mut thread_rng());
let mut i = 0;
let max_step = entries_per_slot * 4;
let repair_interval_ms = 10;
let mut rng = rand::thread_rng();
let num_shreds = shreds.len();
while i < num_shreds {
let step = rng.gen_range(1, max_step + 1) as usize;
let step = std::cmp::min(step, num_shreds - i);
let shreds_to_insert = shreds.drain(..step).collect_vec();
blockstore_
.insert_shreds(shreds_to_insert, None, false)
.unwrap();
sleep(Duration::from_millis(repair_interval_ms));
i += step;
}
})
.unwrap();
let mut completed_slots = BTreeSet::new();
let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
node_info.info.clone(),
));
let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
while completed_slots.len() < num_slots as usize {
RepairService::update_epoch_slots(
Pubkey::default(),
root,
blockstore.lowest_slot(),
&mut completed_slots,
&mut old_incomplete_slots,
&cluster_info,
&completed_slots_receiver,
);
}
let mut expected: BTreeSet<_> = (1..num_slots + 1).collect();
assert_eq!(completed_slots, expected);
// Update with new root, should filter out the slots <= root
root = num_slots / 2;
let (shreds, _) = make_slot_entries(num_slots + 2, num_slots + 1, entries_per_slot);
blockstore.insert_shreds(shreds, None, false).unwrap();
RepairService::update_epoch_slots(
Pubkey::default(),
root,
0,
&mut completed_slots,
&mut old_incomplete_slots,
&cluster_info,
&completed_slots_receiver,
);
expected.insert(num_slots + 2);
assert_eq!(completed_slots, expected);
writer.join().unwrap();
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}
#[test]
fn test_stash_old_incomplete_slots() {
let mut cache: BTreeSet<Slot> = BTreeSet::new();
let mut stash: BTreeSet<Slot> = BTreeSet::new();
// When cache is empty.
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
assert_eq!(stash.len(), 0);
// Insert some slots in cache ( < MAX_COMPLETED_SLOT_CACHE_LEN + 1)
cache.insert(101);
cache.insert(102);
cache.insert(104);
cache.insert(105);
// Not enough slots in cache. So stash should remain empty.
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
assert_eq!(stash.len(), 0);
assert_eq!(cache.len(), 4);
// Insert slots in cache ( = MAX_COMPLETED_SLOT_CACHE_LEN)
let mut cache: BTreeSet<Slot> = BTreeSet::new();
(0..MAX_COMPLETED_SLOT_CACHE_LEN as u64)
.into_iter()
.for_each(|slot| {
cache.insert(slot);
});
// Not enough slots in cache. So stash should remain empty.
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
assert_eq!(stash.len(), 0);
assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN);
// Insert 1 more to cross the threshold
cache.insert(MAX_COMPLETED_SLOT_CACHE_LEN as u64);
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
// Stash is still empty, as no missing slots
assert_eq!(stash.len(), 0);
// It removed some entries from cache
assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
// Insert more slots to create a missing slot
let mut cache: BTreeSet<Slot> = BTreeSet::new();
cache.insert(0);
(2..=MAX_COMPLETED_SLOT_CACHE_LEN as u64 + 2)
.into_iter()
.for_each(|slot| {
cache.insert(slot);
});
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
// Stash is not empty
assert!(stash.contains(&1));
// It removed some entries from cache
assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
// Test multiple missing slots at dispersed locations
let mut cache: BTreeSet<Slot> = BTreeSet::new();
(0..MAX_COMPLETED_SLOT_CACHE_LEN as u64 * 2)
.into_iter()
.for_each(|slot| {
cache.insert(slot);
});
cache.remove(&10);
cache.remove(&11);
cache.remove(&28);
cache.remove(&29);
cache.remove(&148);
cache.remove(&149);
cache.remove(&150);
cache.remove(&151);
RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
// Stash is not empty
assert!(stash.contains(&10));
assert!(stash.contains(&11));
assert!(stash.contains(&28));
assert!(stash.contains(&29));
assert!(stash.contains(&148));
assert!(stash.contains(&149));
assert!(stash.contains(&150));
assert!(stash.contains(&151));
assert!(!stash.contains(&147));
assert!(!stash.contains(&152));
// It removed some entries from cache
assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
(MAX_COMPLETED_SLOT_CACHE_LEN + 1..MAX_COMPLETED_SLOT_CACHE_LEN * 2)
.into_iter()
.for_each(|slot| {
let slot: u64 = slot as u64;
assert!(cache.contains(&slot));
});
}
#[test]
fn test_prune_incomplete_slot_stash() {
// Prune empty stash
let mut stash: BTreeSet<Slot> = BTreeSet::new();
RepairService::prune_incomplete_slot_stash(&mut stash, 0);
assert!(stash.is_empty());
// Prune stash with slots < DEFAULT_SLOTS_PER_EPOCH
stash.insert(0);
stash.insert(10);
stash.insert(11);
stash.insert(50);
assert_eq!(stash.len(), 4);
RepairService::prune_incomplete_slot_stash(&mut stash, 100);
assert_eq!(stash.len(), 4);
// Prune stash with slots > DEFAULT_SLOTS_PER_EPOCH, but < 1.5 * DEFAULT_SLOTS_PER_EPOCH
stash.insert(DEFAULT_SLOTS_PER_EPOCH + 50);
assert_eq!(stash.len(), 5);
RepairService::prune_incomplete_slot_stash(&mut stash, DEFAULT_SLOTS_PER_EPOCH + 100);
assert_eq!(stash.len(), 5);
// Prune stash with slots > 1.5 * DEFAULT_SLOTS_PER_EPOCH
stash.insert(DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2);
assert_eq!(stash.len(), 6);
RepairService::prune_incomplete_slot_stash(
&mut stash,
DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2 + 1,
);
assert_eq!(stash.len(), 2);
}
#[test]
fn test_find_incomplete_slots() {
let blockstore_path = get_tmp_ledger_path!();
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let num_entries_per_slot = 100;
let (mut shreds, _) = make_slot_entries(0, 0, num_entries_per_slot);
assert!(shreds.len() > 1);
let (shreds4, _) = make_slot_entries(4, 0, num_entries_per_slot);
shreds.extend(shreds4);
blockstore.insert_shreds(shreds, None, false).unwrap();
// Nothing is incomplete
assert!(RepairService::find_incomplete_slots(&blockstore, 0).is_empty());
// Insert a slot 5 that chains to an incomplete orphan slot 3
let (shreds5, _) = make_slot_entries(5, 3, num_entries_per_slot);
blockstore.insert_shreds(shreds5, None, false).unwrap();
assert_eq!(
RepairService::find_incomplete_slots(&blockstore, 0),
vec![3].into_iter().collect()
);
// Insert another incomplete orphan slot 2 that is the parent of slot 3.
// Both should be incomplete
let (shreds3, _) = make_slot_entries(3, 2, num_entries_per_slot);
blockstore
.insert_shreds(shreds3[1..].to_vec(), None, false)
.unwrap();
assert_eq!(
RepairService::find_incomplete_slots(&blockstore, 0),
vec![2, 3].into_iter().collect()
);
// Insert a incomplete slot 6 that chains to the root 0,
// should also be incomplete
let (shreds6, _) = make_slot_entries(6, 0, num_entries_per_slot);
blockstore
.insert_shreds(shreds6[1..].to_vec(), None, false)
.unwrap();
assert_eq!(
RepairService::find_incomplete_slots(&blockstore, 0),
vec![2, 3, 6].into_iter().collect()
);
// Complete slot 3, should no longer be marked incomplete
blockstore
.insert_shreds(shreds3[..].to_vec(), None, false)
.unwrap();
assert_eq!(
RepairService::find_incomplete_slots(&blockstore, 0),
vec![2, 6].into_iter().collect()
);
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -2,9 +2,10 @@
use crate::{
cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
cluster_slots::ClusterSlots,
packet::Packets,
repair_service::RepairStrategy,
result::{Error, Result},
streamer::PacketReceiver,
window_service::{should_retransmit_and_persist, WindowService},
};
use crossbeam_channel::Receiver as CrossbeamReceiver;
@@ -16,9 +17,7 @@ use solana_ledger::{
};
use solana_measure::measure::Measure;
use solana_metrics::inc_new_counter_error;
use solana_perf::packet::Packets;
use solana_sdk::epoch_schedule::EpochSchedule;
use solana_streamer::streamer::PacketReceiver;
use std::{
cmp,
net::UdpSocket,
@@ -215,7 +214,6 @@ impl RetransmitStage {
epoch_schedule: EpochSchedule,
cfg: Option<Arc<AtomicBool>>,
shred_version: u16,
cluster_slots: Arc<ClusterSlots>,
) -> Self {
let (retransmit_sender, retransmit_receiver) = channel();
@@ -258,7 +256,6 @@ impl RetransmitStage {
);
rv && is_connected
},
cluster_slots,
);
let thread_hdls = t_retransmit;
@@ -281,11 +278,11 @@ impl RetransmitStage {
mod tests {
use super::*;
use crate::contact_info::ContactInfo;
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use crate::packet::{self, Meta, Packet, Packets};
use solana_ledger::blockstore_processor::{process_blockstore, ProcessOptions};
use solana_ledger::create_new_tmp_ledger;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_net_utils::find_available_port_in_range;
use solana_perf::packet::{Meta, Packet, Packets};
use solana_sdk::pubkey::Pubkey;
use std::net::{IpAddr, Ipv4Addr};
@@ -336,7 +333,7 @@ mod tests {
// it should send this over the sockets.
retransmit_sender.send(packets).unwrap();
let mut packets = Packets::new(vec![]);
solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
assert_eq!(packets.packets.len(), 1);
assert_eq!(packets.packets[0].meta.repair, false);
@@ -352,7 +349,7 @@ mod tests {
let packets = Packets::new(vec![repair, Packet::default()]);
retransmit_sender.send(packets).unwrap();
let mut packets = Packets::new(vec![]);
solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
assert_eq!(packets.packets.len(), 1);
assert_eq!(packets.packets[0].meta.repair, false);
}

View File

@@ -2,7 +2,7 @@
use crate::{
cluster_info::ClusterInfo, commitment::BlockCommitmentCache, contact_info::ContactInfo,
storage_stage::StorageState, validator::ValidatorExit,
packet::PACKET_DATA_SIZE, storage_stage::StorageState, validator::ValidatorExit,
};
use bincode::serialize;
use jsonrpc_core::{Error, Metadata, Result};
@@ -12,7 +12,6 @@ use solana_faucet::faucet::request_airdrop_transaction;
use solana_ledger::{
bank_forks::BankForks, blockstore::Blockstore, rooted_slot_iterator::RootedSlotIterator,
};
use solana_perf::packet::PACKET_DATA_SIZE;
use solana_runtime::bank::Bank;
use solana_sdk::{
clock::{Slot, UnixTimestamp},
@@ -434,9 +433,7 @@ impl JsonRpcRequestProcessor {
signature: Signature,
commitment: Option<CommitmentConfig>,
) -> Option<transaction::Result<()>> {
self.bank(commitment)
.get_signature_status_slot(&signature)
.map(|(_, status)| status)
self.bank(commitment).get_signature_status(&signature)
}
pub fn get_signature_statuses(
@@ -1392,17 +1389,16 @@ impl RpcSol for RpcSolImpl {
pub mod tests {
use super::*;
use crate::{
commitment::BlockCommitment, contact_info::ContactInfo,
commitment::BlockCommitment,
contact_info::ContactInfo,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
replay_stage::tests::create_test_transactions_and_populate_blockstore,
};
use bincode::deserialize;
use jsonrpc_core::{MetaIoHandler, Output, Response, Value};
use solana_ledger::{
blockstore::entries_to_test_shreds,
blockstore_processor::fill_blockstore_slot_with_ticks,
entry::next_entry_mut,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
blockstore::entries_to_test_shreds, blockstore_processor::fill_blockstore_slot_with_ticks,
entry::next_entry_mut, get_tmp_ledger_path,
};
use solana_sdk::{
fee_calculator::DEFAULT_BURN_PERCENT,

View File

@@ -316,13 +316,13 @@ mod tests {
use super::*;
use crate::{
commitment::{BlockCommitment, BlockCommitmentCache},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
rpc_subscriptions::tests::robust_poll_or_panic,
};
use jsonrpc_core::{futures::sync::mpsc, Response};
use jsonrpc_pubsub::{PubSubHandler, Session};
use solana_budget_program::{self, budget_instruction};
use solana_ledger::bank_forks::BankForks;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_runtime::bank::Bank;
use solana_sdk::{
pubkey::Pubkey,

View File

@@ -85,19 +85,9 @@ impl RpcRequestMiddleware {
}
fn get(&self, path: &str) -> RequestMiddlewareAction {
let stem = path.split_at(1).1; // Drop leading '/' from path
let filename = {
match path {
"/genesis.tar.bz2" => self.ledger_path.join(stem),
_ => self
.snapshot_config
.as_ref()
.unwrap()
.snapshot_package_output_path
.join(stem),
}
};
let filename = self.ledger_path.join(
path.split_at(1).1, // Drop leading '/' from path
);
info!("get {} -> {:?}", path, filename);
RequestMiddlewareAction::Respond {
@@ -248,11 +238,12 @@ impl JsonRpcService {
#[cfg(test)]
mod tests {
use super::*;
use crate::{contact_info::ContactInfo, rpc::tests::create_validator_exit};
use solana_ledger::{
use crate::{
contact_info::ContactInfo,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
rpc::tests::create_validator_exit,
};
use solana_ledger::get_tmp_ledger_path;
use solana_runtime::bank::Bank;
use solana_sdk::signature::Signer;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};

View File

@@ -616,11 +616,13 @@ impl RpcSubscriptions {
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::commitment::BlockCommitment;
use crate::{
commitment::BlockCommitment,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
};
use jsonrpc_core::futures::{self, stream::Stream};
use jsonrpc_pubsub::typed::Subscriber;
use solana_budget_program;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_sdk::{
signature::{Keypair, Signer},
system_transaction,

View File

@@ -4,7 +4,7 @@ use std::io;
use std::net::{SocketAddr, UdpSocket};
#[cfg(not(target_os = "linux"))]
pub fn send_mmsg(sock: &UdpSocket, packets: &[(&Vec<u8>, &SocketAddr)]) -> io::Result<usize> {
pub fn send_mmsg(sock: &UdpSocket, packets: &mut [(Vec<u8>, &SocketAddr)]) -> io::Result<usize> {
let count = packets.len();
for (p, a) in packets {
sock.send_to(p, *a)?;
@@ -18,7 +18,7 @@ use libc::{iovec, mmsghdr, sockaddr_in, sockaddr_in6};
#[cfg(target_os = "linux")]
fn mmsghdr_for_packet(
packet: &[u8],
packet: &mut [u8],
dest: &SocketAddr,
index: usize,
addr_in_len: u32,
@@ -32,7 +32,7 @@ fn mmsghdr_for_packet(
use std::mem;
iovs.push(iovec {
iov_base: packet.as_ptr() as *mut c_void,
iov_base: packet.as_mut_ptr() as *mut c_void,
iov_len: packet.len(),
});
@@ -57,7 +57,7 @@ fn mmsghdr_for_packet(
}
#[cfg(target_os = "linux")]
pub fn send_mmsg(sock: &UdpSocket, packets: &[(&Vec<u8>, &SocketAddr)]) -> io::Result<usize> {
pub fn send_mmsg(sock: &UdpSocket, packets: &mut [(Vec<u8>, &SocketAddr)]) -> io::Result<usize> {
use libc::{sendmmsg, socklen_t};
use std::mem;
use std::os::unix::io::AsRawFd;
@@ -73,7 +73,7 @@ pub fn send_mmsg(sock: &UdpSocket, packets: &[(&Vec<u8>, &SocketAddr)]) -> io::R
let sock_fd = sock.as_raw_fd();
let mut hdrs: Vec<mmsghdr> = packets
.iter()
.iter_mut()
.enumerate()
.map(|(i, (packet, dest))| {
mmsghdr_for_packet(
@@ -160,11 +160,12 @@ mod tests {
let addr = reader.local_addr().unwrap();
let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
let packets: Vec<_> = (0..32).map(|_| vec![0u8; PACKET_DATA_SIZE]).collect();
let packet_refs: Vec<_> = packets.iter().map(|p| (p, &addr)).collect();
let mut packets: Vec<_> = (0..32)
.map(|_| (vec![0u8; PACKET_DATA_SIZE], &addr))
.collect();
let sent = send_mmsg(&sender, &packet_refs).ok();
assert_eq!(sent, Some(32));
let sent = send_mmsg(&sender, &mut packets);
assert_matches!(sent, Ok(32));
let mut packets = vec![Packet::default(); 32];
let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
@@ -181,15 +182,18 @@ mod tests {
let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
let packets: Vec<_> = (0..32).map(|_| vec![0u8; PACKET_DATA_SIZE]).collect();
let packet_refs: Vec<_> = packets
.iter()
.enumerate()
.map(|(i, p)| if i < 16 { (p, &addr) } else { (p, &addr2) })
let mut packets: Vec<_> = (0..32)
.map(|i| {
if i < 16 {
(vec![0u8; PACKET_DATA_SIZE], &addr)
} else {
(vec![0u8; PACKET_DATA_SIZE], &addr2)
}
})
.collect();
let sent = send_mmsg(&sender, &packet_refs).ok();
assert_eq!(sent, Some(32));
let sent = send_mmsg(&sender, &mut packets);
assert_matches!(sent, Ok(32));
let mut packets = vec![Packet::default(); 32];
let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
@@ -222,9 +226,8 @@ mod tests {
&sender,
&mut packet.data[..packet.meta.size],
&[&addr, &addr2, &addr3, &addr4],
)
.ok();
assert_eq!(sent, Some(4));
);
assert_matches!(sent, Ok(4));
let mut packets = vec![Packet::default(); 32];
let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;

View File

@@ -1,26 +1,24 @@
use crate::packet::limited_deserialize;
use crate::streamer::{PacketReceiver, PacketSender};
use crate::{
cluster_info::{ClusterInfo, ClusterInfoError},
cluster_slots::ClusterSlots,
contact_info::ContactInfo,
repair_service::RepairStats,
packet::Packet,
result::{Error, Result},
weighted_shuffle::weighted_best,
};
use bincode::serialize;
use rand::{thread_rng, Rng};
use solana_ledger::blockstore::Blockstore;
use solana_measure::measure::Measure;
use solana_measure::thread_mem_usage;
use solana_metrics::{datapoint_debug, inc_new_counter_debug};
use solana_perf::packet::{limited_deserialize, Packet, Packets, PacketsRecycler};
use solana_perf::packet::{Packets, PacketsRecycler};
use solana_sdk::{
clock::Slot,
pubkey::Pubkey,
signature::{Keypair, Signer},
timing::duration_as_ms,
};
use solana_streamer::streamer::{PacketReceiver, PacketSender};
use std::{
collections::HashMap,
net::SocketAddr,
sync::atomic::{AtomicBool, Ordering},
sync::{Arc, RwLock},
@@ -48,17 +46,6 @@ impl RepairType {
}
}
#[derive(Default)]
pub struct ServeRepairStats {
pub total_packets: usize,
pub dropped_packets: usize,
pub processed: usize,
pub self_repair: usize,
pub window_index: usize,
pub highest_window_index: usize,
pub orphan: usize,
}
/// Window protocol messages
#[derive(Serialize, Deserialize, Debug)]
enum RepairProtocol {
@@ -75,8 +62,6 @@ pub struct ServeRepair {
cluster_info: Arc<RwLock<ClusterInfo>>,
}
type RepairCache = HashMap<Slot, (Vec<ContactInfo>, Vec<(u64, usize)>)>;
impl ServeRepair {
/// Without a valid keypair gossip will not function. Only useful for tests.
pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
@@ -119,7 +104,6 @@ impl ServeRepair {
from_addr: &SocketAddr,
blockstore: Option<&Arc<Blockstore>>,
request: RepairProtocol,
stats: &mut ServeRepairStats,
) -> Option<Packets> {
let now = Instant::now();
@@ -127,14 +111,18 @@ impl ServeRepair {
let my_id = me.read().unwrap().keypair.pubkey();
let from = Self::get_repair_sender(&request);
if from.id == my_id {
stats.self_repair += 1;
warn!(
"{}: Ignored received repair request from ME {}",
my_id, from.id,
);
inc_new_counter_debug!("serve_repair-handle-repair--eq", 1);
return None;
}
let (res, label) = {
match &request {
RepairProtocol::WindowIndex(from, slot, shred_index) => {
stats.window_index += 1;
inc_new_counter_debug!("serve_repair-request-window-index", 1);
(
Self::run_window_request(
recycler,
@@ -150,7 +138,7 @@ impl ServeRepair {
}
RepairProtocol::HighestWindowIndex(_, slot, highest_index) => {
stats.highest_window_index += 1;
inc_new_counter_debug!("serve_repair-request-highest-window-index", 1);
(
Self::run_highest_window_request(
recycler,
@@ -163,7 +151,7 @@ impl ServeRepair {
)
}
RepairProtocol::Orphan(_, slot) => {
stats.orphan += 1;
inc_new_counter_debug!("serve_repair-request-orphan", 1);
(
Self::run_orphan(
recycler,
@@ -197,7 +185,6 @@ impl ServeRepair {
blockstore: Option<&Arc<Blockstore>>,
requests_receiver: &PacketReceiver,
response_sender: &PacketSender,
stats: &mut ServeRepairStats,
max_packets: &mut usize,
) -> Result<()> {
//TODO cache connections
@@ -205,23 +192,17 @@ impl ServeRepair {
let mut reqs_v = vec![requests_receiver.recv_timeout(timeout)?];
let mut total_packets = reqs_v[0].packets.len();
let mut dropped_packets = 0;
while let Ok(more) = requests_receiver.try_recv() {
total_packets += more.packets.len();
if total_packets < *max_packets {
// Drop the rest in the channel in case of dos
reqs_v.push(more);
} else {
dropped_packets += more.packets.len();
}
}
stats.dropped_packets += dropped_packets;
stats.total_packets += total_packets;
let mut time = Measure::start("repair::handle_packets");
for reqs in reqs_v {
Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender, stats);
Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
}
time.stop();
if total_packets >= *max_packets {
@@ -234,34 +215,6 @@ impl ServeRepair {
Ok(())
}
fn report_reset_stats(me: &Arc<RwLock<Self>>, stats: &mut ServeRepairStats) {
if stats.self_repair > 0 {
let my_id = me.read().unwrap().keypair.pubkey();
warn!(
"{}: Ignored received repair requests from ME: {}",
my_id, stats.self_repair,
);
inc_new_counter_debug!("serve_repair-handle-repair--eq", stats.self_repair);
}
inc_new_counter_info!("serve_repair-total_packets", stats.total_packets);
inc_new_counter_info!("serve_repair-dropped_packets", stats.dropped_packets);
debug!(
"repair_listener: total_packets: {} passed: {}",
stats.total_packets, stats.processed
);
inc_new_counter_debug!("serve_repair-request-window-index", stats.window_index);
inc_new_counter_debug!(
"serve_repair-request-highest-window-index",
stats.highest_window_index
);
inc_new_counter_debug!("serve_repair-request-orphan", stats.orphan);
*stats = ServeRepairStats::default();
}
pub fn listen(
me: Arc<RwLock<Self>>,
blockstore: Option<Arc<Blockstore>>,
@@ -274,8 +227,6 @@ impl ServeRepair {
Builder::new()
.name("solana-repair-listen".to_string())
.spawn(move || {
let mut last_print = Instant::now();
let mut stats = ServeRepairStats::default();
let mut max_packets = 1024;
loop {
let result = Self::run_listen(
@@ -284,7 +235,6 @@ impl ServeRepair {
blockstore.as_ref(),
&requests_receiver,
&response_sender,
&mut stats,
&mut max_packets,
);
match result {
@@ -294,10 +244,6 @@ impl ServeRepair {
if exit.load(Ordering::Relaxed) {
return;
}
if last_print.elapsed().as_secs() > 2 {
Self::report_reset_stats(&me, &mut stats);
last_print = Instant::now();
}
thread_mem_usage::datapoint("solana-repair-listen");
}
})
@@ -310,7 +256,6 @@ impl ServeRepair {
blockstore: Option<&Arc<Blockstore>>,
packets: Packets,
response_sender: &PacketSender,
stats: &mut ServeRepairStats,
) {
// iter over the packets, collect pulls separately and process everything else
let allocated = thread_mem_usage::Allocatedp::default();
@@ -320,9 +265,7 @@ impl ServeRepair {
limited_deserialize(&packet.data[..packet.meta.size])
.into_iter()
.for_each(|request| {
stats.processed += 1;
let rsp =
Self::handle_repair(me, recycler, &from_addr, blockstore, request, stats);
let rsp = Self::handle_repair(me, recycler, &from_addr, blockstore, request);
if let Some(rsp) = rsp {
let _ignore_disconnect = response_sender.send(rsp);
}
@@ -352,50 +295,44 @@ impl ServeRepair {
Ok(out)
}
pub fn repair_request(
&self,
cluster_slots: &ClusterSlots,
repair_request: &RepairType,
cache: &mut RepairCache,
repair_stats: &mut RepairStats,
) -> Result<(SocketAddr, Vec<u8>)> {
pub fn repair_request(&self, repair_request: &RepairType) -> Result<(SocketAddr, Vec<u8>)> {
// find a peer that appears to be accepting replication and has the desired slot, as indicated
// by a valid tvu port location
if cache.get(&repair_request.slot()).is_none() {
let repair_peers: Vec<_> = self
.cluster_info
.read()
.unwrap()
.repair_peers(repair_request.slot());
if repair_peers.is_empty() {
return Err(ClusterInfoError::NoPeers.into());
}
let weights = cluster_slots.compute_weights(repair_request.slot(), &repair_peers);
cache.insert(repair_request.slot(), (repair_peers, weights));
let valid: Vec<_> = self
.cluster_info
.read()
.unwrap()
.repair_peers(repair_request.slot());
if valid.is_empty() {
return Err(ClusterInfoError::NoPeers.into());
}
let (repair_peers, weights) = cache.get(&repair_request.slot()).unwrap();
let n = weighted_best(&weights, Pubkey::new_rand().to_bytes());
let addr = repair_peers[n].serve_repair; // send the request to the peer's serve_repair port
let out = self.map_repair_request(repair_request, repair_stats)?;
let n = thread_rng().gen::<usize>() % valid.len();
let addr = valid[n].serve_repair; // send the request to the peer's serve_repair port
let out = self.map_repair_request(repair_request)?;
Ok((addr, out))
}
pub fn map_repair_request(
&self,
repair_request: &RepairType,
repair_stats: &mut RepairStats,
) -> Result<Vec<u8>> {
pub fn map_repair_request(&self, repair_request: &RepairType) -> Result<Vec<u8>> {
match repair_request {
RepairType::Shred(slot, shred_index) => {
repair_stats.shred.update(*slot);
datapoint_debug!(
"serve_repair-repair",
("repair-slot", *slot, i64),
("repair-ix", *shred_index, i64)
);
Ok(self.window_index_request_bytes(*slot, *shred_index)?)
}
RepairType::HighestShred(slot, shred_index) => {
repair_stats.highest_shred.update(*slot);
datapoint_info!(
"serve_repair-repair_highest",
("repair-highest-slot", *slot, i64),
("repair-highest-ix", *shred_index, i64)
);
Ok(self.window_highest_index_request_bytes(*slot, *shred_index)?)
}
RepairType::Orphan(slot) => {
repair_stats.orphan.update(*slot);
datapoint_info!("serve_repair-repair_orphan", ("repair-orphan", *slot, i64));
Ok(self.orphan_bytes(*slot)?)
}
}
@@ -652,16 +589,10 @@ mod tests {
#[test]
fn window_index_request() {
let cluster_slots = ClusterSlots::default();
let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(me)));
let serve_repair = ServeRepair::new(cluster_info.clone());
let rv = serve_repair.repair_request(
&cluster_slots,
&RepairType::Shred(0, 0),
&mut HashMap::new(),
&mut RepairStats::default(),
);
let rv = serve_repair.repair_request(&RepairType::Shred(0, 0));
assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
let serve_repair_addr = socketaddr!([127, 0, 0, 1], 1243);
@@ -682,12 +613,7 @@ mod tests {
};
cluster_info.write().unwrap().insert_info(nxt.clone());
let rv = serve_repair
.repair_request(
&cluster_slots,
&RepairType::Shred(0, 0),
&mut HashMap::new(),
&mut RepairStats::default(),
)
.repair_request(&RepairType::Shred(0, 0))
.unwrap();
assert_eq!(nxt.serve_repair, serve_repair_addr);
assert_eq!(rv.0, nxt.serve_repair);
@@ -714,12 +640,7 @@ mod tests {
while !one || !two {
//this randomly picks an option, so eventually it should pick both
let rv = serve_repair
.repair_request(
&cluster_slots,
&RepairType::Shred(0, 0),
&mut HashMap::new(),
&mut RepairStats::default(),
)
.repair_request(&RepairType::Shred(0, 0))
.unwrap();
if rv.0 == serve_repair_addr {
one = true;

View File

@@ -1,7 +1,7 @@
use crate::serve_repair::ServeRepair;
use crate::streamer;
use solana_ledger::blockstore::Blockstore;
use solana_perf::recycler::Recycler;
use solana_streamer::streamer;
use std::net::UdpSocket;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::channel;

View File

@@ -1,5 +1,7 @@
//! The `shred_fetch_stage` pulls shreds from UDP sockets and sends it to a channel.
use crate::packet::{Packet, PacketsRecycler};
use crate::streamer::{self, PacketReceiver, PacketSender};
use bv::BitVec;
use solana_ledger::bank_forks::BankForks;
use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
@@ -7,10 +9,9 @@ use solana_ledger::shred::{
OFFSET_OF_SHRED_INDEX, OFFSET_OF_SHRED_SLOT, SIZE_OF_SHRED_INDEX, SIZE_OF_SHRED_SLOT,
};
use solana_perf::cuda_runtime::PinnedVec;
use solana_perf::packet::{limited_deserialize, Packet, PacketsRecycler};
use solana_perf::packet::limited_deserialize;
use solana_perf::recycler::Recycler;
use solana_sdk::clock::Slot;
use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
use std::collections::HashMap;
use std::net::UdpSocket;
use std::sync::atomic::AtomicBool;
@@ -101,7 +102,9 @@ impl ShredFetchStage {
last_root = bank_forks_r.root();
let working_bank = bank_forks_r.working_bank();
last_slot = working_bank.slot();
let root_bank = bank_forks_r.root_bank();
let root_bank = bank_forks_r
.get(bank_forks_r.root())
.expect("Root bank should exist");
slots_per_epoch = root_bank.get_slots_in_epoch(root_bank.epoch());
}
}

View File

@@ -1,11 +1,11 @@
#![allow(clippy::implicit_hasher)]
use crate::packet::{limited_deserialize, Packets};
use crate::sigverify;
use crate::sigverify_stage::SigVerifier;
use solana_ledger::bank_forks::BankForks;
use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
use solana_ledger::shred::{OFFSET_OF_SHRED_SLOT, SIZE_OF_SHRED_SLOT};
use solana_ledger::sigverify_shreds::verify_shreds_gpu;
use solana_perf::packet::{limited_deserialize, Packets};
use solana_perf::recycler_cache::RecyclerCache;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, RwLock};
@@ -72,9 +72,9 @@ impl SigVerifier for ShredSigVerifier {
#[cfg(test)]
pub mod tests {
use super::*;
use solana_ledger::genesis_utils::create_genesis_config_with_leader;
use crate::genesis_utils::create_genesis_config_with_leader;
use crate::packet::Packet;
use solana_ledger::shred::{Shred, Shredder};
use solana_perf::packet::Packet;
use solana_runtime::bank::Bank;
use solana_sdk::signature::{Keypair, Signer};

View File

@@ -5,14 +5,14 @@
//! transaction. All processing is done on the CPU by default and on a GPU
//! if perf-libs are available
use crate::packet::Packets;
use crate::sigverify;
use crate::streamer::{self, PacketReceiver, StreamerError};
use crossbeam_channel::{SendError, Sender as CrossbeamSender};
use solana_measure::measure::Measure;
use solana_metrics::datapoint_debug;
use solana_perf::packet::Packets;
use solana_perf::perf_libs;
use solana_sdk::timing;
use solana_streamer::streamer::{self, PacketReceiver, StreamerError};
use std::sync::mpsc::{Receiver, RecvTimeoutError};
use std::sync::{Arc, Mutex};
use std::thread::{self, Builder, JoinHandle};

View File

@@ -137,12 +137,10 @@ mod tests {
.collect();
// Create some fake snapshot
fs::create_dir_all(&snapshots_dir).unwrap();
let snapshots_paths: Vec<_> = (0..5)
.map(|i| {
let snapshot_file_name = format!("{}", i);
let snapshots_dir = snapshots_dir.join(&snapshot_file_name);
fs::create_dir_all(&snapshots_dir).unwrap();
let fake_snapshot_path = snapshots_dir.join(&snapshot_file_name);
let fake_snapshot_path = snapshots_dir.join(format!("fake_snapshot_{}", i));
let mut fake_snapshot_file = OpenOptions::new()
.read(true)
.write(true)
@@ -159,9 +157,7 @@ mod tests {
let link_snapshots_dir = tempfile::tempdir_in(&temp_dir).unwrap();
for snapshots_path in snapshots_paths {
let snapshot_file_name = snapshots_path.file_name().unwrap();
let link_snapshots_dir = link_snapshots_dir.path().join(snapshot_file_name);
fs::create_dir_all(&link_snapshots_dir).unwrap();
let link_path = link_snapshots_dir.join(snapshot_file_name);
let link_path = link_snapshots_dir.path().join(snapshot_file_name);
fs::hard_link(&snapshots_path, &link_path).unwrap();
}

View File

@@ -661,8 +661,8 @@ pub fn test_cluster_info(id: &Pubkey) -> Arc<RwLock<ClusterInfo>> {
#[cfg(test)]
mod tests {
use super::*;
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use rayon::prelude::*;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_runtime::bank::Bank;
use solana_sdk::{
hash::Hasher,

View File

@@ -3,18 +3,16 @@
use crate::{
banking_stage::BankingStage,
broadcast_stage::{BroadcastStage, BroadcastStageType, RetransmitSlotsReceiver},
broadcast_stage::{BroadcastStage, BroadcastStageType},
cluster_info::ClusterInfo,
cluster_info_vote_listener::{ClusterInfoVoteListener, VoteTracker},
cluster_info_vote_listener::ClusterInfoVoteListener,
fetch_stage::FetchStage,
poh_recorder::{PohRecorder, WorkingBankEntry},
sigverify::TransactionSigVerifier,
sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
};
use crossbeam_channel::unbounded;
use solana_ledger::{
bank_forks::BankForks, blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
};
use solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusSender};
use std::{
net::UdpSocket,
sync::{
@@ -39,7 +37,6 @@ impl Tpu {
cluster_info: &Arc<RwLock<ClusterInfo>>,
poh_recorder: &Arc<Mutex<PohRecorder>>,
entry_receiver: Receiver<WorkingBankEntry>,
retransmit_slots_receiver: RetransmitSlotsReceiver,
transactions_sockets: Vec<UdpSocket>,
tpu_forwards_sockets: Vec<UdpSocket>,
broadcast_sockets: Vec<UdpSocket>,
@@ -49,8 +46,6 @@ impl Tpu {
broadcast_type: &BroadcastStageType,
exit: &Arc<AtomicBool>,
shred_version: u16,
vote_tracker: Arc<VoteTracker>,
bank_forks: Arc<RwLock<BankForks>>,
) -> Self {
let (packet_sender, packet_receiver) = channel();
let fetch_stage = FetchStage::new_with_sender(
@@ -77,8 +72,6 @@ impl Tpu {
sigverify_disabled,
verified_vote_sender,
&poh_recorder,
vote_tracker,
bank_forks,
);
let banking_stage = BankingStage::new(
@@ -93,7 +86,6 @@ impl Tpu {
broadcast_sockets,
cluster_info.clone(),
entry_receiver,
retransmit_slots_receiver,
&exit,
blockstore,
shred_version,

View File

@@ -2,12 +2,9 @@
//! validation pipeline in software.
use crate::{
accounts_background_service::AccountsBackgroundService,
accounts_hash_verifier::AccountsHashVerifier,
broadcast_stage::RetransmitSlotsSender,
blockstream_service::BlockstreamService,
cluster_info::ClusterInfo,
cluster_info_vote_listener::VoteTracker,
cluster_slots::ClusterSlots,
commitment::BlockCommitmentCache,
ledger_cleanup_service::LedgerCleanupService,
poh_recorder::PohRecorder,
@@ -35,6 +32,7 @@ use solana_sdk::{
use std::collections::HashSet;
use std::{
net::UdpSocket,
path::PathBuf,
sync::{
atomic::AtomicBool,
mpsc::{channel, Receiver},
@@ -48,8 +46,8 @@ pub struct Tvu {
sigverify_stage: SigVerifyStage,
retransmit_stage: RetransmitStage,
replay_stage: ReplayStage,
blockstream_service: Option<BlockstreamService>,
ledger_cleanup_service: Option<LedgerCleanupService>,
accounts_background_service: AccountsBackgroundService,
storage_stage: StorageStage,
accounts_hash_verifier: AccountsHashVerifier,
}
@@ -81,13 +79,14 @@ impl Tvu {
#[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
pub fn new(
vote_account: &Pubkey,
authorized_voter_keypairs: Vec<Arc<Keypair>>,
voting_keypair: Option<Arc<Keypair>>,
storage_keypair: &Arc<Keypair>,
bank_forks: &Arc<RwLock<BankForks>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sockets: Sockets,
blockstore: Arc<Blockstore>,
storage_state: &StorageState,
blockstream_unix_socket: Option<&PathBuf>,
ledger_signal_receiver: Receiver<bool>,
subscriptions: &Arc<RpcSubscriptions>,
poh_recorder: &Arc<Mutex<PohRecorder>>,
@@ -99,8 +98,6 @@ impl Tvu {
transaction_status_sender: Option<TransactionStatusSender>,
rewards_recorder_sender: Option<RewardsRecorderSender>,
snapshot_package_sender: Option<SnapshotPackageSender>,
vote_tracker: Arc<VoteTracker>,
retransmit_slots_sender: RetransmitSlotsSender,
tvu_config: TvuConfig,
) -> Self {
let keypair: Arc<Keypair> = cluster_info
@@ -146,7 +143,6 @@ impl Tvu {
)
};
let cluster_slots = Arc::new(ClusterSlots::default());
let retransmit_stage = RetransmitStage::new(
bank_forks.clone(),
leader_schedule_cache,
@@ -160,9 +156,9 @@ impl Tvu {
*bank_forks.read().unwrap().working_bank().epoch_schedule(),
cfg,
tvu_config.shred_version,
cluster_slots.clone(),
);
let (blockstream_slot_sender, blockstream_slot_receiver) = channel();
let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();
let (accounts_hash_sender, accounts_hash_receiver) = channel();
@@ -179,10 +175,11 @@ impl Tvu {
let replay_stage_config = ReplayStageConfig {
my_pubkey: keypair.pubkey(),
vote_account: *vote_account,
authorized_voter_keypairs,
voting_keypair,
exit: exit.clone(),
subscriptions: subscriptions.clone(),
leader_schedule_cache: leader_schedule_cache.clone(),
slot_full_senders: vec![blockstream_slot_sender],
latest_root_senders: vec![ledger_cleanup_slot_sender],
accounts_hash_sender: Some(accounts_hash_sender),
block_commitment_cache: block_commitment_cache.clone(),
@@ -197,11 +194,20 @@ impl Tvu {
cluster_info.clone(),
ledger_signal_receiver,
poh_recorder.clone(),
vote_tracker,
cluster_slots,
retransmit_slots_sender,
);
let blockstream_service = if let Some(blockstream_unix_socket) = blockstream_unix_socket {
let blockstream_service = BlockstreamService::new(
blockstream_slot_receiver,
blockstore.clone(),
blockstream_unix_socket,
&exit,
);
Some(blockstream_service)
} else {
None
};
let ledger_cleanup_service = tvu_config.max_ledger_slots.map(|max_ledger_slots| {
LedgerCleanupService::new(
ledger_cleanup_slot_receiver,
@@ -211,8 +217,6 @@ impl Tvu {
)
});
let accounts_background_service = AccountsBackgroundService::new(bank_forks.clone(), &exit);
let storage_stage = StorageStage::new(
storage_state,
root_bank_receiver,
@@ -230,8 +234,8 @@ impl Tvu {
sigverify_stage,
retransmit_stage,
replay_stage,
blockstream_service,
ledger_cleanup_service,
accounts_background_service,
storage_stage,
accounts_hash_verifier,
}
@@ -242,10 +246,12 @@ impl Tvu {
self.fetch_stage.join()?;
self.sigverify_stage.join()?;
self.storage_stage.join()?;
if self.blockstream_service.is_some() {
self.blockstream_service.unwrap().join()?;
}
if self.ledger_cleanup_service.is_some() {
self.ledger_cleanup_service.unwrap().join()?;
}
self.accounts_background_service.join()?;
self.replay_stage.join()?;
self.accounts_hash_verifier.join()?;
Ok(())
@@ -257,8 +263,8 @@ pub mod tests {
use super::*;
use crate::banking_stage::create_test_recorder;
use crate::cluster_info::{ClusterInfo, Node};
use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_ledger::create_new_tmp_ledger;
use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
use solana_runtime::bank::Bank;
use std::sync::atomic::Ordering;
@@ -287,14 +293,13 @@ pub mod tests {
let bank = bank_forks.working_bank();
let (exit, poh_recorder, poh_service, _entry_receiver) =
create_test_recorder(&bank, &blockstore, None);
let vote_keypair = Keypair::new();
let voting_keypair = Keypair::new();
let storage_keypair = Arc::new(Keypair::new());
let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
let (retransmit_slots_sender, _retransmit_slots_receiver) = unbounded();
let tvu = Tvu::new(
&vote_keypair.pubkey(),
vec![Arc::new(vote_keypair)],
&voting_keypair.pubkey(),
Some(Arc::new(voting_keypair)),
&storage_keypair,
&Arc::new(RwLock::new(bank_forks)),
&cref1,
@@ -308,6 +313,7 @@ pub mod tests {
},
blockstore,
&StorageState::default(),
None,
l_receiver,
&Arc::new(RpcSubscriptions::new(
&exit,
@@ -322,8 +328,6 @@ pub mod tests {
None,
None,
None,
Arc::new(VoteTracker::new(&bank)),
retransmit_slots_sender,
TvuConfig::default(),
);
exit.store(true, Ordering::Relaxed);

View File

@@ -3,7 +3,6 @@
use crate::{
broadcast_stage::BroadcastStageType,
cluster_info::{ClusterInfo, Node},
cluster_info_vote_listener::VoteTracker,
commitment::BlockCommitmentCache,
contact_info::ContactInfo,
gossip_service::{discover_cluster, GossipService},
@@ -30,7 +29,6 @@ use solana_ledger::{
blockstore::{Blockstore, CompletedSlotsReceiver},
blockstore_processor::{self, BankForksInfo},
create_new_tmp_ledger,
hardened_unpack::open_genesis_config,
leader_schedule::FixedSchedule,
leader_schedule_cache::LeaderScheduleCache,
};
@@ -65,6 +63,7 @@ pub struct ValidatorConfig {
pub expected_genesis_hash: Option<Hash>,
pub expected_shred_version: Option<u16>,
pub voting_disabled: bool,
pub blockstream_unix_socket: Option<PathBuf>,
pub storage_slots_per_turn: u64,
pub account_paths: Vec<PathBuf>,
pub rpc_config: JsonRpcConfig,
@@ -91,6 +90,7 @@ impl Default for ValidatorConfig {
expected_genesis_hash: None,
expected_shred_version: None,
voting_disabled: false,
blockstream_unix_socket: None,
storage_slots_per_turn: DEFAULT_SLOTS_PER_TURN,
max_ledger_slots: None,
account_paths: Vec::new(),
@@ -151,7 +151,7 @@ impl Validator {
keypair: &Arc<Keypair>,
ledger_path: &Path,
vote_account: &Pubkey,
mut authorized_voter_keypairs: Vec<Arc<Keypair>>,
authorized_voter: &Arc<Keypair>,
storage_keypair: &Arc<Keypair>,
entrypoint_info_option: Option<&ContactInfo>,
poh_verify: bool,
@@ -162,15 +162,7 @@ impl Validator {
warn!("identity: {}", id);
warn!("vote account: {}", vote_account);
if config.voting_disabled {
warn!("voting disabled");
authorized_voter_keypairs.clear();
} else {
for authorized_voter_keypair in &authorized_voter_keypairs {
warn!("authorized voter: {}", authorized_voter_keypair.pubkey());
}
}
warn!("authorized voter: {}", authorized_voter.pubkey());
report_target_features();
info!("entrypoint: {:?}", entrypoint_info_option);
@@ -189,7 +181,12 @@ impl Validator {
completed_slots_receiver,
leader_schedule_cache,
snapshot_hash,
) = new_banks_from_blockstore(config, ledger_path, poh_verify);
) = new_banks_from_blockstore(
config.expected_genesis_hash,
ledger_path,
poh_verify,
config,
);
let leader_schedule_cache = Arc::new(leader_schedule_cache);
let exit = Arc::new(AtomicBool::new(false));
@@ -389,12 +386,13 @@ impl Validator {
"New shred signal for the TVU should be the same as the clear bank signal."
);
let vote_tracker = Arc::new({ VoteTracker::new(bank_forks.read().unwrap().root_bank()) });
let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
let tvu = Tvu::new(
vote_account,
authorized_voter_keypairs,
if config.voting_disabled {
None
} else {
Some(authorized_voter.clone())
},
storage_keypair,
&bank_forks,
&cluster_info,
@@ -425,6 +423,7 @@ impl Validator {
},
blockstore.clone(),
&storage_state,
config.blockstream_unix_socket.as_ref(),
ledger_signal_receiver,
&subscriptions,
&poh_recorder,
@@ -436,8 +435,6 @@ impl Validator {
transaction_status_sender.clone(),
rewards_recorder_sender,
snapshot_package_sender,
vote_tracker.clone(),
retransmit_slots_sender,
TvuConfig {
max_ledger_slots: config.max_ledger_slots,
sigverify_disabled: config.dev_sigverify_disabled,
@@ -457,7 +454,6 @@ impl Validator {
&cluster_info,
&poh_recorder,
entry_receiver,
retransmit_slots_receiver,
node.sockets.tpu,
node.sockets.tpu_forwards,
node.sockets.broadcast,
@@ -467,8 +463,6 @@ impl Validator {
&config.broadcast_stage_type,
&exit,
node.info.shred_version,
vote_tracker,
bank_forks,
);
datapoint_info!("validator-new", ("id", id.to_string(), String));
@@ -557,9 +551,10 @@ impl Validator {
#[allow(clippy::type_complexity)]
fn new_banks_from_blockstore(
config: &ValidatorConfig,
expected_genesis_hash: Option<Hash>,
blockstore_path: &Path,
poh_verify: bool,
config: &ValidatorConfig,
) -> (
GenesisConfig,
BankForks,
@@ -570,7 +565,10 @@ fn new_banks_from_blockstore(
LeaderScheduleCache,
Option<(Slot, Hash)>,
) {
let genesis_config = open_genesis_config(blockstore_path);
let genesis_config = GenesisConfig::load(blockstore_path).unwrap_or_else(|err| {
error!("Failed to load genesis from {:?}: {}", blockstore_path, err);
process::exit(1);
});
// This needs to be limited otherwise the state in the VoteAccount data
// grows too large
@@ -582,7 +580,7 @@ fn new_banks_from_blockstore(
let genesis_hash = genesis_config.hash();
info!("genesis hash: {}", genesis_hash);
if let Some(expected_genesis_hash) = config.expected_genesis_hash {
if let Some(expected_genesis_hash) = expected_genesis_hash {
if genesis_hash != expected_genesis_hash {
error!("genesis hash mismatch: expected {}", expected_genesis_hash);
error!(
@@ -670,16 +668,14 @@ pub struct TestValidator {
pub struct TestValidatorOptions {
pub fees: u64,
pub bootstrap_validator_lamports: u64,
pub mint_lamports: u64,
}
impl Default for TestValidatorOptions {
fn default() -> Self {
use solana_ledger::genesis_utils::BOOTSTRAP_VALIDATOR_LAMPORTS;
use crate::genesis_utils::BOOTSTRAP_VALIDATOR_LAMPORTS;
TestValidatorOptions {
fees: 0,
bootstrap_validator_lamports: BOOTSTRAP_VALIDATOR_LAMPORTS,
mint_lamports: 1_000_000,
}
}
}
@@ -690,15 +686,12 @@ impl TestValidator {
}
pub fn run_with_options(options: TestValidatorOptions) -> Self {
use solana_ledger::genesis_utils::{
create_genesis_config_with_leader_ex, GenesisConfigInfo,
};
use crate::genesis_utils::{create_genesis_config_with_leader_ex, GenesisConfigInfo};
use solana_sdk::fee_calculator::FeeRateGovernor;
let TestValidatorOptions {
fees,
bootstrap_validator_lamports,
mint_lamports,
} = options;
let node_keypair = Arc::new(Keypair::new());
let node = Node::new_localhost_with_pubkey(&node_keypair.pubkey());
@@ -709,7 +702,7 @@ impl TestValidator {
mint_keypair,
voting_keypair,
} = create_genesis_config_with_leader_ex(
mint_lamports,
1_000_000,
&contact_info.id,
42,
bootstrap_validator_lamports,
@@ -735,7 +728,7 @@ impl TestValidator {
&node_keypair,
&ledger_path,
&leader_voting_keypair.pubkey(),
vec![leader_voting_keypair.clone()],
&leader_voting_keypair,
&storage_keypair,
None,
true,
@@ -763,19 +756,16 @@ fn report_target_features() {
}
);
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
// Validator binaries built on a machine with AVX support will generate invalid opcodes
// when run on machines without AVX causing a non-obvious process abort. Instead detect
// the mismatch and error cleanly.
#[target_feature(enable = "avx")]
{
// Validator binaries built on a machine with AVX support will generate invalid opcodes
// when run on machines without AVX causing a non-obvious process abort. Instead detect
// the mismatch and error cleanly.
#[target_feature(enable = "avx")]
{
if is_x86_feature_detected!("avx") {
info!("AVX detected");
} else {
error!("Your machine does not have AVX support, please rebuild from source on your machine");
process::exit(1);
}
if is_x86_feature_detected!("avx") {
info!("AVX detected");
} else {
error!("Your machine does not have AVX support, please rebuild from source on your machine");
process::exit(1);
}
}
}
@@ -813,7 +803,7 @@ fn get_stake_percent_in_gossip(bank: &Arc<Bank>, cluster_info: &Arc<RwLock<Clust
#[cfg(test)]
mod tests {
use super::*;
use solana_ledger::genesis_utils::create_genesis_config_with_leader;
use crate::genesis_utils::create_genesis_config_with_leader;
use std::fs::remove_dir_all;
#[test]
@@ -843,7 +833,7 @@ mod tests {
&Arc::new(validator_keypair),
&validator_ledger_path,
&voting_keypair.pubkey(),
vec![voting_keypair.clone()],
&voting_keypair,
&storage_keypair,
Some(&leader_node.info),
true,
@@ -868,7 +858,7 @@ mod tests {
.genesis_config;
let (validator_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
ledger_paths.push(validator_ledger_path.clone());
let vote_account_keypair = Arc::new(Keypair::new());
let voting_keypair = Arc::new(Keypair::new());
let storage_keypair = Arc::new(Keypair::new());
let config = ValidatorConfig {
rpc_ports: Some((
@@ -881,8 +871,8 @@ mod tests {
validator_node,
&Arc::new(validator_keypair),
&validator_ledger_path,
&vote_account_keypair.pubkey(),
vec![vote_account_keypair.clone()],
&voting_keypair.pubkey(),
&voting_keypair,
&storage_keypair,
Some(&leader_node.info),
true,
@@ -891,7 +881,7 @@ mod tests {
})
.collect();
// Each validator can exit in parallel to speed many sequential calls to join`
// Each validator can exit in parallel to speed many sequential calls to `join`
validators.iter_mut().for_each(|v| v.exit());
// While join is called sequentially, the above exit call notified all the
// validators to exit from all their threads

View File

@@ -1,163 +0,0 @@
use crate::{
cluster_info_vote_listener::VerifiedVotePacketsReceiver, crds_value::CrdsValueLabel,
result::Result,
};
use solana_perf::packet::Packets;
use std::{collections::HashMap, ops::Deref, time::Duration};
#[derive(Default)]
pub struct VerifiedVotePackets(HashMap<CrdsValueLabel, (u64, Packets)>);
impl Deref for VerifiedVotePackets {
type Target = HashMap<CrdsValueLabel, (u64, Packets)>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl VerifiedVotePackets {
pub fn get_and_process_vote_packets(
&mut self,
vote_packets_receiver: &VerifiedVotePacketsReceiver,
last_update_version: &mut u64,
) -> Result<()> {
let timer = Duration::from_millis(200);
let vote_packets = vote_packets_receiver.recv_timeout(timer)?;
*last_update_version += 1;
for (label, packet) in vote_packets {
self.0.insert(label, (*last_update_version, packet));
}
while let Ok(vote_packets) = vote_packets_receiver.try_recv() {
for (label, packet) in vote_packets {
self.0.insert(label, (*last_update_version, packet));
}
}
Ok(())
}
pub fn get_latest_votes(&self, last_update_version: u64) -> (u64, Vec<Packets>) {
let mut new_update_version = last_update_version;
let msgs: Vec<_> = self
.iter()
.filter_map(|(_, (msg_update_version, msg))| {
if *msg_update_version > last_update_version {
new_update_version = std::cmp::max(*msg_update_version, new_update_version);
Some(msg)
} else {
None
}
})
.cloned()
.collect();
(new_update_version, msgs)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::result::Error;
use crossbeam_channel::{unbounded, RecvTimeoutError};
use solana_perf::packet::{Meta, Packet};
use solana_sdk::pubkey::Pubkey;
#[test]
fn test_get_latest_votes() {
let pubkey = Pubkey::new_rand();
let label1 = CrdsValueLabel::Vote(0 as u8, pubkey);
let label2 = CrdsValueLabel::Vote(1 as u8, pubkey);
let mut verified_vote_packets = VerifiedVotePackets(HashMap::new());
let data = Packet {
meta: Meta {
repair: true,
..Meta::default()
},
..Packet::default()
};
let none_empty_packets = Packets::new(vec![data, Packet::default()]);
verified_vote_packets
.0
.insert(label1, (2, none_empty_packets));
verified_vote_packets
.0
.insert(label2, (1, Packets::default()));
// Both updates have timestamps greater than 0, so both should be returned
let (new_update_version, updates) = verified_vote_packets.get_latest_votes(0);
assert_eq!(new_update_version, 2);
assert_eq!(updates.len(), 2);
// Only the nonempty packet had a timestamp greater than 1
let (new_update_version, updates) = verified_vote_packets.get_latest_votes(1);
assert_eq!(new_update_version, 2);
assert_eq!(updates.len(), 1);
assert!(updates[0].packets.len() > 0);
// If the given timestamp is greater than all timestamps in any update,
// returned timestamp should be the same as the given timestamp, and
// no updates should be returned
let (new_update_version, updates) = verified_vote_packets.get_latest_votes(3);
assert_eq!(new_update_version, 3);
assert!(updates.is_empty());
}
#[test]
fn test_get_and_process_vote_packets() {
let (s, r) = unbounded();
let pubkey = Pubkey::new_rand();
let label1 = CrdsValueLabel::Vote(0 as u8, pubkey);
let label2 = CrdsValueLabel::Vote(1 as u8, pubkey);
let mut update_version = 0;
s.send(vec![(label1.clone(), Packets::default())]).unwrap();
s.send(vec![(label2.clone(), Packets::default())]).unwrap();
let data = Packet {
meta: Meta {
repair: true,
..Meta::default()
},
..Packet::default()
};
let later_packets = Packets::new(vec![data, Packet::default()]);
s.send(vec![(label1.clone(), later_packets.clone())])
.unwrap();
let mut verified_vote_packets = VerifiedVotePackets(HashMap::new());
verified_vote_packets
.get_and_process_vote_packets(&r, &mut update_version)
.unwrap();
// Test timestamps for same batch are the same
let update_version1 = verified_vote_packets.get(&label1).unwrap().0;
assert_eq!(
update_version1,
verified_vote_packets.get(&label2).unwrap().0
);
// Test the later value overwrote the earlier one for this label
assert!(verified_vote_packets.get(&label1).unwrap().1.packets.len() > 1);
assert_eq!(
verified_vote_packets.get(&label2).unwrap().1.packets.len(),
0
);
// Test timestamp for next batch overwrites the original
s.send(vec![(label2.clone(), Packets::default())]).unwrap();
verified_vote_packets
.get_and_process_vote_packets(&r, &mut update_version)
.unwrap();
let update_version2 = verified_vote_packets.get(&label2).unwrap().0;
assert!(update_version2 > update_version1);
// Test empty doesn't bump the version
let before = update_version;
assert_matches!(
verified_vote_packets.get_and_process_vote_packets(&r, &mut update_version),
Err(Error::CrossbeamRecvTimeoutError(RecvTimeoutError::Timeout))
);
assert_eq!(before, update_version);
}
}

View File

@@ -1,12 +1,11 @@
//! `window_service` handles the data plane incoming shreds, storing them in
//! blockstore and retransmitting where required
//!
use crate::{
cluster_info::ClusterInfo,
cluster_slots::ClusterSlots,
repair_service::{RepairService, RepairStrategy},
result::{Error, Result},
};
use crate::cluster_info::ClusterInfo;
use crate::packet::Packets;
use crate::repair_service::{RepairService, RepairStrategy};
use crate::result::{Error, Result};
use crate::streamer::PacketSender;
use crossbeam_channel::{
unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
};
@@ -14,18 +13,14 @@ use rayon::iter::IntoParallelRefMutIterator;
use rayon::iter::ParallelIterator;
use rayon::ThreadPool;
use solana_ledger::bank_forks::BankForks;
use solana_ledger::blockstore::{
self, Blockstore, BlockstoreInsertionMetrics, MAX_DATA_SHREDS_PER_SLOT,
};
use solana_ledger::blockstore::{self, Blockstore, MAX_DATA_SHREDS_PER_SLOT};
use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
use solana_ledger::shred::Shred;
use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
use solana_perf::packet::Packets;
use solana_rayon_threadlimit::get_thread_count;
use solana_runtime::bank::Bank;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::timing::duration_as_ms;
use solana_streamer::streamer::PacketSender;
use std::net::UdpSocket;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, RwLock};
@@ -112,7 +107,6 @@ fn run_insert<F>(
blockstore: &Arc<Blockstore>,
leader_schedule_cache: &Arc<LeaderScheduleCache>,
handle_duplicate: F,
metrics: &mut BlockstoreInsertionMetrics,
) -> Result<()>
where
F: Fn(Shred) -> (),
@@ -124,13 +118,14 @@ where
shreds.append(&mut more_shreds)
}
blockstore.insert_shreds_handle_duplicate(
let blockstore_insert_metrics = blockstore.insert_shreds_handle_duplicate(
shreds,
Some(leader_schedule_cache),
false,
&handle_duplicate,
metrics,
)?;
blockstore_insert_metrics.report_metrics("recv-window-insert-shreds");
Ok(())
}
@@ -257,7 +252,6 @@ impl WindowService {
repair_strategy: RepairStrategy,
leader_schedule_cache: &Arc<LeaderScheduleCache>,
shred_filter: F,
cluster_slots: Arc<ClusterSlots>,
) -> WindowService
where
F: 'static
@@ -276,7 +270,6 @@ impl WindowService {
repair_socket,
cluster_info.clone(),
repair_strategy,
cluster_slots,
);
let (insert_sender, insert_receiver) = unbounded();
@@ -360,8 +353,6 @@ impl WindowService {
let handle_duplicate = |shred| {
let _ = duplicate_sender.send(shred);
};
let mut metrics = BlockstoreInsertionMetrics::default();
let mut last_print = Instant::now();
loop {
if exit.load(Ordering::Relaxed) {
break;
@@ -372,18 +363,11 @@ impl WindowService {
&blockstore,
&leader_schedule_cache,
&handle_duplicate,
&mut metrics,
) {
if Self::should_exit_on_error(e, &mut handle_timeout, &handle_error) {
break;
}
}
if last_print.elapsed().as_secs() > 2 {
metrics.report_metrics("recv-window-insert-shreds");
metrics = BlockstoreInsertionMetrics::default();
last_print = Instant::now();
}
}
})
.unwrap()
@@ -492,18 +476,20 @@ impl WindowService {
mod test {
use super::*;
use crate::{
cluster_info::ClusterInfo, contact_info::ContactInfo, repair_service::RepairSlotRange,
cluster_info::ClusterInfo,
contact_info::ContactInfo,
genesis_utils::create_genesis_config_with_leader,
packet::{Packet, Packets},
repair_service::RepairSlotRange,
};
use rand::thread_rng;
use solana_ledger::shred::DataShredHeader;
use solana_ledger::{
blockstore::{make_many_slot_entries, Blockstore},
entry::{create_ticks, Entry},
genesis_utils::create_genesis_config_with_leader,
get_tmp_ledger_path,
shred::Shredder,
};
use solana_perf::packet::Packet;
use solana_sdk::{
clock::Slot,
epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
@@ -542,7 +528,10 @@ mod test {
.insert_shreds(shreds, None, false)
.expect("Expect successful processing of shred");
assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), original_entries);
assert_eq!(
blockstore.get_slot_entries(0, 0, None).unwrap(),
original_entries
);
drop(blockstore);
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
@@ -633,7 +622,6 @@ mod test {
let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
ContactInfo::new_localhost(&Pubkey::default(), 0),
)));
let cluster_slots = Arc::new(ClusterSlots::default());
let repair_sock = Arc::new(UdpSocket::bind(socketaddr_any!()).unwrap());
let window = WindowService::new(
blockstore,
@@ -645,7 +633,6 @@ mod test {
RepairStrategy::RepairRange(RepairSlotRange { start: 0, end: 0 }),
&Arc::new(LeaderScheduleCache::default()),
|_, _, _, _| true,
cluster_slots,
);
window
}

View File

@@ -7,10 +7,12 @@ mod tests {
use itertools::Itertools;
use solana_core::cluster_info::ClusterInfo;
use solana_core::contact_info::ContactInfo;
use solana_core::snapshot_packager_service::SnapshotPackagerService;
use solana_core::{
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_packager_service::SnapshotPackagerService,
};
use solana_ledger::{
bank_forks::{BankForks, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils,
};
use solana_runtime::{

View File

@@ -5,7 +5,7 @@ use rayon::iter::*;
use solana_core::cluster_info::{ClusterInfo, Node};
use solana_core::gossip_service::GossipService;
use solana_perf::packet::Packet;
use solana_core::packet::Packet;
use solana_sdk::signature::{Keypair, Signer};
use solana_sdk::timing::timestamp;
use std::net::UdpSocket;

View File

@@ -13,14 +13,15 @@ use solana_client::{
};
use solana_core::{rpc_pubsub::gen_client::Client as PubsubClient, validator::TestValidator};
use solana_sdk::{
commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, signature::Signer,
system_transaction, transaction::Transaction,
commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, system_transaction,
transaction::Transaction,
};
use std::{
collections::HashSet,
fs::remove_dir_all,
net::UdpSocket,
sync::mpsc::channel,
sync::{Arc, Mutex},
thread::sleep,
time::{Duration, Instant},
};
@@ -206,11 +207,9 @@ fn test_rpc_subscriptions() {
..
} = TestValidator::run();
let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
transactions_socket.connect(leader_data.tpu).unwrap();
// Create transaction signatures to subscribe to
let transactions: Vec<Transaction> = (0..1000)
let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
let transactions: Vec<Transaction> = (0..100)
.map(|_| system_transaction::transfer(&alice, &Pubkey::new_rand(), 1, genesis_hash))
.collect();
let mut signature_set: HashSet<String> = transactions
@@ -218,15 +217,15 @@ fn test_rpc_subscriptions() {
.map(|tx| tx.signatures[0].to_string())
.collect();
// Track when subscriptions are ready
let (ready_sender, ready_receiver) = channel::<()>();
// Track when status notifications are received
let (status_sender, status_receiver) = channel::<(String, Response<RpcSignatureResult>)>();
// Create the pub sub runtime
let mut rt = Runtime::new().unwrap();
let rpc_pubsub_url = format!("ws://{}/", leader_data.rpc_pubsub);
let (status_sender, status_receiver) = channel::<(String, Response<RpcSignatureResult>)>();
let status_sender = Arc::new(Mutex::new(status_sender));
let (sent_sender, sent_receiver) = channel::<()>();
let sent_sender = Arc::new(Mutex::new(sent_sender));
// Subscribe to all signatures
rt.spawn({
let connect = ws::try_connect::<PubsubClient>(&rpc_pubsub_url).unwrap();
@@ -235,12 +234,18 @@ fn test_rpc_subscriptions() {
.and_then(move |client| {
for sig in signature_set {
let status_sender = status_sender.clone();
let sent_sender = sent_sender.clone();
tokio::spawn(
client
.signature_subscribe(sig.clone(), None)
.and_then(move |sig_stream| {
sent_sender.lock().unwrap().send(()).unwrap();
sig_stream.for_each(move |result| {
status_sender.send((sig.clone(), result)).unwrap();
status_sender
.lock()
.unwrap()
.send((sig.clone(), result))
.unwrap();
future::ok(())
})
})
@@ -249,50 +254,37 @@ fn test_rpc_subscriptions() {
}),
);
}
tokio::spawn(
client
.slot_subscribe()
.and_then(move |slot_stream| {
slot_stream.for_each(move |_| {
ready_sender.send(()).unwrap();
future::ok(())
})
})
.map_err(|err| {
eprintln!("slot sub err: {:#?}", err);
}),
);
future::ok(())
})
.map_err(|_| ())
});
// Wait for signature subscriptions
ready_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
let deadline = Instant::now() + Duration::from_secs(2);
(0..transactions.len()).for_each(|_| {
sent_receiver
.recv_timeout(deadline.saturating_duration_since(Instant::now()))
.unwrap();
});
let rpc_client = RpcClient::new_socket(leader_data.rpc);
let mut mint_balance = rpc_client
.get_balance_with_commitment(&alice.pubkey(), CommitmentConfig::recent())
.unwrap()
.value;
assert!(mint_balance >= transactions.len() as u64);
let mut transaction_count = rpc_client
.get_transaction_count_with_commitment(CommitmentConfig::recent())
.unwrap();
// Send all transactions to tpu socket for processing
transactions.iter().for_each(|tx| {
transactions_socket
.send(&bincode::serialize(&tx).unwrap())
.send_to(&bincode::serialize(&tx).unwrap(), leader_data.tpu)
.unwrap();
});
// Track mint balance to know when transactions have completed
let now = Instant::now();
let expected_mint_balance = mint_balance - transactions.len() as u64;
while mint_balance != expected_mint_balance && now.elapsed() < Duration::from_secs(5) {
mint_balance = rpc_client
.get_balance_with_commitment(&alice.pubkey(), CommitmentConfig::recent())
.unwrap()
.value;
sleep(Duration::from_millis(100));
let expected_transaction_count = transaction_count + transactions.len() as u64;
while transaction_count < expected_transaction_count && now.elapsed() < Duration::from_secs(5) {
transaction_count = rpc_client
.get_transaction_count_with_commitment(CommitmentConfig::recent())
.unwrap();
sleep(Duration::from_millis(200));
}
// Wait for all signature subscriptions
@@ -305,12 +297,12 @@ fn test_rpc_subscriptions() {
assert!(signature_set.remove(&sig));
}
Err(_err) => {
assert!(
false,
eprintln!(
"recv_timeout, {}/{} signatures remaining",
signature_set.len(),
transactions.len()
);
assert!(false)
}
}
}

View File

@@ -5,13 +5,12 @@ mod tests {
use log::*;
use solana_core::{
commitment::BlockCommitmentCache,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
storage_stage::{test_cluster_info, StorageStage, StorageState, SLOTS_PER_TURN_TEST},
};
use solana_ledger::{
bank_forks::BankForks,
blockstore::Blockstore,
blockstore_processor, create_new_tmp_ledger, entry,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
bank_forks::BankForks, blockstore::Blockstore, blockstore_processor, create_new_tmp_ledger,
entry,
};
use solana_runtime::bank::Bank;
use solana_sdk::{

View File

@@ -1,6 +1,6 @@
[package]
name = "solana-crate-features"
version = "1.1.3"
version = "1.0.16"
description = "Solana Crate Features"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
@@ -19,7 +19,7 @@ lazy_static = { version = "1.4.0", features = ["spin", "spin_no_std"] }
libc = { version = "0.2.62", features = ["extra_traits"] }
rand_chacha = { version = "0.1.1" }
regex-syntax = { version = "0.6.12" }
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls", "json"] }
reqwest = { version = "0.9.20", default-features = false, features = ["rustls-tls"] }
serde = { version = "1.0.100", features = ["rc"] }
ed25519-dalek = { version = "=1.0.0-pre.1", features = ["serde"] }
syn_0_15 = { package = "syn", version = "0.15.42", features = ["extra-traits", "fold", "full"] }

Some files were not shown because too many files have changed in this diff Show More