Dedup bloom filter is too slow (#22607)

* Faster dedup

* use ahash

* fixup

* single threaded

* use duration type

* remove the count

* fixup
This commit is contained in:
anatoly yakovenko
2022-01-21 19:23:48 -08:00
committed by GitHub
parent 6d5bbca630
commit d6011ba14d
6 changed files with 181 additions and 85 deletions

View File

@ -69,6 +69,17 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom 0.2.4",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
@ -1158,9 +1169,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.1"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6"
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
dependencies = [
"cfg-if 1.0.0",
"libc",
@ -1218,7 +1229,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
dependencies = [
"ahash",
"ahash 0.4.7",
]
[[package]]
@ -2127,7 +2138,7 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom 0.2.1",
"getrandom 0.2.4",
]
[[package]]
@ -2203,7 +2214,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom 0.2.1",
"getrandom 0.2.4",
"redox_syscall 0.2.10",
]
@ -3325,6 +3336,7 @@ dependencies = [
name = "solana-perf"
version = "1.10.0"
dependencies = [
"ahash 0.7.6",
"bincode",
"bv",
"caps",