Improve BPF SDK dependency caching (#10434) (#10513)

(cherry picked from commit 97f9b63507)

Co-authored-by: Jack May <jack@solana.com>

sdk/bpf/scripts/install.sh

@@ -9,16 +9,24 @@ else
 fi
 
 download() {
-  declare url=$1
+  set -e
+  declare url="$1/$2/$3"
   declare version=$2
   declare filename=$3
-  declare progress=$4
+  declare dirname=$4
+  declare progress=$5
   declare cache_directory=~/.cache/"$version"
-  declare cache_filename=$cache_directory/${filename//:\//_}
+  declare cache_dirname=$cache_directory/${dirname//:\//_}
 
-  if [[ -r $cache_filename ]]; then
+  link() {
-    ln -s "$cache_filename" "$filename"
+    set -e
-    return
+    ln -sf "$cache_dirname" "$dirname"
+    ln -sf "$cache_directory/$dirname-$version.md" "$dirname-$version.md"
+  }
+
+  if [[ -r $cache_dirname ]]; then
+    link
+    return 0
   fi
 
   declare args=(
@@ -27,9 +35,49 @@ download() {
     "--retry-connrefused"
     "--read-timeout=30"
   )
+  set -x
+  mkdir -p "$cache_dirname"
+  pushd "$cache_dirname"
   if wget "${args[@]}"; then
+    tar --strip-components 1 -jxf "$filename"
+    rm -rf "$filename"
+    echo "$url" >"../$dirname-$version.md"
+    popd
+    link
+    return 0
+  fi
+  popd
+  rm -rf "$cache_dirname"
+  return 1
+}
+
+clone() {
+  set -e
+  declare url=$1
+  declare version=$2
+  declare dirname=$3
+  declare cache_directory=~/.cache/"$version"
+  declare cache_dirname=$cache_directory/${dirname//:\//_}
+
+  link() {
+    set -e
+    ln -sf "$cache_dirname" "$dirname"
+    ln -sf "$cache_directory/$dirname-$version.md" "$dirname-$version.md"
+  }
+
+  if [[ -r $cache_dirname ]]; then
+    link
+    return 0
+  fi
+
+  set -x
   mkdir -p "$cache_directory"
-    cp "$filename" "$cache_filename"
+  pushd "$cache_directory"
+  cmd="git clone --recursive --depth 1 --single-branch --branch $version $url"
+  if $cmd; then
+    echo "$cmd" >"$dirname-$version.md"
+    popd
+    link
     return 0
   fi
   return 1
@@ -44,7 +92,7 @@ download() {
   else
     cargo install xargo
   fi
-  xargo --version > xargo.md 2>&1
+  xargo --version >xargo.md 2>&1
 )
 # shellcheck disable=SC2181
 if [[ $? -ne 0 ]]; then
@@ -53,107 +101,85 @@ fi
 
 # Install Criterion
 version=v2.3.2
-if [[ ! -r criterion-$machine-$version.md ]]; then
+if [[ ! -e criterion-$version.md ]]; then
   (
-    filename=criterion-$version-$machine-x86_64.tar.bz2
+    set -e
-
-    set -ex
     rm -rf criterion*
-    mkdir criterion
+    download "https://github.com/Snaipe/Criterion/releases/download" \
-    cd criterion
+      $version \
-
+      "criterion-$version-$machine-x86_64.tar.bz2" \
-    base=https://github.com/Snaipe/Criterion/releases
+      criterion \
-    download $base/download/$version/$filename $version $filename mega
+      mega
-    tar --strip-components 1 -jxf $filename
-    rm -rf $filename
-
-    echo "$base/tag/$version" > ../criterion-$machine-$version.md
   )
-  # shellcheck disable=SC2181
+  exitcode=$?
-  if [[ $? -ne 0 ]]; then
+  if [[ $exitcode -ne 0 ]]; then
-    rm -rf criterion
+    rm -rf criterion-$version.md
     exit 1
   fi
 fi
 
 # Install LLVM
 version=v0.0.15
-if [[ ! -f llvm-native-$machine-$version.md ]]; then
+if [[ ! -e llvm-native-$version.md ]]; then
   (
-    filename=solana-llvm-$machine.tar.bz2
+    set -e
-
-    set -ex
     rm -rf llvm-native*
     rm -rf xargo
-    mkdir -p llvm-native
+    download "https://github.com/solana-labs/llvm-builder/releases/download" \
-    cd llvm-native
+      $version \
-
+      "solana-llvm-$machine.tar.bz2" \
-    base=https://github.com/solana-labs/llvm-builder/releases
+      llvm-native \
-    download $base/download/$version/$filename $version $filename giga
+      giga
-    tar -jxf $filename
-    rm -rf $filename
-
-    echo "$base/tag/$version" > ../llvm-native-$machine-$version.md
   )
   exitcode=$?
   if [[ $exitcode -ne 0 ]]; then
-    rm -rf llvm-native
+    rm -rf llvm-native-$version.md
     exit 1
   fi
 fi
 
 # Install Rust-BPF
 version=v0.2.3
-if [[ ! -f rust-bpf-$machine-$version.md ]]; then
+if [[ ! -e rust-bpf-$version.md ]]; then
   (
-    filename=solana-rust-bpf-$machine.tar.bz2
+    set -e
-
-    set -ex
     rm -rf rust-bpf
     rm -rf rust-bpf-$machine-*
     rm -rf xargo
-    mkdir -p rust-bpf
+    download "https://github.com/solana-labs/rust-bpf-builder/releases/download" \
-    pushd rust-bpf
+      $version \
-
+      "solana-rust-bpf-$machine.tar.bz2" \
-    base=https://github.com/solana-labs/rust-bpf-builder/releases
+      rust-bpf \
-    download $base/download/$version/$filename $version $filename giga
+      giga
-    tar -jxf $filename
-    rm -rf $filename
-    popd
 
     set -ex
     ./rust-bpf/bin/rustc --print sysroot
-
     set +e
     rustup toolchain uninstall bpf
     set -e
     rustup toolchain link bpf rust-bpf
-
-    echo "$base/tag/$version" > rust-bpf-$machine-$version.md
   )
   exitcode=$?
   if [[ $exitcode -ne 0 ]]; then
-    rm -rf rust-bpf
+    rm -rf rust-bpf-$version.md
     exit 1
   fi
 fi
 
 # Install Rust-BPF Sysroot sources
 version=v0.12
-if [[ ! -f rust-bpf-sysroot-$version.md ]]; then
+if [[ ! -e rust-bpf-sysroot-$version.md ]]; then
-
   (
-    set -ex
+    set -e
     rm -rf rust-bpf-sysroot*
     rm -rf xargo
-    cmd="git clone --recursive --depth 1 --single-branch --branch $version https://github.com/solana-labs/rust-bpf-sysroot.git"
+    clone "https://github.com/solana-labs/rust-bpf-sysroot.git" \
-    $cmd
+      $version \
-
+      rust-bpf-sysroot
-    echo "$cmd" > rust-bpf-sysroot-$version.md
   )
   exitcode=$?
   if [[ $exitcode -ne 0 ]]; then
-    rm -rf rust-bpf-sysroot
+    rm -rf rust-bpf-sysroot-$version.md
     exit 1
   fi
 fi