Dockerized LLVM (#1914)

Optionally build with dockererized custom llvm

sdk/docker-llvm/Dockerfile

@@ -0,0 +1,68 @@
+# This docker file is based on the llvm docker file example located here:
+# https://github.com/llvm-mirror/llvm/blob/master/utils/docker/debian8/Dockerfile
+
+FROM launcher.gcr.io/google/debian8:latest as builder
+LABEL maintainer "Solana Maintainers"
+
+# Install build dependencies of llvm.
+# First, Update the apt's source list and include the sources of the packages.
+RUN grep deb /etc/apt/sources.list | \
+    sed 's/^deb/deb-src /g' >> /etc/apt/sources.list
+
+# Install compiler, python and subversion.
+RUN apt-get update && \
+    apt-get install -y \
+                    --no-install-recommends \
+                    ca-certificates gnupg \
+                    build-essential \
+                    python \
+                    wget \
+                    unzip \
+                    git \
+                    ssh && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install a newer ninja release. It seems the older version in the debian repos
+# randomly crashes when compiling llvm.
+RUN wget "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip" && \
+    echo "d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07 ninja-linux.zip" \
+        | sha256sum -c  && \
+    unzip ninja-linux.zip -d /usr/local/bin && \
+    rm ninja-linux.zip
+
+# Import public key required for verifying signature of cmake download.
+RUN gpg --no-tty --keyserver hkp://pgp.mit.edu --recv 0x2D2CEF1034921684
+
+# Download, verify and install cmake version that can compile clang into /usr/local.
+# (Version in debian8 repos is too old)
+RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-SHA-256.txt.asc" && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-SHA-256.txt" && \
+    gpg --verify cmake-3.7.2-SHA-256.txt.asc cmake-3.7.2-SHA-256.txt && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.tar.gz" && \
+    ( grep "cmake-3.7.2-Linux-x86_64.tar.gz" cmake-3.7.2-SHA-256.txt | \
+      sha256sum -c - ) && \
+    tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \
+    cd / && \
+    rm -rf /tmp/cmake-install
+
+# ADD checksums /tmp/checksums
+# ADD scripts /tmp/scripts
+
+# Checkout the source code 
+RUN git clone https://github.com/solana-labs/llvm.git && \
+    git clone https://github.com/solana-labs/clang.git llvm/tools/clang && \
+    git clone https://github.com/solana-labs/clang-tools-extra.git llvm/tools/clang/tools/extra  && \
+    git clone https://github.com/solana-labs/lld.git llvm/tools/lld  && \
+    git clone https://github.com/solana-labs/compiler-rt.git llvm/projects/compiler-rt
+
+RUN mkdir /llvm/build && \
+    cd /llvm/build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_INSTALL_PREFIX=$HOME/local -G "Ninja" .. && \
+    ninja -j6 && \
+    ninja install
+
+# Produce stage 2 docker with just the peices needed
+FROM launcher.gcr.io/google/debian8:latest
+LABEL maintainer "Solana Maintainers"
+COPY --from=builder root/local/bin /usr/local/bin

sdk/docker-llvm/README.md

@@ -0,0 +1,16 @@
+## Solana Customized LLVM
+
+This Docker contains LLVM binaries that incorporate customizations and fixes required
+by Solana but not yet upstreamed into the LLVM mainline.
+
+https://hub.docker.com/r/solanalabs/llvm/
+
+### Usage:
+
+This Docker is optionally used by the SDK BPF build system.
+
+For more information:
+
+```bash
+$ make help
+```

sdk/docker-llvm/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/llvm .
+docker push solanalabs/llvm

sdk/docker-solana/.gitignore

@@ -0,0 +1,2 @@
+cargo-install/
+usr/

sdk/docker-solana/Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:stretch
+
+# JSON RPC port
+EXPOSE 8899/tcp
+
+# Install libssl
+RUN apt update && \
+    apt-get install -y libssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY usr/bin /usr/bin/
+ENTRYPOINT [ "/usr/bin/solana-entrypoint.sh" ]
+CMD [""]

sdk/docker-solana/README.md

@@ -0,0 +1,17 @@
+## Minimal Solana Docker image
+This image is automatically updated by CI
+
+https://hub.docker.com/r/solanalabs/solana/
+
+### Usage:
+Run the latest beta image:
+```bash
+$ docker run --rm -p 8899:8899 solanalabs/solana:beta
+```
+
+Run the latest edge image:
+```bash
+$ docker run --rm -p 8899:8899 solanalabs/solana:edge
+```
+
+Port *8899* is the JSON RPC port, which is used by clients to communicate with the network.

sdk/docker-solana/build.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -ex
+
+cd "$(dirname "$0")"
+eval "$(../channel-info.sh)"
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  CHANNEL=beta
+fi
+
+if [[ -z $CHANNEL ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+rm -rf usr/
+../../ci/docker-run.sh solanalabs/rust:1.30.1 bash -c "
+  set -ex
+  cargo install --path drone --root sdk/docker-solana/usr
+  cargo install --path . --root sdk/docker-solana/usr
+"
+cp -f entrypoint.sh usr/bin/solana-entrypoint.sh
+../../scripts/install-native-programs.sh usr/bin/
+
+docker build -t solanalabs/solana:$CHANNEL .
+
+maybeEcho=
+if [[ -z $CI ]]; then
+  echo "Not CI, skipping |docker push|"
+  maybeEcho="echo"
+else
+  (
+    set +x
+    if [[ -n $DOCKER_PASSWORD && -n $DOCKER_USERNAME ]]; then
+      echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+    fi
+  )
+fi
+$maybeEcho docker push solanalabs/solana:$CHANNEL

sdk/docker-solana/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -ex
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+
+solana-keygen -o /config/leader-keypair.json
+solana-keygen -o /config/drone-keypair.json
+
+solana-fullnode-config --keypair=/config/leader-keypair.json -l > /config/leader-config.json
+solana-genesis --num_tokens 1000000000 --mint /config/drone-keypair.json --bootstrap_leader /config/leader-config.json --ledger /ledger
+
+solana-drone --keypair /config/drone-keypair.json &
+drone=$!
+solana-fullnode --identity /config/leader-config.json --ledger /ledger/ --rpc 8899 &
+fullnode=$!
+
+abort() {
+  kill "$drone" "$fullnode"
+}
+
+trap abort SIGINT SIGTERM
+wait "$fullnode"
+kill "$drone" "$fullnode"