gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

build: add ci.go, use it everywhere

Browse Source 
The new build script, ci.go, replaces some of the older shell scripts.
ci.go can compile go-ethereum, run the tests, create release archives
and debian source packages.
This commit is contained in:
Felix Lange
2016-05-25 14:07:57 +02:00
parent a38be3eb48
commit 6c33ba14a4
19 changed files with 941 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
build/ci-notes.md Normal file
View File

@ -0,0 +1,26 @@
Debian Packaging
----------------
Tagged releases and develop branch commits are available as installable Debian packages
for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
Canonical:
- Trusty Tahr (14.04 LTS)
- Wily Werewolf (15.10)
- Xenial Xerus (16.04 LTS)
Packages of develop branch commits have suffix -unstable and cannot be installed alongside
the stable version. Switching between release streams requires user intervention.
The packages are built and served by launchpad.net. We generate a Debian source package
for each distribution and upload it. Their builder picks up the source package, builds it
and installs the new version into the PPA repository. Launchpad requires a valid signature
by a team member for source package uploads. The signing key is stored in an environment
variable which Travis CI makes available to certain builds.
We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
golang-1.6, which is co-installable alongside the regular golang package. PPA dependencies
can be edited at https://launchpad.net/%7Elp-fjl/+archive/ubuntu/geth-ci-testing/+edit-dependencies

465
build/ci.go Normal file
View File

@ -0,0 +1,465 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build none
/*
The ci command is called from Continuous Integration scripts.
Usage: go run ci.go <command> <command flags/arguments>
Available commands are:
install [ packages... ] -- builds packages and executables
test [ -coverage ] [ -vet ] [ packages... ] -- runs the tests
archive [ -type zip|tar ] -- archives build artefacts
importkeys -- imports signing keys from env
debsrc [ -sign key-id ] [ -upload dest ] -- creates a debian source package
For all commands, -n prevents execution of external programs (dry run mode).
*/
package main
import (
"bytes"
"encoding/base64"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"../internal/build"
)
var (
// Files that end up in the geth*.zip archive.
gethArchiveFiles = []string{
"COPYING",
executablePath("geth"),
}
// Files that end up in the geth-alltools*.zip archive.
allToolsArchiveFiles = []string{
"COPYING",
executablePath("abigen"),
executablePath("evm"),
executablePath("geth"),
executablePath("rlpdump"),
}
// A debian package is created for all executables listed here.
debExecutables = []debExecutable{
{
Name: "geth",
Description: "Ethereum CLI client.",
},
{
Name: "rlpdump",
Description: "Developer utility tool that prints RLP structures.",
},
{
Name: "evm",
Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
},
{
Name: "abigen",
Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
},
}
// Distros for which packages are created.
// Note: vivid is unsupported because there is no golang-1.6 package for it.
debDistros = []string{"trusty", "wily", "xenial", "yakkety"}
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
func executablePath(name string) string {
if runtime.GOOS == "windows" {
name += ".exe"
}
return filepath.Join(GOBIN, name)
}
func main() {
log.SetFlags(log.Lshortfile)
if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
log.Fatal("this script must be run from the root of the repository")
}
if len(os.Args) < 2 {
log.Fatal("need subcommand as first argument")
}
switch os.Args[1] {
case "install":
doInstall(os.Args[2:])
case "test":
doTest(os.Args[2:])
case "archive":
doArchive(os.Args[2:])
case "debsrc":
doDebianSource(os.Args[2:])
case "travis-debsrc":
doTravisDebianSource(os.Args[2:])
default:
log.Fatal("unknown command ", os.Args[1])
}
}
// Compiling
func doInstall(cmdline []string) {
commitHash := flag.String("gitcommit", "", "Git commit hash embedded into binary.")
flag.CommandLine.Parse(cmdline)
// Check Go version. People regularly open issues about compilation
// failure with outdated Go. This should save them the trouble.
if runtime.Version() < "go1.4" && !strings.HasPrefix(runtime.Version(), "devel") {
log.Println("You have Go version", runtime.Version())
log.Println("go-ethereum requires at least Go version 1.4 and cannot")
log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
os.Exit(1)
}
// Compile packages given as arguments, or everything if there are no arguments.
packages := []string{"./..."}
if flag.NArg() > 0 {
packages = flag.Args()
}
goinstall := goTool("install", makeBuildFlags(*commitHash)...)
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
}
func makeBuildFlags(commitHash string) (flags []string) {
// Since Go 1.5, the separator char for link time assignments
// is '=' and using ' ' prints a warning. However, Go < 1.5 does
// not support using '='.
sep := " "
if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") {
sep = "="
}
if os.Getenv("GO_OPENCL") != "" {
flags = append(flags, "-tags", "opencl")
}
// Set gitCommit constant via link-time assignment. If this is a git checkout, we can
// just get the current commit hash through git. Otherwise we fall back to the hash
// that was passed as -gitcommit.
//
// -gitcommit is required for Debian package builds. The source package doesn't
// contain .git but we still want to embed the commit hash into the packaged binary.
// The hash is rendered into the debian/rules build script when the source package is
// created.
if _, err := os.Stat(filepath.Join(".git", "HEAD")); !os.IsNotExist(err) {
if c := build.GitCommit(); c != "" {
commitHash = c
}
}
if commitHash != "" {
flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+commitHash)
}
return flags
}
func goTool(subcmd string, args ...string) *exec.Cmd {
gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
cmd := exec.Command(gocmd, subcmd)
cmd.Args = append(cmd.Args, args...)
cmd.Env = []string{
"GOPATH=" + build.GOPATH(),
"GOBIN=" + GOBIN,
}
for _, e := range os.Environ() {
if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
continue
}
cmd.Env = append(cmd.Env, e)
}
return cmd
}
// Running The Tests
//
// "tests" also includes static analysis tools such as vet.
func doTest(cmdline []string) {
var (
vet = flag.Bool("vet", false, "Whether to run go vet")
coverage = flag.Bool("coverage", false, "Whether to record code coverage")
)
flag.CommandLine.Parse(cmdline)
packages := []string{"./..."}
if len(flag.CommandLine.Args()) > 0 {
packages = flag.CommandLine.Args()
}
// Run analysis tools before the tests.
if *vet {
build.MustRun(goTool("vet", packages...))
}
// Run the actual tests.
gotest := goTool("test")
if *coverage {
gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
}
gotest.Args = append(gotest.Args, packages...)
build.MustRun(gotest)
}
// Release Packaging
func doArchive(cmdline []string) {
var (
atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
ext string
)
flag.CommandLine.Parse(cmdline)
switch *atype {
case "zip":
ext = ".zip"
case "tar":
ext = ".tar.gz"
default:
log.Fatal("unknown archive type: ", atype)
}
base := makeArchiveBasename()
if err := build.WriteArchive("geth-"+base, ext, gethArchiveFiles); err != nil {
log.Fatal(err)
}
if err := build.WriteArchive("geth-alltools-"+base, ext, allToolsArchiveFiles); err != nil {
log.Fatal(err)
}
}
func makeArchiveBasename() string {
// date := time.Now().UTC().Format("200601021504")
platform := runtime.GOOS + "-" + runtime.GOARCH
archive := platform + "-" + build.VERSION()
if commit := build.GitCommit(); commit != "" {
archive += "-" + commit[:8]
}
return archive
}
// Debian Packaging
// CLI entry point for Travis CI.
func doTravisDebianSource(cmdline []string) {
flag.CommandLine.Parse(cmdline)
// Package only whitelisted branches.
switch {
case os.Getenv("TRAVIS_REPO_SLUG") != "ethereum/go-ethereum":
log.Printf("skipping because this is a fork build")
return
case os.Getenv("TRAVIS_PULL_REQUEST") != "false":
log.Printf("skipping because this is a PR build")
return
case os.Getenv("TRAVIS_BRANCH") != "develop" && !strings.HasPrefix(os.Getenv("TRAVIS_TAG"), "v1."):
log.Printf("skipping because branch %q tag %q is not on the whitelist",
os.Getenv("TRAVIS_BRANCH"),
os.Getenv("TRAVIS_TAG"))
return
}
// Import the signing key.
if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
log.Fatal("invalid base64 PPA_SIGNING_KEY")
}
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
}
// Assign unstable status to non-tag builds.
unstable := "true"
if os.Getenv("TRAVIS_BRANCH") != "develop" && os.Getenv("TRAVIS_TAG") != "" {
unstable = "false"
}
doDebianSource([]string{
"-signer", "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>",
"-buildnum", os.Getenv("TRAVIS_BUILD_NUMBER"),
"-upload", "ppa:lp-fjl/geth-ci-testing",
"-unstable", unstable,
})
}
// CLI entry point for doing packaging locally.
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
buildnum = flag.String("buildnum", "", `Build number (included in version)`)
unstable = flag.Bool("unstable", false, `Use package name suffix "-unstable"`)
now = time.Now()
)
flag.CommandLine.Parse(cmdline)
// Create the debian worktree in /tmp.
tmpdir, err := ioutil.TempDir("", "eth-deb-build-")
if err != nil {
log.Fatal(err)
}
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, *buildnum, *unstable, now)
pkgdir := stageDebianSource(tmpdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
debuild.Dir = pkgdir
build.MustRun(debuild)
changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
changes = filepath.Join(tmpdir, changes)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
build.MustRunCommand("dput", *upload, changes)
}
}
}
type debExecutable struct {
Name, Description string
}
type debMetadata struct {
// go-ethereum version being built. Note that this
// is not the debian package version. The package version
// is constructed by VersionString.
Version string
Author string // "name <email>", also selects signing key
Buildnum string // build number
Distro, Commit, Time string
Executables []debExecutable
Unstable bool
}
func newDebMetadata(distro, author, buildnum string, unstable bool, t time.Time) debMetadata {
if author == "" {
// No signing key, use default author.
author = "Ethereum Builds <fjl@ethereum.org>"
}
return debMetadata{
Unstable: unstable,
Author: author,
Distro: distro,
Commit: build.GitCommit(),
Version: build.VERSION(),
Buildnum: buildnum,
Time: t.Format(time.RFC1123Z),
Executables: debExecutables,
}
}
// Name returns the name of the metapackage that depends
// on all executable packages.
func (meta debMetadata) Name() string {
if meta.Unstable {
return "ethereum-unstable"
}
return "ethereum"
}
// VersionString returns the debian version of the packages.
func (meta debMetadata) VersionString() string {
vsn := meta.Version
if meta.Buildnum != "" {
vsn += "+build" + meta.Buildnum
}
if meta.Distro != "" {
vsn += "+" + meta.Distro
}
return vsn
}
// ExeList returns the list of all executable packages.
func (meta debMetadata) ExeList() string {
names := make([]string, len(meta.Executables))
for i, e := range meta.Executables {
names[i] = meta.ExeName(e)
}
return strings.Join(names, ", ")
}
// ExeName returns the package name of an executable package.
func (meta debMetadata) ExeName(exe debExecutable) string {
if meta.Unstable {
return exe.Name + "-unstable"
}
return exe.Name
}
// ExeConflicts returns the content of the Conflicts field
// for executable packages.
func (meta debMetadata) ExeConflicts(exe debExecutable) string {
if meta.Unstable {
// Set up the conflicts list so that the *-unstable packages
// cannot be installed alongside the regular version.
//
// https://www.debian.org/doc/debian-policy/ch-relationships.html
// is very explicit about Conflicts: and says that Breaks: should
// be preferred and the conflicting files should be handled via
// alternates. We might do this eventually but using a conflict is
// easier now.
return "ethereum, " + exe.Name
}
return ""
}
func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
pkg := meta.Name() + "-" + meta.VersionString()
pkgdir = filepath.Join(tmpdir, pkg)
if err := os.Mkdir(pkgdir, 0755); err != nil {
log.Fatal(err)
}
// Copy the source code.
build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
// Put the debian build files in place.
debian := filepath.Join(pkgdir, "debian")
build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
for _, exe := range meta.Executables {
install := filepath.Join(debian, exe.Name+".install")
docs := filepath.Join(debian, exe.Name+".docs")
build.Render("build/deb.install", install, 0644, exe)
build.Render("build/deb.docs", docs, 0644, exe)
}
return pkgdir
}

5
build/deb.changelog Normal file
View File

@ -0,0 +1,5 @@
{{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
* git build of {{.Commit}}
-- {{.Author}} {{.Time}}

25
build/deb.control Normal file
View File

@ -0,0 +1,25 @@
Source: {{.Name}}
Section: science
Priority: extra
Maintainer: {{.Author}}
Build-Depends: debhelper (>= 8.0.0), golang-1.6
Standards-Version: 3.9.5
Homepage: https://ethereum.org
Vcs-Git: git://github.com/ethereum/go-ethereum.git
Vcs-Browser: https://github.com/ethereum/go-ethereum
Package: {{.Name}}
Architecture: any
Depends: ${misc:Depends}, {{.ExeList}}
Description: Meta-package to install geth and other tools
Meta-package to install geth and other tools
{{range .Executables}}
Package: {{$.ExeName .}}
Conflicts: {{$.ExeConflicts .}}
Architecture: any
Depends: ${shlibs:Depends}, ${misc:Depends}
Built-Using: ${misc:Built-Using}
Description: {{.Description}}
{{.Description}}
{{end}}

14
build/deb.copyright Normal file
View File

@ -0,0 +1,14 @@
Copyright 2016 The go-ethereum Authors
go-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
go-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

1
build/deb.docs Normal file
View File

@ -0,0 +1 @@
AUTHORS

1
build/deb.install Normal file
View File

@ -0,0 +1 @@
build/bin/{{.Name}} usr/bin

13
build/deb.rules Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/make -f
# -*- makefile -*-
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1
override_dh_auto_build:
build/env.sh /usr/lib/go-1.6/bin/go run build/ci.go install -gitcommit {{.Commit}}
override_dh_auto_test:
%:
dh $@

5
build/env.sh
View File

@ -20,9 +20,8 @@ fi
# Set up the environment to use the workspace.
# Also add Godeps workspace so we build using canned dependencies.
GOPATH="$ethdir/go-ethereum/Godeps/_workspace:$workspace"
GOBIN="$PWD/build/bin"
export GOPATH GOBIN
GOPATH="$workspace"
export GOPATH
# Run the command inside the workspace.
cd "$ethdir/go-ethereum"

15
build/test-global-coverage.sh
View File

@ -1,15 +0,0 @@
#!/usr/bin/env bash
set -e
echo "" > coverage.txt
for d in $(find ./* -maxdepth 10 -type d -not -path "./build" -not -path "./Godeps/*" ); do
if ls $d/*.go &> /dev/null; then
go test -coverprofile=profile.out -covermode=atomic $d
if [ -f profile.out ]; then
cat profile.out >> coverage.txt
echo '<<<<<< EOF' >> coverage.txt
rm profile.out
fi
fi
done

26
build/win-ci-compile.bat
View File

@ -1,26 +0,0 @@
@echo off
if not exist .\build\win-ci-compile.bat (
echo This script must be run from the root of the repository.
exit /b
)
if not defined GOPATH (
echo GOPATH is not set.
exit /b
)
set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
set GOBIN=%cd%\build\bin
rem set gitCommit when running from a Git checkout.
set goLinkFlags=""
if exist ".git\HEAD" (
where /q git
if not errorlevel 1 (
for /f %%h in ('git rev-parse HEAD') do (
set goLinkFlags="-X main.gitCommit=%%h"
)
)
)
@echo on
go install -v -ldflags %goLinkFlags% ./...

15
build/win-ci-test.bat
View File

@ -1,15 +0,0 @@
@echo off
if not exist .\build\win-ci-test.bat (
echo This script must be run from the root of the repository.
exit /b
)
if not defined GOPATH (
echo GOPATH is not set.
exit /b
)
set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
set GOBIN=%cd%\build\bin
@echo on
go test ./...