Files
go-ethereum/build/ci.go
2019-06-03 15:33:09 +02:00

738 lines
22 KiB
Go

// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build none
/*
The ci command is called from Continuous Integration scripts.
Usage: go run build/ci.go <command> <command flags/arguments>
Available commands are:
install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
test [ -coverage ] [ packages... ] -- runs the tests
lint -- runs certain pre-selected linters
archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
importkeys -- imports signing keys from env
debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package
xgo [ -alltools ] [ options ] -- cross builds according to options
purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore
For all commands, -n prevents execution of external programs (dry run mode).
*/
package main
import (
"bytes"
"encoding/base64"
"flag"
"fmt"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
"github.com/ethersphere/swarm/internal/build"
sv "github.com/ethersphere/swarm/version"
)
var (
// Files that end up in the swarm*.zip archive.
swarmArchiveFiles = []string{
"COPYING",
executablePath("swarm"),
}
// A debian package is created for all executables listed here.
debSwarmExecutables = []debExecutable{
{
BinaryName: "swarm",
PackageName: "ethereum-swarm",
Description: "Ethereum Swarm daemon and tools",
},
}
debSwarm = debPackage{
Name: "ethereum-swarm",
Version: sv.Version,
Executables: debSwarmExecutables,
}
// Debian meta packages to build and push to Ubuntu PPA
debPackages = []debPackage{
debSwarm,
}
// Packages to be cross-compiled by the xgo command
allCrossCompiledArchiveFiles = swarmArchiveFiles
// Distros for which packages are created.
// Note: vivid is unsupported because there is no golang-1.6 package for it.
// Note: wily is unsupported because it was officially deprecated on lanchpad.
// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
// Note: zesty is unsupported because it was officially deprecated on lanchpad.
// Note: artful is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
func executablePath(name string) string {
if runtime.GOOS == "windows" {
name += ".exe"
}
return filepath.Join(GOBIN, name)
}
func main() {
log.SetFlags(log.Lshortfile)
if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
log.Fatal("this script must be run from the root of the repository")
}
if len(os.Args) < 2 {
log.Fatal("need subcommand as first argument")
}
switch os.Args[1] {
case "install":
doInstall(os.Args[2:])
case "test":
doTest(os.Args[2:])
case "lint":
doLint(os.Args[2:])
case "archive":
doArchive(os.Args[2:])
case "debsrc":
doDebianSource(os.Args[2:])
case "xgo":
doXgo(os.Args[2:])
case "purge":
doPurge(os.Args[2:])
default:
log.Fatal("unknown command ", os.Args[1])
}
}
// Compiling
func doInstall(cmdline []string) {
var (
arch = flag.String("arch", "", "Architecture to cross build for")
cc = flag.String("cc", "", "C compiler to cross build with")
)
flag.CommandLine.Parse(cmdline)
env := build.Env()
// Check Go version. People regularly open issues about compilation
// failure with outdated Go. This should save them the trouble.
if !strings.Contains(runtime.Version(), "devel") {
// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
var minor int
fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
if minor < 9 {
log.Println("You have Go version", runtime.Version())
log.Println("go-ethereum requires at least Go version 1.9 and cannot")
log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
os.Exit(1)
}
}
// Compile packages given as arguments, or everything if there are no arguments.
packages := []string{"./..."}
if flag.NArg() > 0 {
packages = flag.Args()
}
packages = build.ExpandPackagesNoVendor(packages)
if *arch == "" || *arch == runtime.GOARCH {
goinstall := goTool("install", buildFlags(env)...)
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
return
}
// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
if *arch == "arm" {
os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
for _, path := range filepath.SplitList(build.GOPATH()) {
os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
}
}
// Seems we are cross compiling, work around forbidden GOBIN
goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
if cmds, err := ioutil.ReadDir("cmd"); err == nil {
for _, cmd := range cmds {
pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
if err != nil {
log.Fatal(err)
}
for name := range pkgs {
if name == "main" {
gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
gobuild.Args = append(gobuild.Args, "-v")
gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
build.MustRun(gobuild)
break
}
}
}
}
}
func buildFlags(env build.Environment) (flags []string) {
var ld []string
if env.Commit != "" {
ld = append(ld, "-X", "main.gitCommit="+env.Commit)
}
if runtime.GOOS == "darwin" {
ld = append(ld, "-s")
}
if len(ld) > 0 {
flags = append(flags, "-ldflags", strings.Join(ld, " "))
}
return flags
}
func goTool(subcmd string, args ...string) *exec.Cmd {
return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
}
func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
cmd := build.GoTool(subcmd, args...)
cmd.Env = []string{"GOPATH=" + build.GOPATH()}
if arch == "" || arch == runtime.GOARCH {
cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
} else {
cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
cmd.Env = append(cmd.Env, "GOARCH="+arch)
}
if cc != "" {
cmd.Env = append(cmd.Env, "CC="+cc)
}
for _, e := range os.Environ() {
if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
continue
}
cmd.Env = append(cmd.Env, e)
}
return cmd
}
// Running The Tests
//
// "tests" also includes static analysis tools such as vet.
func doTest(cmdline []string) {
coverage := flag.Bool("coverage", false, "Whether to record code coverage")
flag.CommandLine.Parse(cmdline)
env := build.Env()
packages := []string{"./..."}
if len(flag.CommandLine.Args()) > 0 {
packages = flag.CommandLine.Args()
}
packages = build.ExpandPackagesNoVendor(packages)
// Run the actual tests.
// Test a single package at a time. CI builders are slow
// and some tests run into timeouts under load.
gotest := goTool("test", buildFlags(env)...)
gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
if *coverage {
gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
}
gotest.Args = append(gotest.Args, packages...)
build.MustRun(gotest)
}
// runs gometalinter on requested packages
func doLint(cmdline []string) {
flag.CommandLine.Parse(cmdline)
packages := []string{"./..."}
if len(flag.CommandLine.Args()) > 0 {
packages = flag.CommandLine.Args()
}
// Get metalinter and install all supported linters
build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
// Run fast linters batched together
configs := []string{
"--vendor",
"--tests",
"--deadline=2m",
"--disable-all",
"--enable=goimports",
"--enable=varcheck",
"--enable=vet",
"--enable=gofmt",
"--enable=misspell",
"--enable=goconst",
"--min-occurrences=6", // for goconst
}
build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
// Run slow linters one by one
for _, linter := range []string{"unconvert", "gosimple"} {
configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
}
}
// Release Packaging
func doArchive(cmdline []string) {
var (
arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
upload = flag.String("upload", "", `Destination to upload the archives (usually "ethswarm/builds")`)
ext string
)
flag.CommandLine.Parse(cmdline)
switch *atype {
case "zip":
ext = ".zip"
case "tar":
ext = ".tar.gz"
default:
log.Fatal("unknown archive type: ", atype)
}
var (
env = build.Env()
baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit))
swarm = "swarm-" + baseswarm + ext
)
maybeSkipArchive(env)
if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil {
log.Fatal(err)
}
for _, archive := range []string{swarm} {
if err := archiveUpload(archive, *upload, *signer); err != nil {
log.Fatal(err)
}
}
}
func archiveBasename(arch string, archiveVersion string) string {
platform := runtime.GOOS + "-" + arch
if arch == "arm" {
platform += os.Getenv("GOARM")
}
return platform + "-" + archiveVersion
}
func archiveUpload(archive string, blobstore string, signer string) error {
// If signing was requested, generate the signature files
if signer != "" {
key := getenvBase64(signer)
if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
return err
}
}
// If uploading to Azure was requested, push the archive possibly with its signature
if blobstore != "" {
auth := build.AzureBlobstoreConfig{
Account: strings.Split(blobstore, "/")[0],
Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"),
Container: strings.SplitN(blobstore, "/", 2)[1],
}
if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil {
return err
}
if signer != "" {
if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil {
return err
}
}
}
return nil
}
// skips archiving for some build configurations.
func maybeSkipArchive(env build.Environment) {
if env.IsPullRequest {
log.Printf("skipping because this is a PR build")
os.Exit(0)
}
if env.IsCronJob {
log.Printf("skipping because this is a cron job")
os.Exit(0)
}
validArchiveTag, _ := regexp.MatchString("v([0-9]+).([0-9]+).([0-9]+)", "v10.2.3")
if env.Branch != "master" && !validArchiveTag {
log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
os.Exit(0)
}
}
// Debian Packaging
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
flag.CommandLine.Parse(cmdline)
*workdir = makeWorkdir(*workdir)
env := build.Env()
maybeSkipArchive(env)
// Import the signing key.
if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
}
// Create Debian packages and upload them
for _, pkg := range debPackages {
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
debuild.Dir = pkgdir
build.MustRun(debuild)
var (
basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
source = filepath.Join(*workdir, basename+".tar.xz")
dsc = filepath.Join(*workdir, basename+".dsc")
changes = filepath.Join(*workdir, basename+"_source.changes")
buildinfo = filepath.Join(*workdir, basename+"_source.buildinfo")
)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes, buildinfo})
}
}
}
}
func ppaUpload(workdir, ppa, sshUser string, files []string) {
p := strings.Split(ppa, "/")
if len(p) != 2 {
log.Fatal("-upload PPA name must contain single /")
}
if sshUser == "" {
sshUser = p[0]
}
incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
// Create the SSH identity file if it doesn't exist.
var idfile string
if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
idfile = filepath.Join(workdir, "sshkey")
if _, err := os.Stat(idfile); os.IsNotExist(err) {
ioutil.WriteFile(idfile, sshkey, 0600)
}
}
// Upload
dest := sshUser + "@ppa.launchpad.net"
if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
log.Fatal(err)
}
}
func getenvBase64(variable string) []byte {
dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
if err != nil {
log.Fatal("invalid base64 " + variable)
}
return []byte(dec)
}
func makeWorkdir(wdflag string) string {
var err error
if wdflag != "" {
err = os.MkdirAll(wdflag, 0744)
} else {
wdflag, err = ioutil.TempDir("", "geth-build-")
}
if err != nil {
log.Fatal(err)
}
return wdflag
}
func isUnstableBuild(env build.Environment) bool {
if env.Tag != "" {
return false
}
return true
}
type debPackage struct {
Name string // the name of the Debian package to produce, e.g. "ethereum", or "ethereum-swarm"
Version string // the clean version of the debPackage, e.g. 1.8.12 or 0.3.0, without any metadata
Executables []debExecutable // executables to be included in the package
}
type debMetadata struct {
Env build.Environment
PackageName string
// go-ethereum version being built. Note that this
// is not the debian package version. The package version
// is constructed by VersionString.
Version string
Author string // "name <email>", also selects signing key
Distro, Time string
Executables []debExecutable
}
type debExecutable struct {
PackageName string
BinaryName string
Description string
}
// Package returns the name of the package if present, or
// fallbacks to BinaryName
func (d debExecutable) Package() string {
if d.PackageName != "" {
return d.PackageName
}
return d.BinaryName
}
func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
if author == "" {
// No signing key, use default author.
author = "Ethereum Builds <fjl@ethereum.org>"
}
return debMetadata{
PackageName: name,
Env: env,
Author: author,
Distro: distro,
Version: version,
Time: t.Format(time.RFC1123Z),
Executables: exes,
}
}
// Name returns the name of the metapackage that depends
// on all executable packages.
func (meta debMetadata) Name() string {
if isUnstableBuild(meta.Env) {
return meta.PackageName + "-unstable"
}
return meta.PackageName
}
// VersionString returns the debian version of the packages.
func (meta debMetadata) VersionString() string {
vsn := meta.Version
if meta.Env.Buildnum != "" {
vsn += "+build" + meta.Env.Buildnum
}
if meta.Distro != "" {
vsn += "+" + meta.Distro
}
return vsn
}
// ExeList returns the list of all executable packages.
func (meta debMetadata) ExeList() string {
names := make([]string, len(meta.Executables))
for i, e := range meta.Executables {
names[i] = meta.ExeName(e)
}
return strings.Join(names, ", ")
}
// ExeName returns the package name of an executable package.
func (meta debMetadata) ExeName(exe debExecutable) string {
if isUnstableBuild(meta.Env) {
return exe.Package() + "-unstable"
}
return exe.Package()
}
// ExeConflicts returns the content of the Conflicts field
// for executable packages.
func (meta debMetadata) ExeConflicts(exe debExecutable) string {
if isUnstableBuild(meta.Env) {
// Set up the conflicts list so that the *-unstable packages
// cannot be installed alongside the regular version.
//
// https://www.debian.org/doc/debian-policy/ch-relationships.html
// is very explicit about Conflicts: and says that Breaks: should
// be preferred and the conflicting files should be handled via
// alternates. We might do this eventually but using a conflict is
// easier now.
return "ethereum, " + exe.Package()
}
return ""
}
func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
pkg := meta.Name() + "-" + meta.VersionString()
pkgdir = filepath.Join(tmpdir, pkg)
if err := os.Mkdir(pkgdir, 0755); err != nil {
log.Fatal(err)
}
// Copy the source code.
build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
// Put the debian build files in place.
debian := filepath.Join(pkgdir, "debian")
build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta)
build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
for _, exe := range meta.Executables {
install := filepath.Join(debian, meta.ExeName(exe)+".install")
docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
}
return pkgdir
}
// Cross compilation
func doXgo(cmdline []string) {
var (
alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
)
flag.CommandLine.Parse(cmdline)
env := build.Env()
// Make sure xgo is available for cross compilation
gogetxgo := goTool("get", "github.com/karalabe/xgo")
build.MustRun(gogetxgo)
// If all tools building is requested, build everything the builder wants
args := append(buildFlags(env), flag.Args()...)
if *alltools {
args = append(args, []string{"--dest", GOBIN}...)
for _, res := range allCrossCompiledArchiveFiles {
if strings.HasPrefix(res, GOBIN) {
// Binary tool found, cross build it explicitly
args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
xgo := xgoTool(args)
build.MustRun(xgo)
args = args[:len(args)-1]
}
}
return
}
// Otherwise xxecute the explicit cross compilation
path := args[len(args)-1]
args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
xgo := xgoTool(args)
build.MustRun(xgo)
}
func xgoTool(args []string) *exec.Cmd {
cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
cmd.Env = []string{
"GOPATH=" + build.GOPATH(),
"GOBIN=" + GOBIN,
}
for _, e := range os.Environ() {
if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
continue
}
cmd.Env = append(cmd.Env, e)
}
return cmd
}
// Binary distribution cleanups
func doPurge(cmdline []string) {
var (
store = flag.String("store", "", `Destination from where to purge archives (usually "ethswarm/builds")`)
limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
)
flag.CommandLine.Parse(cmdline)
if env := build.Env(); !env.IsCronJob {
log.Printf("skipping because not a cron job")
os.Exit(0)
}
// Create the azure authentication and list the current archives
auth := build.AzureBlobstoreConfig{
Account: strings.Split(*store, "/")[0],
Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"),
Container: strings.SplitN(*store, "/", 2)[1],
}
blobs, err := build.AzureBlobstoreList(auth)
if err != nil {
log.Fatal(err)
}
// Iterate over the blobs, collect and sort all unstable builds
for i := 0; i < len(blobs); i++ {
if !strings.Contains(blobs[i].Name, "unstable") {
blobs = append(blobs[:i], blobs[i+1:]...)
i--
}
}
for i := 0; i < len(blobs); i++ {
for j := i + 1; j < len(blobs); j++ {
if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) {
blobs[i], blobs[j] = blobs[j], blobs[i]
}
}
}
// Filter out all archives more recent that the given threshold
for i, blob := range blobs {
if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
blobs = blobs[:i]
break
}
}
// Delete all marked as such and return
if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
log.Fatal(err)
}
}