build: add -dlgo flag in ci.go (#21824)

This new flag downloads a known version of Go and builds with it. This
is meant for environments where we can't easily upgrade the installed Go
version.

* .travis.yml: remove install step for PR test builders

We added this step originally to avoid re-building everything
for every test. go test has become much smarter in recent go
releases, so we no longer need to install anything here.

internal/build/archive.go

@@ -184,24 +184,35 @@ func (a *TarballArchive) Close() error {
 	return a.file.Close()
 }
 
-func ExtractTarballArchive(archive string, dest string) error {
-	// We're only interested in gzipped archives, wrap the reader now
+// ExtractArchive unpacks a .zip or .tar.gz archive to the destination directory.
+func ExtractArchive(archive string, dest string) error {
 	ar, err := os.Open(archive)
 	if err != nil {
 		return err
 	}
 	defer ar.Close()
 
+	switch {
+	case strings.HasSuffix(archive, ".tar.gz"):
+		return extractTarball(ar, dest)
+	case strings.HasSuffix(archive, ".zip"):
+		return extractZip(ar, dest)
+	default:
+		return fmt.Errorf("unhandled archive type %s", archive)
+	}
+}
+
+// extractTarball unpacks a .tar.gz file.
+func extractTarball(ar io.Reader, dest string) error {
 	gzr, err := gzip.NewReader(ar)
 	if err != nil {
 		return err
 	}
 	defer gzr.Close()
 
-	// Iterate over all the files in the tarball
 	tr := tar.NewReader(gzr)
 	for {
-		// Fetch the next tarball header and abort if needed
+		// Move to the next file header.
 		header, err := tr.Next()
 		if err != nil {
 			if err == io.EOF {
@@ -209,22 +220,69 @@ func ExtractTarballArchive(archive string, dest string) error {
 			}
 			return err
 		}
-		// Figure out the target and create it
-		target := filepath.Join(dest, header.Name)
-
-		switch header.Typeflag {
-		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-				return err
-			}
-			file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+		// We only care about regular files, directory modes
+		// and special file types are not supported.
+		if header.Typeflag == tar.TypeReg {
+			armode := header.FileInfo().Mode()
+			err := extractFile(header.Name, armode, tr, dest)
 			if err != nil {
-				return err
+				return fmt.Errorf("extract %s: %v", header.Name, err)
 			}
-			if _, err := io.Copy(file, tr); err != nil {
-				return err
-			}
-			file.Close()
 		}
 	}
 }
+
+// extractZip unpacks the given .zip file.
+func extractZip(ar *os.File, dest string) error {
+	info, err := ar.Stat()
+	if err != nil {
+		return err
+	}
+	zr, err := zip.NewReader(ar, info.Size())
+	if err != nil {
+		return err
+	}
+
+	for _, zf := range zr.File {
+		if !zf.Mode().IsRegular() {
+			continue
+		}
+
+		data, err := zf.Open()
+		if err != nil {
+			return err
+		}
+		err = extractFile(zf.Name, zf.Mode(), data, dest)
+		data.Close()
+		if err != nil {
+			return fmt.Errorf("extract %s: %v", zf.Name, err)
+		}
+	}
+	return nil
+}
+
+// extractFile extracts a single file from an archive.
+func extractFile(arpath string, armode os.FileMode, data io.Reader, dest string) error {
+	// Check that path is inside destination directory.
+	target := filepath.Join(dest, filepath.FromSlash(arpath))
+	if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) {
+		return fmt.Errorf("path %q escapes archive destination", target)
+	}
+
+	// Ensure the destination directory exists.
+	if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+		return err
+	}
+
+	// Copy file data.
+	file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, armode)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(file, data); err != nil {
+		file.Close()
+		os.Remove(target)
+		return err
+	}
+	return file.Close()
+}

internal/build/util.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"go/parser"
+	"go/token"
 	"io"
 	"io/ioutil"
 	"log"
@@ -152,3 +154,28 @@ func UploadSFTP(identityFile, host, dir string, files []string) error {
 	stdin.Close()
 	return sftp.Wait()
 }
+
+// FindMainPackages finds all 'main' packages in the given directory and returns their
+// package paths.
+func FindMainPackages(dir string) []string {
+	var commands []string
+	cmds, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, cmd := range cmds {
+		pkgdir := filepath.Join(dir, cmd.Name())
+		pkgs, err := parser.ParseDir(token.NewFileSet(), pkgdir, nil, parser.PackageClauseOnly)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for name := range pkgs {
+			if name == "main" {
+				path := "./" + filepath.ToSlash(pkgdir)
+				commands = append(commands, path)
+				break
+			}
+		}
+	}
+	return commands
+}