Major rewrite

* use dep for vendoring
* lets encrypt
* moved web to transfer.sh-web repo
* single command install
* added first tests

vendor/github.com/golang/gddo/database/database_test.go

@@ -0,0 +1,274 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"math"
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/garyburd/redigo/redis"
+	"golang.org/x/net/context"
+	"google.golang.org/appengine/aetest"
+
+	"github.com/golang/gddo/doc"
+)
+
+func newDB(t *testing.T) *Database {
+	p := redis.NewPool(func() (redis.Conn, error) {
+		c, err := redis.DialTimeout("tcp", ":6379", 0, 1*time.Second, 1*time.Second)
+		if err != nil {
+			return nil, err
+		}
+		_, err = c.Do("SELECT", "9")
+		if err != nil {
+			c.Close()
+			return nil, err
+		}
+		return c, nil
+	}, 1)
+
+	c := p.Get()
+	defer c.Close()
+	n, err := redis.Int(c.Do("DBSIZE"))
+	if n != 0 || err != nil {
+		t.Errorf("DBSIZE returned %d, %v", n, err)
+	}
+	return &Database{Pool: p}
+}
+
+func closeDB(db *Database) {
+	c := db.Pool.Get()
+	c.Do("FLUSHDB")
+	c.Close()
+}
+
+func TestPutGet(t *testing.T) {
+	var nextCrawl = time.Unix(time.Now().Add(time.Hour).Unix(), 0).UTC()
+	ctx, done, err := aetest.NewContext()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+	bgCtx = func() context.Context {
+		return ctx
+	}
+
+	db := newDB(t)
+	defer closeDB(db)
+	pdoc := &doc.Package{
+		ImportPath:  "github.com/user/repo/foo/bar",
+		Name:        "bar",
+		Synopsis:    "hello",
+		ProjectRoot: "github.com/user/repo",
+		ProjectName: "foo",
+		Updated:     time.Now().Add(-time.Hour),
+		Imports:     []string{"C", "errors", "github.com/user/repo/foo/bar"}, // self import for testing convenience.
+	}
+	if err := db.Put(pdoc, nextCrawl, false); err != nil {
+		t.Errorf("db.Put() returned error %v", err)
+	}
+	if err := db.Put(pdoc, time.Time{}, false); err != nil {
+		t.Errorf("second db.Put() returned error %v", err)
+	}
+
+	actualPdoc, actualSubdirs, actualCrawl, err := db.Get("github.com/user/repo/foo/bar")
+	if err != nil {
+		t.Fatalf("db.Get(.../foo/bar) returned %v", err)
+	}
+	if len(actualSubdirs) != 0 {
+		t.Errorf("db.Get(.../foo/bar) returned subdirs %v, want none", actualSubdirs)
+	}
+	if !reflect.DeepEqual(actualPdoc, pdoc) {
+		t.Errorf("db.Get(.../foo/bar) returned doc %v, want %v", actualPdoc, pdoc)
+	}
+	if !nextCrawl.Equal(actualCrawl) {
+		t.Errorf("db.Get(.../foo/bar) returned crawl %v, want %v", actualCrawl, nextCrawl)
+	}
+
+	before := time.Now().Unix()
+	if err := db.BumpCrawl(pdoc.ProjectRoot); err != nil {
+		t.Errorf("db.BumpCrawl() returned %v", err)
+	}
+	after := time.Now().Unix()
+
+	_, _, actualCrawl, _ = db.Get("github.com/user/repo/foo/bar")
+	if actualCrawl.Unix() < before || after < actualCrawl.Unix() {
+		t.Errorf("actualCrawl=%v, expect value between %v and %v", actualCrawl.Unix(), before, after)
+	}
+
+	// Popular
+
+	if err := db.IncrementPopularScore(pdoc.ImportPath); err != nil {
+		t.Errorf("db.IncrementPopularScore() returned %v", err)
+	}
+
+	// Get "-"
+
+	actualPdoc, _, _, err = db.Get("-")
+	if err != nil {
+		t.Fatalf("db.Get(-) returned %v", err)
+	}
+	if !reflect.DeepEqual(actualPdoc, pdoc) {
+		t.Errorf("db.Get(-) returned doc %v, want %v", actualPdoc, pdoc)
+	}
+
+	actualPdoc, actualSubdirs, _, err = db.Get("github.com/user/repo/foo")
+	if err != nil {
+		t.Fatalf("db.Get(.../foo) returned %v", err)
+	}
+	if actualPdoc != nil {
+		t.Errorf("db.Get(.../foo) returned doc %v, want %v", actualPdoc, nil)
+	}
+	expectedSubdirs := []Package{{Path: "github.com/user/repo/foo/bar", Synopsis: "hello"}}
+	if !reflect.DeepEqual(actualSubdirs, expectedSubdirs) {
+		t.Errorf("db.Get(.../foo) returned subdirs %v, want %v", actualSubdirs, expectedSubdirs)
+	}
+	actualImporters, err := db.Importers("github.com/user/repo/foo/bar")
+	if err != nil {
+		t.Fatalf("db.Importers() returned error %v", err)
+	}
+	expectedImporters := []Package{{Path: "github.com/user/repo/foo/bar", Synopsis: "hello"}}
+	if !reflect.DeepEqual(actualImporters, expectedImporters) {
+		t.Errorf("db.Importers() = %v, want %v", actualImporters, expectedImporters)
+	}
+	actualImports, err := db.Packages(pdoc.Imports)
+	if err != nil {
+		t.Fatalf("db.Imports() returned error %v", err)
+	}
+	for i := range actualImports {
+		if actualImports[i].Path == "C" {
+			actualImports[i].Synopsis = ""
+		}
+	}
+	expectedImports := []Package{
+		{Path: "C", Synopsis: ""},
+		{Path: "errors", Synopsis: ""},
+		{Path: "github.com/user/repo/foo/bar", Synopsis: "hello"},
+	}
+	if !reflect.DeepEqual(actualImports, expectedImports) {
+		t.Errorf("db.Imports() = %v, want %v", actualImports, expectedImports)
+	}
+	importerCount, _ := db.ImporterCount("github.com/user/repo/foo/bar")
+	if importerCount != 1 {
+		t.Errorf("db.ImporterCount() = %d, want %d", importerCount, 1)
+	}
+	if err := db.Delete("github.com/user/repo/foo/bar"); err != nil {
+		t.Errorf("db.Delete() returned error %v", err)
+	}
+
+	db.Query("bar")
+
+	if err := db.Put(pdoc, time.Time{}, false); err != nil {
+		t.Errorf("db.Put() returned error %v", err)
+	}
+
+	if err := db.Block("github.com/user/repo"); err != nil {
+		t.Errorf("db.Block() returned error %v", err)
+	}
+
+	blocked, err := db.IsBlocked("github.com/user/repo/foo/bar")
+	if !blocked || err != nil {
+		t.Errorf("db.IsBlocked(github.com/user/repo/foo/bar) returned %v, %v, want true, nil", blocked, err)
+	}
+
+	blocked, err = db.IsBlocked("github.com/foo/bar")
+	if blocked || err != nil {
+		t.Errorf("db.IsBlocked(github.com/foo/bar) returned %v, %v, want false, nil", blocked, err)
+	}
+
+	c := db.Pool.Get()
+	defer c.Close()
+	c.Send("DEL", "maxQueryId")
+	c.Send("DEL", "maxPackageId")
+	c.Send("DEL", "block")
+	c.Send("DEL", "popular:0")
+	c.Send("DEL", "newCrawl")
+	keys, err := redis.Values(c.Do("HKEYS", "ids"))
+	for _, key := range keys {
+		t.Errorf("unexpected id %s", key)
+	}
+	keys, err = redis.Values(c.Do("KEYS", "*"))
+	for _, key := range keys {
+		t.Errorf("unexpected key %s", key)
+	}
+}
+
+const epsilon = 0.000001
+
+func TestPopular(t *testing.T) {
+	db := newDB(t)
+	defer closeDB(db)
+	c := db.Pool.Get()
+	defer c.Close()
+
+	// Add scores for packages. On each iteration, add half-life to time and
+	// divide the score by two. All packages should have the same score.
+
+	now := time.Now()
+	score := float64(4048)
+	for id := 12; id >= 0; id-- {
+		path := "github.com/user/repo/p" + strconv.Itoa(id)
+		c.Do("HSET", "ids", path, id)
+		err := db.incrementPopularScoreInternal(path, score, now)
+		if err != nil {
+			t.Fatal(err)
+		}
+		now = now.Add(popularHalfLife)
+		score /= 2
+	}
+
+	values, _ := redis.Values(c.Do("ZRANGE", "popular", "0", "100000", "WITHSCORES"))
+	if len(values) != 26 {
+		t.Fatalf("Expected 26 values, got %d", len(values))
+	}
+
+	// Check for equal scores.
+	score, err := redis.Float64(values[1], nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 3; i < len(values); i += 2 {
+		s, _ := redis.Float64(values[i], nil)
+		if math.Abs(score-s)/score > epsilon {
+			t.Errorf("Bad score, score[1]=%g, score[%d]=%g", score, i, s)
+		}
+	}
+}
+
+func TestCounter(t *testing.T) {
+	db := newDB(t)
+	defer closeDB(db)
+
+	const key = "127.0.0.1"
+
+	now := time.Now()
+	n, err := db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-1.0) > epsilon {
+		t.Errorf("1: got n=%g, want 1", n)
+	}
+	n, err = db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-2.0)/2.0 > epsilon {
+		t.Errorf("2: got n=%g, want 2", n)
+	}
+	now = now.Add(counterHalflife)
+	n, err = db.incrementCounterInternal(key, 1, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if math.Abs(n-2.0)/2.0 > epsilon {
+		t.Errorf("3: got n=%g, want 2", n)
+	}
+}

vendor/github.com/golang/gddo/database/index.go

@@ -0,0 +1,243 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"path"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/golang/gddo/doc"
+	"github.com/golang/gddo/gosrc"
+)
+
+func isStandardPackage(path string) bool {
+	return strings.Index(path, ".") < 0
+}
+
+func isTermSep(r rune) bool {
+	return unicode.IsSpace(r) ||
+		r != '.' && unicode.IsPunct(r) ||
+		unicode.IsSymbol(r)
+}
+
+func normalizeProjectRoot(projectRoot string) string {
+	if projectRoot == "" {
+		return "go"
+	}
+	return projectRoot
+}
+
+var synonyms = map[string]string{
+	"redis":    "redisdb", // append db to avoid stemming to 'red'
+	"rand":     "random",
+	"postgres": "postgresql",
+	"mongo":    "mongodb",
+}
+
+func term(s string) string {
+	s = strings.ToLower(s)
+	if x, ok := synonyms[s]; ok {
+		s = x
+	}
+
+	// Trim the trailing period at the end of any sentence.
+	return stem(strings.TrimSuffix(s, "."))
+}
+
+var httpPat = regexp.MustCompile(`https?://\S+`)
+
+func collectSynopsisTerms(terms map[string]bool, synopsis string) {
+
+	synopsis = httpPat.ReplaceAllLiteralString(synopsis, "")
+
+	fields := strings.FieldsFunc(synopsis, isTermSep)
+	for i := range fields {
+		fields[i] = strings.ToLower(fields[i])
+	}
+
+	// Ignore boilerplate in the following common patterns:
+	//  Package foo ...
+	//  Command foo ...
+	//  Package foo implements ... (and provides, contains)
+	//  The foo package ...
+	//  The foo package implements ...
+	//  The foo command ...
+
+	checkPackageVerb := false
+	switch {
+	case len(fields) >= 1 && fields[0] == "package":
+		fields = fields[1:]
+		checkPackageVerb = true
+	case len(fields) >= 1 && fields[0] == "command":
+		fields = fields[1:]
+	case len(fields) >= 3 && fields[0] == "the" && fields[2] == "package":
+		fields[2] = fields[1]
+		fields = fields[2:]
+		checkPackageVerb = true
+	case len(fields) >= 3 && fields[0] == "the" && fields[2] == "command":
+		fields[2] = fields[1]
+		fields = fields[2:]
+	}
+
+	if checkPackageVerb && len(fields) >= 2 &&
+		(fields[1] == "implements" || fields[1] == "provides" || fields[1] == "contains") {
+		fields[1] = fields[0]
+		fields = fields[1:]
+	}
+
+	for _, s := range fields {
+		if !stopWord[s] {
+			terms[term(s)] = true
+		}
+	}
+}
+
+func termSlice(terms map[string]bool) []string {
+	result := make([]string, 0, len(terms))
+	for term := range terms {
+		result = append(result, term)
+	}
+	return result
+}
+
+func documentTerms(pdoc *doc.Package, score float64) []string {
+
+	terms := make(map[string]bool)
+
+	// Project root
+
+	projectRoot := normalizeProjectRoot(pdoc.ProjectRoot)
+	terms["project:"+projectRoot] = true
+
+	if strings.HasPrefix(pdoc.ImportPath, "golang.org/x/") {
+		terms["project:subrepo"] = true
+	}
+
+	// Imports
+
+	for _, path := range pdoc.Imports {
+		if gosrc.IsValidPath(path) {
+			terms["import:"+path] = true
+		}
+	}
+
+	if score > 0 {
+
+		for _, term := range parseQuery(pdoc.ImportPath) {
+			terms[term] = true
+		}
+		if !isStandardPackage(pdoc.ImportPath) {
+			terms["all:"] = true
+			for _, term := range parseQuery(pdoc.ProjectName) {
+				terms[term] = true
+			}
+			for _, term := range parseQuery(pdoc.Name) {
+				terms[term] = true
+			}
+		}
+
+		// Synopsis
+
+		collectSynopsisTerms(terms, pdoc.Synopsis)
+
+	}
+
+	return termSlice(terms)
+}
+
+// vendorPat matches the path of a vendored package.
+var vendorPat = regexp.MustCompile(
+	// match directories used by tools to vendor packages.
+	`/(?:_?third_party|vendors|Godeps/_workspace/src)/` +
+		// match a domain name.
+		`[^./]+\.[^/]+`)
+
+func documentScore(pdoc *doc.Package) float64 {
+	if pdoc.Name == "" ||
+		pdoc.Status != gosrc.Active ||
+		len(pdoc.Errors) > 0 ||
+		strings.HasSuffix(pdoc.ImportPath, ".go") ||
+		strings.HasPrefix(pdoc.ImportPath, "gist.github.com/") ||
+		strings.HasSuffix(pdoc.ImportPath, "/internal") ||
+		strings.Contains(pdoc.ImportPath, "/internal/") ||
+		vendorPat.MatchString(pdoc.ImportPath) {
+		return 0
+	}
+
+	for _, p := range pdoc.Imports {
+		if strings.HasSuffix(p, ".go") {
+			return 0
+		}
+	}
+
+	r := 1.0
+	if pdoc.IsCmd {
+		if pdoc.Doc == "" {
+			// Do not include command in index if it does not have documentation.
+			return 0
+		}
+		if !importsGoPackages(pdoc) {
+			// Penalize commands that don't use the "go/*" packages.
+			r *= 0.9
+		}
+	} else {
+		if !pdoc.Truncated &&
+			len(pdoc.Consts) == 0 &&
+			len(pdoc.Vars) == 0 &&
+			len(pdoc.Funcs) == 0 &&
+			len(pdoc.Types) == 0 &&
+			len(pdoc.Examples) == 0 {
+			// Do not include package in index if it does not have exports.
+			return 0
+		}
+		if pdoc.Doc == "" {
+			// Penalty for no documentation.
+			r *= 0.95
+		}
+		if path.Base(pdoc.ImportPath) != pdoc.Name {
+			// Penalty for last element of path != package name.
+			r *= 0.9
+		}
+		for i := 0; i < strings.Count(pdoc.ImportPath[len(pdoc.ProjectRoot):], "/"); i++ {
+			// Penalty for deeply nested packages.
+			r *= 0.99
+		}
+		if strings.Index(pdoc.ImportPath[len(pdoc.ProjectRoot):], "/src/") > 0 {
+			r *= 0.95
+		}
+		for _, p := range pdoc.Imports {
+			if vendorPat.MatchString(p) {
+				// Penalize packages that import vendored packages.
+				r *= 0.1
+				break
+			}
+		}
+	}
+	return r
+}
+
+func parseQuery(q string) []string {
+	var terms []string
+	q = strings.ToLower(q)
+	for _, s := range strings.FieldsFunc(q, isTermSep) {
+		if !stopWord[s] {
+			terms = append(terms, term(s))
+		}
+	}
+	return terms
+}
+
+func importsGoPackages(pdoc *doc.Package) bool {
+	for _, m := range pdoc.Imports {
+		if strings.HasPrefix(m, "go/") {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/golang/gddo/database/index_test.go

@@ -0,0 +1,202 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/golang/gddo/doc"
+)
+
+var indexTests = []struct {
+	pdoc  *doc.Package
+	terms []string
+}{
+	{&doc.Package{
+		ImportPath:  "strconv",
+		ProjectRoot: "",
+		ProjectName: "Go",
+		Name:        "strconv",
+		Synopsis:    "Package strconv implements conversions to and from string representations of basic data types.",
+		Doc:         "Package strconv implements conversions to and from string representations\nof basic data types.",
+		Imports:     []string{"errors", "math", "unicode/utf8"},
+		Funcs:       []*doc.Func{{}},
+	},
+		[]string{
+			"bas",
+			"convert",
+			"dat",
+			"import:errors",
+			"import:math",
+			"import:unicode/utf8",
+			"project:go",
+			"repres",
+			"strconv",
+			"string",
+			"typ"},
+	},
+	{&doc.Package{
+		ImportPath:  "github.com/user/repo/dir",
+		ProjectRoot: "github.com/user/repo",
+		ProjectName: "go-oauth",
+		ProjectURL:  "https://github.com/user/repo/",
+		Name:        "dir",
+		Synopsis:    "Package dir implements a subset of the OAuth client interface as defined in RFC 5849.",
+		Doc: "Package oauth implements a subset of the OAuth client interface as defined in RFC 5849.\n\n" +
+			"This package assumes that the application writes request URL paths to the\nnetwork using " +
+			"the encoding implemented by the net/url URL RequestURI method.\n" +
+			"The HTTP client in the standard net/http package uses this encoding.",
+		IsCmd: false,
+		Imports: []string{
+			"bytes",
+			"crypto/hmac",
+			"crypto/sha1",
+			"encoding/base64",
+			"encoding/binary",
+			"errors",
+			"fmt",
+			"io",
+			"io/ioutil",
+			"net/http",
+			"net/url",
+			"regexp",
+			"sort",
+			"strconv",
+			"strings",
+			"sync",
+			"time",
+		},
+		TestImports: []string{"bytes", "net/url", "testing"},
+		Funcs:       []*doc.Func{{}},
+	},
+		[]string{
+			"all:",
+			"5849", "cly", "defin", "dir", "github.com", "go",
+			"import:bytes", "import:crypto/hmac", "import:crypto/sha1",
+			"import:encoding/base64", "import:encoding/binary", "import:errors",
+			"import:fmt", "import:io", "import:io/ioutil", "import:net/http",
+			"import:net/url", "import:regexp", "import:sort", "import:strconv",
+			"import:strings", "import:sync", "import:time", "interfac",
+			"oau", "project:github.com/user/repo", "repo", "rfc", "subset", "us",
+		},
+	},
+}
+
+func TestDocTerms(t *testing.T) {
+	for _, tt := range indexTests {
+		score := documentScore(tt.pdoc)
+		terms := documentTerms(tt.pdoc, score)
+		sort.Strings(terms)
+		sort.Strings(tt.terms)
+		if !reflect.DeepEqual(terms, tt.terms) {
+			t.Errorf("documentTerms(%s) ->\n got: %#v\nwant: %#v", tt.pdoc.ImportPath, terms, tt.terms)
+		}
+	}
+}
+
+var vendorPatTests = []struct {
+	path  string
+	match bool
+}{
+	{"camlistore.org/third_party/github.com/user/repo", true},
+	{"camlistore.org/third_party/dir", false},
+	{"camlistore.org/third_party", false},
+	{"camlistore.org/xthird_party/github.com/user/repo", false},
+	{"camlistore.org/third_partyx/github.com/user/repo", false},
+
+	{"example.org/_third_party/github.com/user/repo/dir", true},
+	{"example.org/_third_party/dir", false},
+
+	{"github.com/user/repo/Godeps/_workspace/src/github.com/user/repo", true},
+	{"github.com/user/repo/Godeps/_workspace/src/dir", false},
+
+	{"github.com/user/repo", false},
+}
+
+func TestVendorPat(t *testing.T) {
+	for _, tt := range vendorPatTests {
+		match := vendorPat.MatchString(tt.path)
+		if match != tt.match {
+			t.Errorf("match(%q) = %v, want %v", tt.path, match, match)
+		}
+	}
+}
+
+var synopsisTermTests = []struct {
+	synopsis string
+	terms    []string
+}{
+	{
+		"Package foo implements bar.",
+		[]string{"bar", "foo"},
+	},
+	{
+		"Package foo provides bar.",
+		[]string{"bar", "foo"},
+	},
+	{
+		"The foo package provides bar.",
+		[]string{"bar", "foo"},
+	},
+	{
+		"Package foo contains an implementation of bar.",
+		[]string{"bar", "foo", "impl"},
+	},
+	{
+		"Package foo is awesome",
+		[]string{"awesom", "foo"},
+	},
+	{
+		"The foo package is awesome",
+		[]string{"awesom", "foo"},
+	},
+	{
+		"The foo command is awesome",
+		[]string{"awesom", "foo"},
+	},
+	{
+		"Command foo is awesome",
+		[]string{"awesom", "foo"},
+	},
+	{
+		"The foo package",
+		[]string{"foo"},
+	},
+	{
+		"Package foo",
+		[]string{"foo"},
+	},
+	{
+		"Command foo",
+		[]string{"foo"},
+	},
+	{
+		"Package",
+		[]string{},
+	},
+	{
+		"Command",
+		[]string{},
+	},
+}
+
+func TestSynopsisTerms(t *testing.T) {
+	for _, tt := range synopsisTermTests {
+		terms := make(map[string]bool)
+		collectSynopsisTerms(terms, tt.synopsis)
+
+		actual := termSlice(terms)
+		expected := tt.terms
+		sort.Strings(actual)
+		sort.Strings(expected)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("%q ->\n got: %#v\nwant: %#v", tt.synopsis, actual, expected)
+		}
+	}
+}

vendor/github.com/golang/gddo/database/indexae.go

@@ -0,0 +1,207 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"strings"
+	"unicode"
+
+	"golang.org/x/net/context"
+	"google.golang.org/appengine/search"
+
+	"github.com/golang/gddo/doc"
+)
+
+func (p *Package) Load(fields []search.Field, meta *search.DocumentMetadata) error {
+	for _, f := range fields {
+		switch f.Name {
+		case "Name":
+			if v, ok := f.Value.(search.Atom); ok {
+				p.Name = string(v)
+			}
+		case "Path":
+			if v, ok := f.Value.(string); ok {
+				p.Path = v
+			}
+		case "Synopsis":
+			if v, ok := f.Value.(string); ok {
+				p.Synopsis = v
+			}
+		case "ImportCount":
+			if v, ok := f.Value.(float64); ok {
+				p.ImportCount = int(v)
+			}
+		case "Stars":
+			if v, ok := f.Value.(float64); ok {
+				p.Stars = int(v)
+			}
+		case "Score":
+			if v, ok := f.Value.(float64); ok {
+				p.Score = v
+			}
+		}
+	}
+	if p.Path == "" {
+		return errors.New("Invalid document: missing Path field")
+	}
+	for _, f := range meta.Facets {
+		if f.Name == "Fork" {
+			p.Fork = f.Value.(search.Atom) == "true"
+		}
+	}
+	return nil
+}
+
+func (p *Package) Save() ([]search.Field, *search.DocumentMetadata, error) {
+	fields := []search.Field{
+		{Name: "Name", Value: search.Atom(p.Name)},
+		{Name: "Path", Value: p.Path},
+		{Name: "Synopsis", Value: p.Synopsis},
+		{Name: "Score", Value: p.Score},
+		{Name: "ImportCount", Value: float64(p.ImportCount)},
+		{Name: "Stars", Value: float64(p.Stars)},
+	}
+	fork := fmt.Sprint(p.Fork) // "true" or "false"
+	meta := &search.DocumentMetadata{
+		// Customize the rank property by the product of the package score and
+		// natural logarithm of the import count. Rank must be a positive integer.
+		// Use 1 as minimum rank and keep 3 digits of precision to distinguish
+		// close ranks.
+		Rank: int(math.Max(1, 1000*p.Score*math.Log(math.E+float64(p.ImportCount)))),
+		Facets: []search.Facet{
+			{Name: "Fork", Value: search.Atom(fork)},
+		},
+	}
+	return fields, meta, nil
+}
+
+// PutIndex creates or updates a package entry in the search index. id identifies the document in the index.
+// If pdoc is non-nil, PutIndex will update the package's name, path and synopsis supplied by pdoc.
+// pdoc must be non-nil for a package's first call to PutIndex.
+// PutIndex updates the Score to score, if non-negative.
+func PutIndex(c context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
+	if id == "" {
+		return errors.New("indexae: no id assigned")
+	}
+	idx, err := search.Open("packages")
+	if err != nil {
+		return err
+	}
+
+	var pkg Package
+	if err := idx.Get(c, id, &pkg); err != nil {
+		if err != search.ErrNoSuchDocument {
+			return err
+		} else if pdoc == nil {
+			// Cannot update a non-existing document.
+			return errors.New("indexae: cannot create new document with nil pdoc")
+		}
+		// No such document in the index, fall through.
+	}
+
+	// Update document information accordingly.
+	if pdoc != nil {
+		pkg.Name = pdoc.Name
+		pkg.Path = pdoc.ImportPath
+		pkg.Synopsis = pdoc.Synopsis
+		pkg.Stars = pdoc.Stars
+		pkg.Fork = pdoc.Fork
+	}
+	if score >= 0 {
+		pkg.Score = score
+	}
+	pkg.ImportCount = importCount
+
+	if _, err := idx.Put(c, id, &pkg); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Search searches the packages index for a given query. A path-like query string
+// will be passed in unchanged, whereas single words will be stemmed.
+func Search(c context.Context, q string) ([]Package, error) {
+	index, err := search.Open("packages")
+	if err != nil {
+		return nil, err
+	}
+	var pkgs []Package
+	opt := &search.SearchOptions{
+		Limit: 100,
+	}
+	for it := index.Search(c, parseQuery2(q), opt); ; {
+		var p Package
+		_, err := it.Next(&p)
+		if err == search.Done {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		pkgs = append(pkgs, p)
+	}
+	return pkgs, nil
+}
+
+func parseQuery2(q string) string {
+	var buf bytes.Buffer
+	for _, s := range strings.FieldsFunc(q, isTermSep2) {
+		if strings.ContainsAny(s, "./") {
+			// Quote terms with / or . for path like query.
+			fmt.Fprintf(&buf, "%q ", s)
+		} else {
+			// Stem for single word terms.
+			fmt.Fprintf(&buf, "~%v ", s)
+		}
+	}
+	return buf.String()
+}
+
+func isTermSep2(r rune) bool {
+	return unicode.IsSpace(r) ||
+		r != '.' && r != '/' && unicode.IsPunct(r) ||
+		unicode.IsSymbol(r)
+}
+
+func deleteIndex(c context.Context, id string) error {
+	idx, err := search.Open("packages")
+	if err != nil {
+		return err
+	}
+	return idx.Delete(c, id)
+}
+
+// PurgeIndex deletes all the packages from the search index.
+func PurgeIndex(c context.Context) error {
+	idx, err := search.Open("packages")
+	if err != nil {
+		return err
+	}
+	n := 0
+
+	for it := idx.List(c, &search.ListOptions{IDsOnly: true}); ; n++ {
+		var pkg Package
+		id, err := it.Next(&pkg)
+		if err == search.Done {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if err := idx.Delete(c, id); err != nil {
+			log.Printf("Failed to delete package %s: %v", id, err)
+			continue
+		}
+	}
+	log.Printf("Purged %d packages from the search index.", n)
+	return nil
+}

vendor/github.com/golang/gddo/database/indexae_test.go

@@ -0,0 +1,126 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"math"
+	"strconv"
+	"testing"
+
+	"google.golang.org/appengine/aetest"
+	"google.golang.org/appengine/search"
+
+	"github.com/golang/gddo/doc"
+)
+
+var pdoc = &doc.Package{
+	ImportPath: "github.com/golang/test",
+	Name:       "test",
+	Synopsis:   "This is a test package.",
+	Fork:       true,
+	Stars:      10,
+}
+
+func TestPutIndexWithEmptyId(t *testing.T) {
+	c, done, err := aetest.NewContext()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	if err := PutIndex(c, nil, "", 0, 0); err == nil {
+		t.Errorf("PutIndex succeeded unexpectedly")
+	}
+}
+
+func TestPutIndexCreateNilDoc(t *testing.T) {
+	c, done, err := aetest.NewContext()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	if err := PutIndex(c, nil, "12345", -1, 2); err == nil {
+		t.Errorf("PutIndex succeeded unexpectedly")
+	}
+}
+
+func TestPutIndexNewPackageAndUpdate(t *testing.T) {
+	c, done, err := aetest.NewContext()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	// Put a new package into search index.
+	if err := PutIndex(c, pdoc, "12345", 0.99, 1); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the package was put in is as expected.
+	idx, err := search.Open("packages")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got Package
+	if err = idx.Get(c, "12345", &got); err != nil && err != search.ErrNoSuchDocument {
+		t.Fatal(err)
+	}
+	wanted := Package{
+		Name:        pdoc.Name,
+		Path:        pdoc.ImportPath,
+		Synopsis:    pdoc.Synopsis,
+		ImportCount: 1,
+		Fork:        true,
+		Stars:       10,
+		Score:       0.99,
+	}
+	if got != wanted {
+		t.Errorf("PutIndex got %v, want %v", got, wanted)
+	}
+
+	// Update the import count of the package.
+	if err := PutIndex(c, nil, "12345", -1, 2); err != nil {
+		t.Fatal(err)
+	}
+	if err := idx.Get(c, "12345", &got); err != nil && err != search.ErrNoSuchDocument {
+		t.Fatal(err)
+	}
+	wanted.ImportCount = 2
+	if got != wanted {
+		t.Errorf("PutIndex got %v, want %v", got, wanted)
+	}
+}
+
+func TestSearchResultSorted(t *testing.T) {
+	c, done, err := aetest.NewContext()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	// Put multiple packages into the search index and the search result
+	// should be sorted properly.
+	id := "1"
+	for i := 2; i < 6; i++ {
+		id += strconv.Itoa(i)
+		pdoc.Synopsis = id
+		if err := PutIndex(c, pdoc, id, math.Pow(0.9, float64(i)), 10*i); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, err := Search(c, "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wanted := []string{"123", "12", "1234", "12345"}
+	for i, p := range got {
+		if p.Synopsis != wanted[i] {
+			t.Errorf("Search got %v, want %v", p.Synopsis, wanted[i])
+		}
+	}
+}

vendor/github.com/golang/gddo/database/stem.go

@@ -0,0 +1,123 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// This file implements the Paice/Husk stemming algorithm.
+// http://www.comp.lancs.ac.uk/computing/research/stemming/Links/paice.htm
+
+package database
+
+import (
+	"bytes"
+	"regexp"
+	"strconv"
+)
+
+const stemRuleText = `
+ai*2. a*1. 
+bb1. 
+city3s. ci2> cn1t> 
+dd1. dei3y> deec2ss. dee1. de2> dooh4> 
+e1> 
+feil1v. fi2> 
+gni3> gai3y. ga2> gg1. 
+ht*2. hsiug5ct. hsi3> 
+i*1. i1y> 
+ji1d. juf1s. ju1d. jo1d. jeh1r. jrev1t. jsim2t. jn1d. j1s. 
+lbaifi6. lbai4y. lba3> lbi3. lib2l> lc1. lufi4y. luf3> lu2. lai3> lau3> la2> ll1. 
+mui3. mu*2. msi3> mm1. 
+nois4j> noix4ct. noi3> nai3> na2> nee0. ne2> nn1. 
+pihs4> pp1. 
+re2> rae0. ra2. ro2> ru2> rr1. rt1> rei3y> 
+sei3y> sis2. si2> ssen4> ss0. suo3> su*2. s*1> s0. 
+tacilp4y. ta2> tnem4> tne3> tna3> tpir2b. tpro2b. tcud1. tpmus2. tpec2iv. tulo2v. tsis0. tsi3> tt1. 
+uqi3. ugo1. 
+vis3j> vie0. vi2> 
+ylb1> yli3y> ylp0. yl2> ygo1. yhp1. ymo1. ypo1. yti3> yte3> ytl2. yrtsi5. yra3> yro3> yfi3. ycn2t> yca3> 
+zi2> zy1s. 
+`
+
+type stemRule struct {
+	text   string
+	suffix []byte
+	intact bool
+	remove int
+	append []byte
+	more   bool
+}
+
+func parseStemRules() map[byte][]*stemRule {
+
+	rules := make(map[byte][]*stemRule)
+	for _, m := range regexp.MustCompile(`(?m)(?:^| )([a-zA-Z]*)(\*?)([0-9])([a-zA-z]*)([.>])`).FindAllStringSubmatch(stemRuleText, -1) {
+
+		suffix := []byte(m[1])
+		for i := 0; i < len(suffix)/2; i++ {
+			j := len(suffix) - 1 - i
+			suffix[i], suffix[j] = suffix[j], suffix[i]
+		}
+
+		remove, _ := strconv.Atoi(m[3])
+		r := &stemRule{
+			text:   m[0],
+			suffix: suffix,
+			intact: m[2] == "*",
+			remove: remove,
+			append: []byte(m[4]),
+			more:   m[5] == ">",
+		}
+		c := suffix[len(suffix)-1]
+		rules[c] = append(rules[c], r)
+	}
+	return rules
+}
+
+var stemRules = parseStemRules()
+
+func firstVowel(offset int, p []byte) int {
+	for i, b := range p {
+		switch b {
+		case 'a', 'e', 'i', 'o', 'u':
+			return offset + i
+		case 'y':
+			if offset+i > 0 {
+				return offset + i
+			}
+		}
+	}
+	return -1
+}
+
+func acceptableStem(a, b []byte) bool {
+	i := firstVowel(0, a)
+	if i < 0 {
+		i = firstVowel(len(a), b)
+	}
+	l := len(a) + len(b)
+	if i == 0 {
+		return l > 1
+	}
+	return i >= 0 && l > 2
+}
+
+func stem(s string) string {
+	stem := bytes.ToLower([]byte(s))
+	intact := true
+	run := acceptableStem(stem, []byte{})
+	for run {
+		run = false
+		for _, rule := range stemRules[stem[len(stem)-1]] {
+			if bytes.HasSuffix(stem, rule.suffix) &&
+				(intact || !rule.intact) &&
+				acceptableStem(stem[:len(stem)-rule.remove], rule.append) {
+				stem = append(stem[:len(stem)-rule.remove], rule.append...)
+				intact = false
+				run = rule.more
+				break
+			}
+		}
+	}
+	return string(stem)
+}

vendor/github.com/golang/gddo/database/stem_test.go

@@ -0,0 +1,31 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"testing"
+)
+
+var stemTests = []struct {
+	s, expected string
+}{
+	{"html", "html"},
+	{"strings", "string"},
+	{"ballroom", "ballroom"},
+	{"mechanicalization", "mech"},
+	{"pragmaticality", "pragm"},
+	{"rationalistically", "rat"},
+}
+
+func TestStem(t *testing.T) {
+	for _, tt := range stemTests {
+		actual := stem(tt.s)
+		if actual != tt.expected {
+			t.Errorf("stem(%q) = %q, want %q", tt.s, actual, tt.expected)
+		}
+	}
+}

vendor/github.com/golang/gddo/database/stop.go

@@ -0,0 +1,141 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"strings"
+)
+
+var stopWord = createStopWordMap()
+
+func createStopWordMap() map[string]bool {
+	m := make(map[string]bool)
+	for _, s := range strings.Fields(stopText) {
+		m[s] = true
+	}
+	return m
+}
+
+const stopText = `
+a
+about
+after
+all
+also
+am
+an
+and
+another
+any
+are
+as
+at
+b
+be
+because
+been
+before
+being
+between
+both
+but
+by
+c
+came
+can
+come
+could
+d
+did
+do
+e
+each
+f
+for
+from
+g
+get
+got
+h
+had
+has
+have
+he
+her
+here
+him
+himself
+his
+how
+i
+if
+in
+into
+is
+it
+j
+k
+l
+like
+m
+make
+many
+me
+might
+more
+most
+much
+must
+my
+n
+never
+now
+o
+of
+on
+only
+or
+other
+our
+out
+over
+p
+q
+r
+s
+said
+same
+see
+should
+since
+some
+still
+such
+t
+take
+than
+that
+the
+their
+them
+then
+there
+these
+they
+this
+those
+through
+to
+too
+u
+under
+v
+w
+x
+y
+z
+`