Major rewrite

* use dep for vendoring
* lets encrypt
* moved web to transfer.sh-web repo
* single command install
* added first tests

vendor/github.com/golang/gddo/database/index.go

@@ -0,0 +1,243 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package database
+
+import (
+	"path"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/golang/gddo/doc"
+	"github.com/golang/gddo/gosrc"
+)
+
+func isStandardPackage(path string) bool {
+	return strings.Index(path, ".") < 0
+}
+
+func isTermSep(r rune) bool {
+	return unicode.IsSpace(r) ||
+		r != '.' && unicode.IsPunct(r) ||
+		unicode.IsSymbol(r)
+}
+
+func normalizeProjectRoot(projectRoot string) string {
+	if projectRoot == "" {
+		return "go"
+	}
+	return projectRoot
+}
+
+var synonyms = map[string]string{
+	"redis":    "redisdb", // append db to avoid stemming to 'red'
+	"rand":     "random",
+	"postgres": "postgresql",
+	"mongo":    "mongodb",
+}
+
+func term(s string) string {
+	s = strings.ToLower(s)
+	if x, ok := synonyms[s]; ok {
+		s = x
+	}
+
+	// Trim the trailing period at the end of any sentence.
+	return stem(strings.TrimSuffix(s, "."))
+}
+
+var httpPat = regexp.MustCompile(`https?://\S+`)
+
+func collectSynopsisTerms(terms map[string]bool, synopsis string) {
+
+	synopsis = httpPat.ReplaceAllLiteralString(synopsis, "")
+
+	fields := strings.FieldsFunc(synopsis, isTermSep)
+	for i := range fields {
+		fields[i] = strings.ToLower(fields[i])
+	}
+
+	// Ignore boilerplate in the following common patterns:
+	//  Package foo ...
+	//  Command foo ...
+	//  Package foo implements ... (and provides, contains)
+	//  The foo package ...
+	//  The foo package implements ...
+	//  The foo command ...
+
+	checkPackageVerb := false
+	switch {
+	case len(fields) >= 1 && fields[0] == "package":
+		fields = fields[1:]
+		checkPackageVerb = true
+	case len(fields) >= 1 && fields[0] == "command":
+		fields = fields[1:]
+	case len(fields) >= 3 && fields[0] == "the" && fields[2] == "package":
+		fields[2] = fields[1]
+		fields = fields[2:]
+		checkPackageVerb = true
+	case len(fields) >= 3 && fields[0] == "the" && fields[2] == "command":
+		fields[2] = fields[1]
+		fields = fields[2:]
+	}
+
+	if checkPackageVerb && len(fields) >= 2 &&
+		(fields[1] == "implements" || fields[1] == "provides" || fields[1] == "contains") {
+		fields[1] = fields[0]
+		fields = fields[1:]
+	}
+
+	for _, s := range fields {
+		if !stopWord[s] {
+			terms[term(s)] = true
+		}
+	}
+}
+
+func termSlice(terms map[string]bool) []string {
+	result := make([]string, 0, len(terms))
+	for term := range terms {
+		result = append(result, term)
+	}
+	return result
+}
+
+func documentTerms(pdoc *doc.Package, score float64) []string {
+
+	terms := make(map[string]bool)
+
+	// Project root
+
+	projectRoot := normalizeProjectRoot(pdoc.ProjectRoot)
+	terms["project:"+projectRoot] = true
+
+	if strings.HasPrefix(pdoc.ImportPath, "golang.org/x/") {
+		terms["project:subrepo"] = true
+	}
+
+	// Imports
+
+	for _, path := range pdoc.Imports {
+		if gosrc.IsValidPath(path) {
+			terms["import:"+path] = true
+		}
+	}
+
+	if score > 0 {
+
+		for _, term := range parseQuery(pdoc.ImportPath) {
+			terms[term] = true
+		}
+		if !isStandardPackage(pdoc.ImportPath) {
+			terms["all:"] = true
+			for _, term := range parseQuery(pdoc.ProjectName) {
+				terms[term] = true
+			}
+			for _, term := range parseQuery(pdoc.Name) {
+				terms[term] = true
+			}
+		}
+
+		// Synopsis
+
+		collectSynopsisTerms(terms, pdoc.Synopsis)
+
+	}
+
+	return termSlice(terms)
+}
+
+// vendorPat matches the path of a vendored package.
+var vendorPat = regexp.MustCompile(
+	// match directories used by tools to vendor packages.
+	`/(?:_?third_party|vendors|Godeps/_workspace/src)/` +
+		// match a domain name.
+		`[^./]+\.[^/]+`)
+
+func documentScore(pdoc *doc.Package) float64 {
+	if pdoc.Name == "" ||
+		pdoc.Status != gosrc.Active ||
+		len(pdoc.Errors) > 0 ||
+		strings.HasSuffix(pdoc.ImportPath, ".go") ||
+		strings.HasPrefix(pdoc.ImportPath, "gist.github.com/") ||
+		strings.HasSuffix(pdoc.ImportPath, "/internal") ||
+		strings.Contains(pdoc.ImportPath, "/internal/") ||
+		vendorPat.MatchString(pdoc.ImportPath) {
+		return 0
+	}
+
+	for _, p := range pdoc.Imports {
+		if strings.HasSuffix(p, ".go") {
+			return 0
+		}
+	}
+
+	r := 1.0
+	if pdoc.IsCmd {
+		if pdoc.Doc == "" {
+			// Do not include command in index if it does not have documentation.
+			return 0
+		}
+		if !importsGoPackages(pdoc) {
+			// Penalize commands that don't use the "go/*" packages.
+			r *= 0.9
+		}
+	} else {
+		if !pdoc.Truncated &&
+			len(pdoc.Consts) == 0 &&
+			len(pdoc.Vars) == 0 &&
+			len(pdoc.Funcs) == 0 &&
+			len(pdoc.Types) == 0 &&
+			len(pdoc.Examples) == 0 {
+			// Do not include package in index if it does not have exports.
+			return 0
+		}
+		if pdoc.Doc == "" {
+			// Penalty for no documentation.
+			r *= 0.95
+		}
+		if path.Base(pdoc.ImportPath) != pdoc.Name {
+			// Penalty for last element of path != package name.
+			r *= 0.9
+		}
+		for i := 0; i < strings.Count(pdoc.ImportPath[len(pdoc.ProjectRoot):], "/"); i++ {
+			// Penalty for deeply nested packages.
+			r *= 0.99
+		}
+		if strings.Index(pdoc.ImportPath[len(pdoc.ProjectRoot):], "/src/") > 0 {
+			r *= 0.95
+		}
+		for _, p := range pdoc.Imports {
+			if vendorPat.MatchString(p) {
+				// Penalize packages that import vendored packages.
+				r *= 0.1
+				break
+			}
+		}
+	}
+	return r
+}
+
+func parseQuery(q string) []string {
+	var terms []string
+	q = strings.ToLower(q)
+	for _, s := range strings.FieldsFunc(q, isTermSep) {
+		if !stopWord[s] {
+			terms = append(terms, term(s))
+		}
+	}
+	return terms
+}
+
+func importsGoPackages(pdoc *doc.Package) bool {
+	for _, m := range pdoc.Imports {
+		if strings.HasPrefix(m, "go/") {
+			return true
+		}
+	}
+	return false
+}