move: log parsers

logparser/functional/Makefile

@@ -0,0 +1,7 @@
+SHELL := /bin/bash
+
+r: 
+	go run . < ../logs/log.txt
+
+t: 
+	time go run . < ../logs/log.txt

logparser/functional/chartwriter.go

@@ -0,0 +1,38 @@
+package main
+
+// func chartWriter(w io.Writer) outputFn {
+// 	return func(res []result) error {
+// 		return chartWrite(w, res)
+// 	}
+// }
+
+// func chartWrite(w io.Writer, res []result) error {
+// 	sort.Slice(res, func(i, j int) bool {
+// 		return res[i].domain > res[j].domain
+// 	})
+
+// 	donut := chart.DonutChart{
+// 		Title: "Total Visits Per Domain",
+// 		TitleStyle: chart.Style{
+// 			FontSize:  35,
+// 			Show:      true,
+// 			FontColor: chart.ColorAlternateGreen,
+// 		},
+// 		Width:  1920,
+// 		Height: 800,
+// 	}
+
+// 	for _, r := range res {
+// 		v := chart.Value{
+// 			Label: r.domain + r.page + ": " + strconv.Itoa(r.visits),
+// 			Value: float64(r.visits),
+// 			Style: chart.Style{
+// 				FontSize: 14,
+// 			},
+// 		}
+
+// 		donut.Values = append(donut.Values, v)
+// 	}
+
+// 	return donut.Render(chart.SVG, w)
+// }

logparser/functional/field.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+// field helps for field parsing
+type field struct{ err error }
+
+// uatoi parses an unsigned integer string and saves the error.
+// it assumes that the val is unsigned.
+// for ease of usability: it returns an int instead of uint.
+func (f *field) uatoi(name, val string) int {
+	n, err := strconv.Atoi(val)
+	if err != nil || n < 0 {
+		f.err = fmt.Errorf("incorrect field -> %q = %q", name, val)
+	}
+	return n
+}
+
+func atoi(input []byte) (int, error) {
+	val := 0
+	for i := 0; i < len(input); i++ {
+		char := input[i]
+		if char < '0' || char > '9' {
+			return 0, errors.New("invalid number")
+		}
+		val = val*10 + int(char) - '0'
+	}
+	return val, nil
+}

logparser/functional/filters.go

@@ -0,0 +1,34 @@
+package main
+
+import "strings"
+
+func noopFilter(r result) bool {
+	return true
+}
+
+func notUsing(filter filterFn) filterFn {
+	return func(r result) bool {
+		return !filter(r)
+	}
+}
+
+func domainExtFilter(domains ...string) filterFn {
+	return func(r result) bool {
+		for _, domain := range domains {
+			if strings.HasSuffix(r.domain, "."+domain) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func domainFilter(domain string) filterFn {
+	return func(r result) bool {
+		return strings.Contains(r.domain, domain)
+	}
+}
+
+func orgDomainsFilter(r result) bool {
+	return strings.HasSuffix(r.domain, ".org")
+}

logparser/functional/groupers.go

@@ -0,0 +1,20 @@
+package main
+
+// domainGrouper groups by domain.
+// but it keeps the other fields.
+// for example: it returns pages as well, but you shouldn't use them.
+// exercise: write a function that erases superfluous data.
+func domainGrouper(r result) string {
+	return r.domain
+}
+
+func pageGrouper(r result) string {
+	return r.domain + r.page
+}
+
+// groupBy allocates map unnecessarily
+func noopGrouper(r result) string {
+	// with something like:
+	// return randomStrings()
+	return ""
+}

logparser/functional/main.go

@@ -0,0 +1,44 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	p := pipeline{
+		read:   textReader(os.Stdin),
+		write:  textWriter(os.Stdout),
+		filter: notUsing(domainExtFilter("io", "com")),
+		group:  domainGrouper,
+	}
+
+	// var p pipeline
+	// p.
+	// 	filterBy(notUsing(domainExtFilter("io", "com"))).
+	// 	groupBy(domainGrouper)
+
+	if err := p.start(); err != nil {
+		fmt.Println("> Err:", err)
+	}
+}
+
+// []outputter{textFile("results.txt"), chartFile("graph.png")}
+
+// func outputs(w io.Writer) outputFn {
+// 	tw := textWriter(w)
+// 	cw := chartWriter(w)
+
+// 	return func(rs []result) error {
+// 		err := tw(rs)
+// 		err = cw(rs)
+// 		return err
+// 	}
+// }

logparser/functional/pipeline.go

@@ -0,0 +1,78 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import "os"
+
+type (
+	processFn func(r result)
+	inputFn   func(processFn) error
+	outputFn  func([]result) error
+	filterFn  func(result) (include bool)
+	groupFn   func(result) (key string)
+)
+
+type pipeline struct {
+	read   inputFn
+	write  outputFn
+	filter filterFn
+	group  groupFn
+}
+
+func (p *pipeline) filterBy(f filterFn) *pipeline { p.filter = f; return p }
+func (p *pipeline) groupBy(f groupFn) *pipeline   { p.group = f; return p }
+func (p *pipeline) from(f inputFn) *pipeline      { p.read = f; return p }
+func (p *pipeline) to(f outputFn) *pipeline       { p.write = f; return p }
+
+func (p *pipeline) defaults() {
+	if p.filter == nil {
+		p.filter = noopFilter
+	}
+
+	if p.group == nil {
+		p.group = domainGrouper
+	}
+
+	if p.read == nil {
+		p.read = textReader(os.Stdin)
+	}
+
+	if p.write == nil {
+		p.write = textWriter(os.Stdout)
+	}
+}
+
+func (p *pipeline) start() error {
+	p.defaults()
+
+	// retrieve and process the lines
+	sum := make(map[string]result)
+
+	process := func(r result) {
+		if !p.filter(r) {
+			return
+		}
+
+		k := p.group(r)
+		sum[k] = r.add(sum[k])
+	}
+
+	// return err from input reader
+	if err := p.read(process); err != nil {
+		return err
+	}
+
+	// prepare the results for outputting
+	var out []result
+	for _, res := range sum {
+		out = append(out, res)
+	}
+
+	// return err from output reader
+	return p.write(out)
+}

logparser/functional/result.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+const fieldsLength = 4
+
+// result stores the parsed result for a domain
+type result struct {
+	domain  string
+	page    string
+	visits  int
+	uniques int
+}
+
+// add adds the metrics of another result
+func (r result) add(other result) result {
+	r.visits += other.visits
+	r.uniques += other.uniques
+	return r
+}
+
+// parseFields parses and returns the parsing result
+func parseFields(line string) (r result, err error) {
+	fields := strings.Fields(line)
+
+	if len(fields) != fieldsLength {
+		return r, fmt.Errorf("wrong number of fields %q", fields)
+	}
+
+	r.domain = fields[0]
+	r.page = fields[1]
+
+	f := new(field)
+	r.visits = f.uatoi("visits", fields[2])
+	r.uniques = f.uatoi("uniques", fields[3])
+
+	return r, f.err
+}
+
+func fastParseFields(data []byte) (res result, err error) {
+	const separator = ' '
+
+	var findex int
+
+	for i, j := 0, 0; i < len(data); i++ {
+		c := data[i]
+		last := len(data) == i+1
+
+		if c != separator && !last {
+			continue
+		}
+
+		if last {
+			i = len(data)
+		}
+
+		switch fval := data[j:i]; findex {
+		case 0:
+			res.domain = string(fval)
+		case 1:
+			res.page = string(fval)
+		case 2:
+			res.visits, err = atoi(fval)
+		case 3:
+			res.uniques, err = atoi(fval)
+		}
+
+		if err != nil {
+			return res, err
+		}
+
+		j = i + 1
+		findex++
+	}
+
+	if findex != fieldsLength {
+		err = fmt.Errorf("wrong number of fields %q", data)
+	}
+	return res, err
+}

logparser/functional/textreader.go

@@ -0,0 +1,39 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+)
+
+func textReader(r io.Reader) inputFn {
+	return func(process processFn) error {
+		var (
+			l  = 1
+			in = bufio.NewScanner(r)
+		)
+
+		for in.Scan() {
+			r, err := fastParseFields(in.Bytes())
+			// r, err := parseFields(in.Text())
+			if err != nil {
+				return fmt.Errorf("line %d: %v", l, err)
+			}
+
+			process(r)
+			l++
+		}
+
+		if c, ok := r.(io.Closer); ok {
+			c.Close()
+		}
+		return in.Err()
+	}
+}

logparser/functional/textwriter.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+)
+
+// TODO: sort by result key interfaces section
+
+const (
+
+	//          DOMAINS PAGES VISITS UNIQUES
+	//             ^      ^     ^    ^
+	//             |      |     |    |
+	header     = "%-25s %-10s %10s %10s\n"
+	line       = "%-25s %-10s %10d %10d\n"
+	footer     = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
+	dash       = "-"
+	dashLength = 58
+)
+
+func textWriter(w io.Writer) outputFn {
+	return func(res []result) error {
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].domain > res[j].domain
+		})
+
+		var total result
+
+		fmt.Fprintf(w, header, "DOMAINS", "PAGES", "VISITS", "UNIQUES")
+		fmt.Fprintf(w, strings.Repeat(dash, dashLength)+"\n")
+
+		for _, r := range res {
+			total = total.add(r)
+			fmt.Fprintf(w, line, r.domain, r.page, r.visits, r.uniques)
+		}
+
+		fmt.Fprintf(w, footer, "", total.visits, total.uniques)
+
+		return nil
+	}
+}
+
+func noWhere() outputFn {
+	return func(res []result) error {
+		return nil
+	}
+}