refactor: logparser v5 to pkgs

logparser/v5/Makefile

@@ -0,0 +1,5 @@
+r: 
+	go run . < ../../logs/log.txt
+
+t: 
+	time go run . < ../../logs/log.txt

logparser/v5/filepipe.go

@@ -0,0 +1,39 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import (
+	"os"
+	"strings"
+
+	"github.com/inancgumus/learngo/logparser/v5/pipe"
+)
+
+// fromFile generates a default pipeline.
+// Detects the correct parser by the file extension.
+// Uses a TextReport and groups by domain.
+func fromFile(path string) (*pipe.Pipeline, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var src pipe.Iterator
+	switch {
+	case strings.HasSuffix(path, ".txt"):
+		src = pipe.NewTextLog(f)
+	case strings.HasSuffix(path, ".jsonl"):
+		src = pipe.NewJSONLog(f)
+	}
+
+	return pipe.New(
+		src,
+		pipe.NewTextReport(os.Stdout),
+		pipe.GroupBy(pipe.DomainGrouper),
+	), nil
+}

logparser/v5/main.go

@@ -0,0 +1,35 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/inancgumus/learngo/logparser/v5/pipe"
+)
+
+func main() {
+	// p := pipe.Default(
+	// 	os.Stdin, os.Stdout,
+	// 	pipe.FilterBy(pipe.DomainExtFilter("com", "io")),
+	// 	pipe.GroupBy(pipe.DomainGrouper),
+	// )
+
+	p := pipe.New(
+		pipe.NewTextLog(os.Stdin),
+		pipe.NewTextReport(os.Stdout),
+		// pipe.NewJSONReport(os.Stdout),
+		pipe.FilterBy(pipe.DomainExtFilter("com", "io")),
+		pipe.GroupBy(pipe.DomainGrouper),
+	)
+
+	if err := p.Run(); err != nil {
+		log.Fatalln(err)
+	}
+}

logparser/v5/pipe/chartreport.go

@@ -0,0 +1,54 @@
+package pipe
+
+/*
+// You need to run:
+// go get -u github.com/wcharczuk/go-chart
+
+// Chart renders a chart.
+type Chart struct {
+	Title         string
+	Width, Height int
+
+	w io.Writer
+}
+
+// NewChartReport returns a Chart report generator.
+func NewChartReport(w io.Writer) *Chart {
+	return &Chart{w: w}
+}
+
+// Consume generates a chart report.
+func (c *Chart) Consume(records Iterator) error {
+	w := os.Stdout
+
+	donut := chart.DonutChart{
+		Title: c.Title,
+		TitleStyle: chart.Style{
+			FontSize:  35,
+			Show:      true,
+			FontColor: chart.ColorAlternateGreen,
+		},
+		Width:  c.Width,
+		Height: c.Height,
+	}
+
+	err := records.Each(func(r Record) error {
+		v := chart.Value{
+			Label: r.Domain + r.Page + ": " + strconv.Itoa(r.Visits),
+			Value: float64(r.Visits),
+			Style: chart.Style{
+				FontSize: 14,
+			},
+		}
+
+		donut.Values = append(donut.Values, v)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return donut.Render(chart.SVG, w)
+}
+*/

logparser/v5/pipe/close.go

@@ -0,0 +1,19 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"io"
+)
+
+// readClose the reader if it's a io.Closer.
+func readClose(r io.Reader) {
+	if rc, ok := r.(io.Closer); ok {
+		rc.Close()
+	}
+}

logparser/v5/pipe/filter.go

@@ -0,0 +1,48 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+// FilterFunc represents a filtering pipeline func.
+type FilterFunc = func(Record) (pass bool)
+
+// Filter the records.
+type Filter struct {
+	src     Iterator
+	filters []FilterFunc
+}
+
+// FilterBy returns a new filter pipeline.
+func FilterBy(fn ...FilterFunc) *Filter {
+	return &Filter{filters: fn}
+}
+
+// Consume saves the iterator for later processing.
+func (f *Filter) Consume(records Iterator) error {
+	f.src = records
+	return nil
+}
+
+// Each yields only the filtered records.
+func (f *Filter) Each(yield func(Record) error) error {
+	return f.src.Each(func(r Record) error {
+		if !f.check(r) {
+			return nil
+		}
+		return yield(r)
+	})
+}
+
+// check all the filters against the record.
+func (f *Filter) check(r Record) bool {
+	for _, fi := range f.filters {
+		if !fi(r) {
+			return false
+		}
+	}
+	return true
+}

logparser/v5/pipe/filters.go

@@ -0,0 +1,41 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import "strings"
+
+// NotFilter reverses a filter. True becomes false, and vice versa.
+func NotFilter(filter FilterFunc) FilterFunc {
+	return func(r Record) bool {
+		return !filter(r)
+	}
+}
+
+// DomainExtFilter filters a set of domain extensions.
+func DomainExtFilter(domains ...string) FilterFunc {
+	return func(r Record) bool {
+		for _, domain := range domains {
+			if strings.HasSuffix(r.Domain, "."+domain) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+// DomainFilter filters a domain if it contains the given text.
+func DomainFilter(text string) FilterFunc {
+	return func(r Record) bool {
+		return strings.Contains(r.Domain, text)
+	}
+}
+
+// DomainOrgFilter filters only the ".org" domains.
+func DomainOrgFilter(r Record) bool {
+	return strings.HasSuffix(r.Domain, ".org")
+}

logparser/v5/pipe/group.go

@@ -0,0 +1,60 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"sort"
+)
+
+// GroupFunc represents a grouping func that returns a grouping key.
+type GroupFunc = func(Record) (key string)
+
+// Group records by a key.
+type Group struct {
+	sum  map[string]Record // metrics per group key
+	keys []string          // unique group keys
+	key  GroupFunc
+}
+
+// GroupBy returns a new Group.
+// It takes a group func that returns a group key.
+// The returned group will group the record using the key.
+func GroupBy(key GroupFunc) *Group {
+	return &Group{
+		sum: make(map[string]Record),
+		key: key,
+	}
+}
+
+// Consume records for grouping.
+func (g *Group) Consume(records Iterator) error {
+	return records.Each(func(r Record) error {
+		k := g.key(r)
+
+		if _, ok := g.sum[k]; !ok {
+			g.keys = append(g.keys, k)
+		}
+
+		g.sum[k] = r.Sum(g.sum[k])
+
+		return nil
+	})
+}
+
+// Each sorts and yields the grouped records.
+func (g *Group) Each(yield func(Record) error) error {
+	sort.Strings(g.keys)
+
+	for _, k := range g.keys {
+		if err := yield(g.sum[k]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

logparser/v5/pipe/groupers.go

@@ -0,0 +1,21 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+// DomainGrouper groups the records by domain.
+// It keeps the other fields intact.
+// For example: It returns the page field as well.
+// Exercise: Write a solution that removes the unnecessary data.
+func DomainGrouper(r Record) string {
+	return r.Domain
+}
+
+// Page groups records by page.
+func Page(r Record) string {
+	return r.Domain + r.Page
+}

logparser/v5/pipe/jsonlog.go

@@ -0,0 +1,47 @@
+// For more tutorials: https://bj.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"encoding/json"
+	"io"
+)
+
+// JSON parses json records.
+type JSON struct {
+	reader io.Reader
+}
+
+// NewJSONLog creates a json parser.
+func NewJSONLog(r io.Reader) *JSON {
+	return &JSON{reader: r}
+}
+
+// Each yields records from a json reader.
+func (j *JSON) Each(yield func(Record) error) error {
+	defer readClose(j.reader)
+
+	dec := json.NewDecoder(j.reader)
+
+	for {
+		var r Record
+
+		err := dec.Decode(&r)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if err := yield(r); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

logparser/v5/pipe/jsonreport.go

@@ -0,0 +1,32 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"encoding/json"
+	"io"
+)
+
+// JSONReport generates a JSON report.
+type JSONReport struct {
+	w io.Writer
+}
+
+// NewJSONReport returns a JSON report generator.
+func NewJSONReport(w io.Writer) *JSONReport {
+	return &JSONReport{w: w}
+}
+
+// Consume generates a JSON report.
+func (t *JSONReport) Consume(records Iterator) error {
+	enc := json.NewEncoder(t.w)
+
+	return records.Each(func(r Record) error {
+		return enc.Encode(r)
+	})
+}

logparser/v5/pipe/logcount.go

@@ -0,0 +1,37 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import "fmt"
+
+// logCount counts the yielded records.
+type logCount struct {
+	Iterator
+	n int
+}
+
+// Each yields to the inner iterator while counting the records.
+// Reports the record number on an error.
+func (lc *logCount) Each(yield func(Record) error) error {
+	err := lc.Iterator.Each(func(r Record) error {
+		lc.n++
+		return yield(r)
+	})
+
+	if err != nil {
+		// lc.n+1: iterator.each won't call yield on err
+		return fmt.Errorf("record %d: %v", lc.n+1, err)
+	}
+
+	return nil
+}
+
+// count returns the last read record number.
+func (lc *logCount) count() int {
+	return lc.n
+}

logparser/v5/pipe/pipe.go

@@ -0,0 +1,29 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+// YieldFunc yields a record from an Iterator to up-stream (Consumer).
+type YieldFunc = func(Record) error
+
+// Iterator yields a record.
+type Iterator interface {
+	Each(YieldFunc) error
+}
+
+// Consumer consumes records from an iterator.
+type Consumer interface {
+	Consume(Iterator) error
+}
+
+// Transform represents both a record consumer and producer.
+// It has an input and output.
+// It takes a single record and provides an iterator for all the records.
+type Transform interface {
+	Iterator // producer: should never return on yield().err == nil
+	Consumer
+}

logparser/v5/pipe/pipeline.go

@@ -0,0 +1,54 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+// Pipeline takes records from a source, transforms, and sends them to a destionation.
+type Pipeline struct {
+	src   Iterator
+	trans []Transform
+	dst   Consumer
+}
+
+// New creates a new pipeline.
+func New(src Iterator, dst Consumer, t ...Transform) *Pipeline {
+	return &Pipeline{
+		src:   &logCount{Iterator: src},
+		dst:   dst,
+		trans: t,
+	}
+}
+
+// Default creates a pipeline that reads from a text log and generates a text report.
+func Default(r io.Reader, w io.Writer, t ...Transform) *Pipeline {
+	return New(NewTextLog(r), NewTextReport(w), t...)
+}
+
+// Run the pipeline.
+func (p *Pipeline) Run() error {
+	defer func() {
+		n := p.src.(*logCount).count()
+		fmt.Fprintf(os.Stderr, "%d records processed.\n", n)
+	}()
+
+	last := p.src
+
+	for _, t := range p.trans {
+		if err := t.Consume(last); err != nil {
+			return err
+		}
+		last = t
+	}
+
+	return p.dst.Consume(last)
+}

logparser/v5/pipe/record.go

@@ -0,0 +1,85 @@
+package pipe
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const fieldsLength = 4
+
+// Record stores fields of a log line.
+type Record struct {
+	Domain  string
+	Page    string
+	Visits  int
+	Uniques int
+}
+
+// Sum the numeric fields with another record.
+func (r Record) Sum(other Record) Record {
+	r.Visits += other.Visits
+	r.Uniques += other.Uniques
+	return r
+}
+
+// UnmarshalText to a *Record.
+func (r *Record) UnmarshalText(p []byte) (err error) {
+	fields := strings.Fields(string(p))
+	if len(fields) != fieldsLength {
+		return fmt.Errorf("wrong number of fields %q", fields)
+	}
+
+	r.Domain, r.Page = fields[0], fields[1]
+
+	if r.Visits, err = parseStr("visits", fields[2]); err != nil {
+		return err
+	}
+	if r.Uniques, err = parseStr("uniques", fields[3]); err != nil {
+		return err
+	}
+	return validate(*r)
+}
+
+// UnmarshalJSON to a *Record.
+func (r *Record) UnmarshalJSON(data []byte) error {
+	// `methodless` doesn't have any methods including UnmarshalJSON.
+	// This trick prevents the stack-overflow (infinite loop).
+	type methodless Record
+
+	var m methodless
+	if err := json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+
+	// Cast back to the Record and save.
+	*r = Record(m)
+
+	return validate(*r)
+}
+
+// parseStr helps UnmarshalText for string to positive int parsing.
+func parseStr(name, v string) (int, error) {
+	n, err := strconv.Atoi(v)
+	if err != nil {
+		return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err)
+	}
+	return n, nil
+}
+
+// validate whether a parsed record is valid or not.
+func validate(r Record) (err error) {
+	switch {
+	case r.Domain == "":
+		err = errors.New("record.domain cannot be empty")
+	case r.Page == "":
+		err = errors.New("record.page cannot be empty")
+	case r.Visits < 0:
+		err = errors.New("record.visits cannot be negative")
+	case r.Uniques < 0:
+		err = errors.New("record.uniques cannot be negative")
+	}
+	return
+}

logparser/v5/pipe/textlog.go

@@ -0,0 +1,44 @@
+// For more tutorials: https://bp.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"bufio"
+	"io"
+)
+
+// TextLog parses text based log lines.
+type TextLog struct {
+	reader io.Reader
+}
+
+// NewTextLog creates a text parser.
+func NewTextLog(r io.Reader) *TextLog {
+	return &TextLog{reader: r}
+}
+
+// Each yields records from a text log.
+func (p *TextLog) Each(yield func(Record) error) error {
+	defer readClose(p.reader)
+
+	in := bufio.NewScanner(p.reader)
+
+	for in.Scan() {
+		r := new(Record)
+
+		if err := r.UnmarshalText(in.Bytes()); err != nil {
+			return err
+		}
+
+		if err := yield(*r); err != nil {
+			return err
+		}
+	}
+
+	return in.Err()
+}

logparser/v5/pipe/textreport.go

@@ -0,0 +1,65 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package pipe
+
+import (
+	"fmt"
+	"io"
+	"text/tabwriter"
+)
+
+const (
+	minWidth = 0
+	tabWidth = 4
+	padding  = 4
+	flags    = 0
+)
+
+// TextReport report generator.
+type TextReport struct {
+	w io.Writer
+}
+
+// NewTextReport returns a TextReport report generator.
+func NewTextReport(w io.Writer) *TextReport {
+	return &TextReport{w: w}
+}
+
+// Consume generates a text report.
+func (t *TextReport) Consume(records Iterator) error {
+	w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags)
+
+	write := fmt.Fprintf
+
+	write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
+	write(w, "-------\t-----\t------\t-------\n")
+
+	var total Record
+
+	err := records.Each(func(r Record) error {
+		total = r.Sum(total)
+
+		write(w, "%s\t%s\t%d\t%d\n",
+			r.Domain, r.Page,
+			r.Visits, r.Uniques,
+		)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	write(w, "\t\t\t\n")
+	write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "",
+		total.Visits,
+		total.Uniques,
+	)
+
+	return w.Flush()
+}