From 81b424697320b71cad096cff61bb7ce4185116c1 Mon Sep 17 00:00:00 2001 From: Inanc Gumus Date: Wed, 28 Aug 2019 23:46:42 +0300 Subject: [PATCH] refactor: logparser v5 to pkgs --- logparser/oop/chartreport.go | 38 --------- logparser/oop/filter.go | 43 ---------- logparser/oop/filters.go | 36 -------- logparser/oop/group.go | 49 ----------- logparser/oop/groupers.go | 15 ---- logparser/oop/main.go | 44 ---------- logparser/oop/pipeline.go | 78 ----------------- logparser/oop/record.go | 82 ------------------ logparser/oop/textreport.go | 49 ----------- logparser/{oop => v5}/Makefile | 0 logparser/v5/filepipe.go | 39 +++++++++ logparser/v5/main.go | 35 ++++++++ logparser/v5/pipe/chartreport.go | 54 ++++++++++++ .../{oop/readclose.go => v5/pipe/close.go} | 3 +- logparser/v5/pipe/filter.go | 48 +++++++++++ logparser/v5/pipe/filters.go | 41 +++++++++ logparser/v5/pipe/group.go | 60 +++++++++++++ logparser/v5/pipe/groupers.go | 21 +++++ logparser/{oop => v5/pipe}/jsonlog.go | 24 +++--- logparser/v5/pipe/jsonreport.go | 32 +++++++ logparser/{oop => v5/pipe}/logcount.go | 16 ++-- logparser/v5/pipe/pipe.go | 29 +++++++ logparser/v5/pipe/pipeline.go | 54 ++++++++++++ logparser/v5/pipe/record.go | 85 +++++++++++++++++++ logparser/{oop => v5/pipe}/textlog.go | 19 +++-- logparser/v5/pipe/textreport.go | 65 ++++++++++++++ 26 files changed, 601 insertions(+), 458 deletions(-) delete mode 100644 logparser/oop/chartreport.go delete mode 100644 logparser/oop/filter.go delete mode 100644 logparser/oop/filters.go delete mode 100644 logparser/oop/group.go delete mode 100644 logparser/oop/groupers.go delete mode 100644 logparser/oop/main.go delete mode 100644 logparser/oop/pipeline.go delete mode 100644 logparser/oop/record.go delete mode 100644 logparser/oop/textreport.go rename logparser/{oop => v5}/Makefile (100%) create mode 100644 logparser/v5/filepipe.go create mode 100644 logparser/v5/main.go create mode 100644 logparser/v5/pipe/chartreport.go rename logparser/{oop/readclose.go => v5/pipe/close.go} (83%) create mode 100644 logparser/v5/pipe/filter.go create mode 100644 logparser/v5/pipe/filters.go create mode 100644 logparser/v5/pipe/group.go create mode 100644 logparser/v5/pipe/groupers.go rename logparser/{oop => v5/pipe}/jsonlog.go (52%) create mode 100644 logparser/v5/pipe/jsonreport.go rename logparser/{oop => v5/pipe}/logcount.go (56%) create mode 100644 logparser/v5/pipe/pipe.go create mode 100644 logparser/v5/pipe/pipeline.go create mode 100644 logparser/v5/pipe/record.go rename logparser/{oop => v5/pipe}/textlog.go (54%) create mode 100644 logparser/v5/pipe/textreport.go diff --git a/logparser/oop/chartreport.go b/logparser/oop/chartreport.go deleted file mode 100644 index a77ba78..0000000 --- a/logparser/oop/chartreport.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -// You need to run: -// go get -u github.com/wcharczuk/go-chart - -// type chartReport struct { -// title string -// width, height int -// } - -// func (s *chartReport) digest(records iterator) error { -// w := os.Stdout - -// donut := chart.DonutChart{ -// Title: s.title, -// TitleStyle: chart.Style{ -// FontSize: 35, -// Show: true, -// FontColor: chart.ColorAlternateGreen, -// }, -// Width: s.width, -// Height: s.height, -// } - -// records.each(func(r record) { -// v := chart.Value{ -// Label: r.domain + r.page + ": " + strconv.Itoa(r.visits), -// Value: float64(r.visits), -// Style: chart.Style{ -// FontSize: 14, -// }, -// } - -// donut.Values = append(donut.Values, v) -// }) - -// return donut.Render(chart.SVG, w) -// } diff --git a/logparser/oop/filter.go b/logparser/oop/filter.go deleted file mode 100644 index 031a556..0000000 --- a/logparser/oop/filter.go +++ /dev/null @@ -1,43 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -type filter struct { - src iterator - filters []filterFunc -} - -func filterBy(fn ...filterFunc) *filter { - return &filter{filters: fn} -} - -// transform the record -func (f *filter) digest(records iterator) error { - f.src = records - return nil -} - -// each yields only the filtered records -func (f *filter) each(yield recordFn) error { - return f.src.each(func(r record) { - if !f.check(r) { - return - } - yield(r) - }) -} - -// check all the filters against the record -func (f *filter) check(r record) bool { - for _, fi := range f.filters { - if !fi(r) { - return false - } - } - return true -} diff --git a/logparser/oop/filters.go b/logparser/oop/filters.go deleted file mode 100644 index ad00098..0000000 --- a/logparser/oop/filters.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import "strings" - -type filterFunc func(record) bool - -func noopFilter(r record) bool { - return true -} - -func notUsing(filter filterFunc) filterFunc { - return func(r record) bool { - return !filter(r) - } -} - -func domainExtFilter(domains ...string) filterFunc { - return func(r record) bool { - for _, domain := range domains { - if strings.HasSuffix(r.domain, "."+domain) { - return true - } - } - return false - } -} - -func domainFilter(domain string) filterFunc { - return func(r record) bool { - return strings.Contains(r.domain, domain) - } -} - -func orgDomainsFilter(r record) bool { - return strings.HasSuffix(r.domain, ".org") -} diff --git a/logparser/oop/group.go b/logparser/oop/group.go deleted file mode 100644 index 5d57955..0000000 --- a/logparser/oop/group.go +++ /dev/null @@ -1,49 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "sort" -) - -type group struct { - sum map[string]record // metrics per group key - keys []string // unique group keys - key groupFunc -} - -func groupBy(key groupFunc) *group { - return &group{ - sum: make(map[string]record), - key: key, - } -} - -// digest the records -func (g *group) digest(records iterator) error { - return records.each(func(r record) { - k := g.key(r) - - if _, ok := g.sum[k]; !ok { - g.keys = append(g.keys, k) - } - - g.sum[k] = r.sum(g.sum[k]) - }) -} - -// each yields the grouped records -func (g *group) each(yield recordFn) error { - sort.Strings(g.keys) - - for _, k := range g.keys { - yield(g.sum[k]) - } - - return nil -} diff --git a/logparser/oop/groupers.go b/logparser/oop/groupers.go deleted file mode 100644 index 79980e7..0000000 --- a/logparser/oop/groupers.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -type groupFunc func(record) string - -// domainGrouper groups by domain. -// but it keeps the other fields. -// for example: it returns pages as well, but you shouldn't use them. -// exercise: write a function that erases the unnecessary data. -func domainGrouper(r record) string { - return r.domain -} - -func pageGrouper(r record) string { - return r.domain + r.page -} diff --git a/logparser/oop/main.go b/logparser/oop/main.go deleted file mode 100644 index 285fdbb..0000000 --- a/logparser/oop/main.go +++ /dev/null @@ -1,44 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "log" - "os" -) - -func main() { - // newGrouper(domainGrouper) - - // s := &chartReport{ - // title: "visits per domain", - // width: 1920, - // height: 800, - // } - - // pipe, err := fromFile("../logs/log.jsonl") - // if err != nil { - // log.Fatalln(err) - // } - - pipe := newPipeline( - newTextLog(os.Stdin), - // newJSONLog(os.Stdin), - newTextReport(), - filterBy(notUsing(domainExtFilter("com", "io"))), - groupBy(domainGrouper), - ) - - if err := pipe.run(); err != nil { - log.Fatalln(err) - } - - // if err := reportFromFile(os.Args[1]); err != nil { - // log.Fatalln(err) - // } -} diff --git a/logparser/oop/pipeline.go b/logparser/oop/pipeline.go deleted file mode 100644 index f2d2f7c..0000000 --- a/logparser/oop/pipeline.go +++ /dev/null @@ -1,78 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "fmt" - "os" - "strings" -) - -type recordFn func(record) - -type iterator interface{ each(recordFn) error } -type digester interface{ digest(iterator) error } - -type transform interface { - digester - iterator -} - -type pipeline struct { - src iterator - trans []transform - dst digester -} - -func (p *pipeline) run() error { - defer func() { - n := p.src.(*logCount).count() - fmt.Printf("%d records processed.\n", n) - }() - - last := p.src - - for _, t := range p.trans { - if err := t.digest(last); err != nil { - return err - } - last = t - } - - return p.dst.digest(last) -} - -func newPipeline(src iterator, dst digester, t ...transform) *pipeline { - return &pipeline{ - src: &logCount{iterator: src}, - dst: dst, - trans: t, - } -} - -// fromFile generates a default report -func fromFile(path string) (*pipeline, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - var src iterator - switch { - case strings.HasSuffix(path, ".txt"): - src = newTextLog(f) - case strings.HasSuffix(path, ".jsonl"): - src = newJSONLog(f) - } - - return newPipeline( - src, - newTextReport(), - groupBy(domainGrouper), - ), nil -} diff --git a/logparser/oop/record.go b/logparser/oop/record.go deleted file mode 100644 index 80b020f..0000000 --- a/logparser/oop/record.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" -) - -const fieldsLength = 4 - -type record struct { - domain string - page string - visits int - uniques int -} - -func (r record) sum(other record) record { - r.visits += other.visits - r.uniques += other.uniques - return r -} - -// UnmarshalText to a *record -func (r *record) UnmarshalText(p []byte) (err error) { - fields := strings.Fields(string(p)) - if len(fields) != fieldsLength { - return fmt.Errorf("wrong number of fields %q", fields) - } - - r.domain, r.page = fields[0], fields[1] - - if r.visits, err = parseStr("visits", fields[2]); err != nil { - return err - } - if r.uniques, err = parseStr("uniques", fields[3]); err != nil { - return err - } - return validate(*r) -} - -// UnmarshalJSON to a *record -func (r *record) UnmarshalJSON(data []byte) error { - var re struct { - Domain string - Page string - Visits int - Uniques int - } - - if err := json.Unmarshal(data, &re); err != nil { - return err - } - - *r = record{re.Domain, re.Page, re.Visits, re.Uniques} - return validate(*r) -} - -// parseStr helps UnmarshalText for string to positive int parsing -func parseStr(name, v string) (int, error) { - n, err := strconv.Atoi(v) - if err != nil { - return 0, fmt.Errorf("record.UnmarshalText %q: %v", name, err) - } - return n, nil -} - -func validate(r record) (err error) { - switch { - case r.domain == "": - err = errors.New("record.domain cannot be empty") - case r.page == "": - err = errors.New("record.page cannot be empty") - case r.visits < 0: - err = errors.New("record.visits cannot be negative") - case r.uniques < 0: - err = errors.New("record.uniques cannot be negative") - } - return -} diff --git a/logparser/oop/textreport.go b/logparser/oop/textreport.go deleted file mode 100644 index b0c42de..0000000 --- a/logparser/oop/textreport.go +++ /dev/null @@ -1,49 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "fmt" - "os" - "text/tabwriter" -) - -// TODO: make this configurable? or exercise? -const ( - minWidth = 0 - tabWidth = 4 - padding = 4 - flags = 0 -) - -type textReport struct{} - -func newTextReport() *textReport { - return new(textReport) -} - -func (s *textReport) digest(records iterator) error { - w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, ' ', flags) - - write := fmt.Fprintf - - write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") - write(w, "-------\t-----\t------\t-------\n") - - var total record - records.each(func(r record) { - total = total.sum(r) - - write(w, "%s\t%s\t%d\t%d\n", r.domain, r.page, r.visits, r.uniques) - }) - - write(w, "\t\t\t\n") - write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", total.visits, total.uniques) - - return w.Flush() -} diff --git a/logparser/oop/Makefile b/logparser/v5/Makefile similarity index 100% rename from logparser/oop/Makefile rename to logparser/v5/Makefile diff --git a/logparser/v5/filepipe.go b/logparser/v5/filepipe.go new file mode 100644 index 0000000..63604f5 --- /dev/null +++ b/logparser/v5/filepipe.go @@ -0,0 +1,39 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package main + +import ( + "os" + "strings" + + "github.com/inancgumus/learngo/logparser/v5/pipe" +) + +// fromFile generates a default pipeline. +// Detects the correct parser by the file extension. +// Uses a TextReport and groups by domain. +func fromFile(path string) (*pipe.Pipeline, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + var src pipe.Iterator + switch { + case strings.HasSuffix(path, ".txt"): + src = pipe.NewTextLog(f) + case strings.HasSuffix(path, ".jsonl"): + src = pipe.NewJSONLog(f) + } + + return pipe.New( + src, + pipe.NewTextReport(os.Stdout), + pipe.GroupBy(pipe.DomainGrouper), + ), nil +} diff --git a/logparser/v5/main.go b/logparser/v5/main.go new file mode 100644 index 0000000..fe0b722 --- /dev/null +++ b/logparser/v5/main.go @@ -0,0 +1,35 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package main + +import ( + "log" + "os" + + "github.com/inancgumus/learngo/logparser/v5/pipe" +) + +func main() { + // p := pipe.Default( + // os.Stdin, os.Stdout, + // pipe.FilterBy(pipe.DomainExtFilter("com", "io")), + // pipe.GroupBy(pipe.DomainGrouper), + // ) + + p := pipe.New( + pipe.NewTextLog(os.Stdin), + pipe.NewTextReport(os.Stdout), + // pipe.NewJSONReport(os.Stdout), + pipe.FilterBy(pipe.DomainExtFilter("com", "io")), + pipe.GroupBy(pipe.DomainGrouper), + ) + + if err := p.Run(); err != nil { + log.Fatalln(err) + } +} diff --git a/logparser/v5/pipe/chartreport.go b/logparser/v5/pipe/chartreport.go new file mode 100644 index 0000000..6263be4 --- /dev/null +++ b/logparser/v5/pipe/chartreport.go @@ -0,0 +1,54 @@ +package pipe + +/* +// You need to run: +// go get -u github.com/wcharczuk/go-chart + +// Chart renders a chart. +type Chart struct { + Title string + Width, Height int + + w io.Writer +} + +// NewChartReport returns a Chart report generator. +func NewChartReport(w io.Writer) *Chart { + return &Chart{w: w} +} + +// Consume generates a chart report. +func (c *Chart) Consume(records Iterator) error { + w := os.Stdout + + donut := chart.DonutChart{ + Title: c.Title, + TitleStyle: chart.Style{ + FontSize: 35, + Show: true, + FontColor: chart.ColorAlternateGreen, + }, + Width: c.Width, + Height: c.Height, + } + + err := records.Each(func(r Record) error { + v := chart.Value{ + Label: r.Domain + r.Page + ": " + strconv.Itoa(r.Visits), + Value: float64(r.Visits), + Style: chart.Style{ + FontSize: 14, + }, + } + + donut.Values = append(donut.Values, v) + + return nil + }) + if err != nil { + return err + } + + return donut.Render(chart.SVG, w) +} +*/ diff --git a/logparser/oop/readclose.go b/logparser/v5/pipe/close.go similarity index 83% rename from logparser/oop/readclose.go rename to logparser/v5/pipe/close.go index 1aa7a30..29a6af2 100644 --- a/logparser/oop/readclose.go +++ b/logparser/v5/pipe/close.go @@ -5,12 +5,13 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package pipe import ( "io" ) +// readClose the reader if it's a io.Closer. func readClose(r io.Reader) { if rc, ok := r.(io.Closer); ok { rc.Close() diff --git a/logparser/v5/pipe/filter.go b/logparser/v5/pipe/filter.go new file mode 100644 index 0000000..9ac01a4 --- /dev/null +++ b/logparser/v5/pipe/filter.go @@ -0,0 +1,48 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +// FilterFunc represents a filtering pipeline func. +type FilterFunc = func(Record) (pass bool) + +// Filter the records. +type Filter struct { + src Iterator + filters []FilterFunc +} + +// FilterBy returns a new filter pipeline. +func FilterBy(fn ...FilterFunc) *Filter { + return &Filter{filters: fn} +} + +// Consume saves the iterator for later processing. +func (f *Filter) Consume(records Iterator) error { + f.src = records + return nil +} + +// Each yields only the filtered records. +func (f *Filter) Each(yield func(Record) error) error { + return f.src.Each(func(r Record) error { + if !f.check(r) { + return nil + } + return yield(r) + }) +} + +// check all the filters against the record. +func (f *Filter) check(r Record) bool { + for _, fi := range f.filters { + if !fi(r) { + return false + } + } + return true +} diff --git a/logparser/v5/pipe/filters.go b/logparser/v5/pipe/filters.go new file mode 100644 index 0000000..d66d393 --- /dev/null +++ b/logparser/v5/pipe/filters.go @@ -0,0 +1,41 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import "strings" + +// NotFilter reverses a filter. True becomes false, and vice versa. +func NotFilter(filter FilterFunc) FilterFunc { + return func(r Record) bool { + return !filter(r) + } +} + +// DomainExtFilter filters a set of domain extensions. +func DomainExtFilter(domains ...string) FilterFunc { + return func(r Record) bool { + for _, domain := range domains { + if strings.HasSuffix(r.Domain, "."+domain) { + return true + } + } + return false + } +} + +// DomainFilter filters a domain if it contains the given text. +func DomainFilter(text string) FilterFunc { + return func(r Record) bool { + return strings.Contains(r.Domain, text) + } +} + +// DomainOrgFilter filters only the ".org" domains. +func DomainOrgFilter(r Record) bool { + return strings.HasSuffix(r.Domain, ".org") +} diff --git a/logparser/v5/pipe/group.go b/logparser/v5/pipe/group.go new file mode 100644 index 0000000..5faef61 --- /dev/null +++ b/logparser/v5/pipe/group.go @@ -0,0 +1,60 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import ( + "sort" +) + +// GroupFunc represents a grouping func that returns a grouping key. +type GroupFunc = func(Record) (key string) + +// Group records by a key. +type Group struct { + sum map[string]Record // metrics per group key + keys []string // unique group keys + key GroupFunc +} + +// GroupBy returns a new Group. +// It takes a group func that returns a group key. +// The returned group will group the record using the key. +func GroupBy(key GroupFunc) *Group { + return &Group{ + sum: make(map[string]Record), + key: key, + } +} + +// Consume records for grouping. +func (g *Group) Consume(records Iterator) error { + return records.Each(func(r Record) error { + k := g.key(r) + + if _, ok := g.sum[k]; !ok { + g.keys = append(g.keys, k) + } + + g.sum[k] = r.Sum(g.sum[k]) + + return nil + }) +} + +// Each sorts and yields the grouped records. +func (g *Group) Each(yield func(Record) error) error { + sort.Strings(g.keys) + + for _, k := range g.keys { + if err := yield(g.sum[k]); err != nil { + return err + } + } + + return nil +} diff --git a/logparser/v5/pipe/groupers.go b/logparser/v5/pipe/groupers.go new file mode 100644 index 0000000..4b3cc59 --- /dev/null +++ b/logparser/v5/pipe/groupers.go @@ -0,0 +1,21 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +// DomainGrouper groups the records by domain. +// It keeps the other fields intact. +// For example: It returns the page field as well. +// Exercise: Write a solution that removes the unnecessary data. +func DomainGrouper(r Record) string { + return r.Domain +} + +// Page groups records by page. +func Page(r Record) string { + return r.Domain + r.Page +} diff --git a/logparser/oop/jsonlog.go b/logparser/v5/pipe/jsonlog.go similarity index 52% rename from logparser/oop/jsonlog.go rename to logparser/v5/pipe/jsonlog.go index 1c3affe..d833a7d 100644 --- a/logparser/oop/jsonlog.go +++ b/logparser/v5/pipe/jsonlog.go @@ -5,29 +5,31 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package pipe import ( - "bufio" "encoding/json" "io" ) -type jsonLog struct { +// JSON parses json records. +type JSON struct { reader io.Reader } -func newJSONLog(r io.Reader) *jsonLog { - return &jsonLog{reader: r} +// NewJSONLog creates a json parser. +func NewJSONLog(r io.Reader) *JSON { + return &JSON{reader: r} } -func (j *jsonLog) each(yield recordFn) error { +// Each yields records from a json reader. +func (j *JSON) Each(yield func(Record) error) error { defer readClose(j.reader) - dec := json.NewDecoder(bufio.NewReader(j.reader)) + dec := json.NewDecoder(j.reader) for { - var r record + var r Record err := dec.Decode(&r) if err == io.EOF { @@ -36,8 +38,10 @@ func (j *jsonLog) each(yield recordFn) error { if err != nil { return err } - - yield(r) + if err := yield(r); err != nil { + return err + } } + return nil } diff --git a/logparser/v5/pipe/jsonreport.go b/logparser/v5/pipe/jsonreport.go new file mode 100644 index 0000000..f2ada20 --- /dev/null +++ b/logparser/v5/pipe/jsonreport.go @@ -0,0 +1,32 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import ( + "encoding/json" + "io" +) + +// JSONReport generates a JSON report. +type JSONReport struct { + w io.Writer +} + +// NewJSONReport returns a JSON report generator. +func NewJSONReport(w io.Writer) *JSONReport { + return &JSONReport{w: w} +} + +// Consume generates a JSON report. +func (t *JSONReport) Consume(records Iterator) error { + enc := json.NewEncoder(t.w) + + return records.Each(func(r Record) error { + return enc.Encode(r) + }) +} diff --git a/logparser/oop/logcount.go b/logparser/v5/pipe/logcount.go similarity index 56% rename from logparser/oop/logcount.go rename to logparser/v5/pipe/logcount.go index 62f214a..ddc56e6 100644 --- a/logparser/oop/logcount.go +++ b/logparser/v5/pipe/logcount.go @@ -5,29 +5,33 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package pipe import "fmt" -// logCount counts the yielded records +// logCount counts the yielded records. type logCount struct { - iterator + Iterator n int } -func (lc *logCount) each(yield recordFn) error { - err := lc.iterator.each(func(r record) { +// Each yields to the inner iterator while counting the records. +// Reports the record number on an error. +func (lc *logCount) Each(yield func(Record) error) error { + err := lc.Iterator.Each(func(r Record) error { lc.n++ - yield(r) + return yield(r) }) if err != nil { // lc.n+1: iterator.each won't call yield on err return fmt.Errorf("record %d: %v", lc.n+1, err) } + return nil } +// count returns the last read record number. func (lc *logCount) count() int { return lc.n } diff --git a/logparser/v5/pipe/pipe.go b/logparser/v5/pipe/pipe.go new file mode 100644 index 0000000..5a02aa6 --- /dev/null +++ b/logparser/v5/pipe/pipe.go @@ -0,0 +1,29 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +// YieldFunc yields a record from an Iterator to up-stream (Consumer). +type YieldFunc = func(Record) error + +// Iterator yields a record. +type Iterator interface { + Each(YieldFunc) error +} + +// Consumer consumes records from an iterator. +type Consumer interface { + Consume(Iterator) error +} + +// Transform represents both a record consumer and producer. +// It has an input and output. +// It takes a single record and provides an iterator for all the records. +type Transform interface { + Iterator // producer: should never return on yield().err == nil + Consumer +} diff --git a/logparser/v5/pipe/pipeline.go b/logparser/v5/pipe/pipeline.go new file mode 100644 index 0000000..d2a0cec --- /dev/null +++ b/logparser/v5/pipe/pipeline.go @@ -0,0 +1,54 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import ( + "fmt" + "io" + "os" +) + +// Pipeline takes records from a source, transforms, and sends them to a destionation. +type Pipeline struct { + src Iterator + trans []Transform + dst Consumer +} + +// New creates a new pipeline. +func New(src Iterator, dst Consumer, t ...Transform) *Pipeline { + return &Pipeline{ + src: &logCount{Iterator: src}, + dst: dst, + trans: t, + } +} + +// Default creates a pipeline that reads from a text log and generates a text report. +func Default(r io.Reader, w io.Writer, t ...Transform) *Pipeline { + return New(NewTextLog(r), NewTextReport(w), t...) +} + +// Run the pipeline. +func (p *Pipeline) Run() error { + defer func() { + n := p.src.(*logCount).count() + fmt.Fprintf(os.Stderr, "%d records processed.\n", n) + }() + + last := p.src + + for _, t := range p.trans { + if err := t.Consume(last); err != nil { + return err + } + last = t + } + + return p.dst.Consume(last) +} diff --git a/logparser/v5/pipe/record.go b/logparser/v5/pipe/record.go new file mode 100644 index 0000000..3fe016c --- /dev/null +++ b/logparser/v5/pipe/record.go @@ -0,0 +1,85 @@ +package pipe + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" +) + +const fieldsLength = 4 + +// Record stores fields of a log line. +type Record struct { + Domain string + Page string + Visits int + Uniques int +} + +// Sum the numeric fields with another record. +func (r Record) Sum(other Record) Record { + r.Visits += other.Visits + r.Uniques += other.Uniques + return r +} + +// UnmarshalText to a *Record. +func (r *Record) UnmarshalText(p []byte) (err error) { + fields := strings.Fields(string(p)) + if len(fields) != fieldsLength { + return fmt.Errorf("wrong number of fields %q", fields) + } + + r.Domain, r.Page = fields[0], fields[1] + + if r.Visits, err = parseStr("visits", fields[2]); err != nil { + return err + } + if r.Uniques, err = parseStr("uniques", fields[3]); err != nil { + return err + } + return validate(*r) +} + +// UnmarshalJSON to a *Record. +func (r *Record) UnmarshalJSON(data []byte) error { + // `methodless` doesn't have any methods including UnmarshalJSON. + // This trick prevents the stack-overflow (infinite loop). + type methodless Record + + var m methodless + if err := json.Unmarshal(data, &m); err != nil { + return err + } + + // Cast back to the Record and save. + *r = Record(m) + + return validate(*r) +} + +// parseStr helps UnmarshalText for string to positive int parsing. +func parseStr(name, v string) (int, error) { + n, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err) + } + return n, nil +} + +// validate whether a parsed record is valid or not. +func validate(r Record) (err error) { + switch { + case r.Domain == "": + err = errors.New("record.domain cannot be empty") + case r.Page == "": + err = errors.New("record.page cannot be empty") + case r.Visits < 0: + err = errors.New("record.visits cannot be negative") + case r.Uniques < 0: + err = errors.New("record.uniques cannot be negative") + } + return +} diff --git a/logparser/oop/textlog.go b/logparser/v5/pipe/textlog.go similarity index 54% rename from logparser/oop/textlog.go rename to logparser/v5/pipe/textlog.go index 9a5e1fa..a81e050 100644 --- a/logparser/oop/textlog.go +++ b/logparser/v5/pipe/textlog.go @@ -5,34 +5,39 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package pipe import ( "bufio" "io" ) -type textLog struct { +// TextLog parses text based log lines. +type TextLog struct { reader io.Reader } -func newTextLog(r io.Reader) *textLog { - return &textLog{reader: r} +// NewTextLog creates a text parser. +func NewTextLog(r io.Reader) *TextLog { + return &TextLog{reader: r} } -func (p *textLog) each(yield recordFn) error { +// Each yields records from a text log. +func (p *TextLog) Each(yield func(Record) error) error { defer readClose(p.reader) in := bufio.NewScanner(p.reader) for in.Scan() { - r := new(record) + r := new(Record) if err := r.UnmarshalText(in.Bytes()); err != nil { return err } - yield(*r) + if err := yield(*r); err != nil { + return err + } } return in.Err() diff --git a/logparser/v5/pipe/textreport.go b/logparser/v5/pipe/textreport.go new file mode 100644 index 0000000..b13925f --- /dev/null +++ b/logparser/v5/pipe/textreport.go @@ -0,0 +1,65 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import ( + "fmt" + "io" + "text/tabwriter" +) + +const ( + minWidth = 0 + tabWidth = 4 + padding = 4 + flags = 0 +) + +// TextReport report generator. +type TextReport struct { + w io.Writer +} + +// NewTextReport returns a TextReport report generator. +func NewTextReport(w io.Writer) *TextReport { + return &TextReport{w: w} +} + +// Consume generates a text report. +func (t *TextReport) Consume(records Iterator) error { + w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags) + + write := fmt.Fprintf + + write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") + write(w, "-------\t-----\t------\t-------\n") + + var total Record + + err := records.Each(func(r Record) error { + total = r.Sum(total) + + write(w, "%s\t%s\t%d\t%d\n", + r.Domain, r.Page, + r.Visits, r.Uniques, + ) + + return nil + }) + if err != nil { + return err + } + + write(w, "\t\t\t\n") + write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", + total.Visits, + total.Uniques, + ) + + return w.Flush() +}