From 0a121cd9113da91ad86aa09aece858225453b411 Mon Sep 17 00:00:00 2001 From: Inanc Gumus Date: Wed, 28 Aug 2019 18:54:57 +0300 Subject: [PATCH] refactor: oop log parser to pkgs --- interfaces/log-parser/oop/chartreport.go | 38 ------ interfaces/log-parser/oop/filepipe.go | 42 +++++++ interfaces/log-parser/oop/filter.go | 43 ------- interfaces/log-parser/oop/filters.go | 36 ------ interfaces/log-parser/oop/group.go | 49 -------- interfaces/log-parser/oop/groupers.go | 15 --- interfaces/log-parser/oop/main.go | 53 ++++---- .../log-parser/oop/pipe/filter/domain.go | 38 ++++++ .../log-parser/oop/pipe/filter/filter.go | 50 ++++++++ interfaces/log-parser/oop/pipe/filter/noop.go | 15 +++ interfaces/log-parser/oop/pipe/filter/not.go | 17 +++ .../log-parser/oop/pipe/group/domain.go | 18 +++ interfaces/log-parser/oop/pipe/group/group.go | 60 +++++++++ interfaces/log-parser/oop/pipe/group/page.go | 15 +++ .../log-parser/oop/{ => pipe}/logcount.go | 13 +- .../oop/{readclose.go => pipe/parse/close.go} | 3 +- .../oop/{jsonlog.go => pipe/parse/json.go} | 18 +-- .../log-parser/oop/pipe/parse/record.go | 116 ++++++++++++++++++ .../oop/{textlog.go => pipe/parse/text.go} | 17 ++- interfaces/log-parser/oop/pipe/pipe.go | 26 ++++ interfaces/log-parser/oop/pipe/pipeline.go | 48 ++++++++ interfaces/log-parser/oop/pipe/record.go | 19 +++ .../log-parser/oop/pipe/report/chart.go | 49 ++++++++ interfaces/log-parser/oop/pipe/report/text.go | 64 ++++++++++ interfaces/log-parser/oop/pipeline.go | 78 ------------ interfaces/log-parser/oop/record.go | 82 ------------- interfaces/log-parser/oop/textreport.go | 49 -------- 27 files changed, 638 insertions(+), 433 deletions(-) delete mode 100644 interfaces/log-parser/oop/chartreport.go create mode 100644 interfaces/log-parser/oop/filepipe.go delete mode 100644 interfaces/log-parser/oop/filter.go delete mode 100644 interfaces/log-parser/oop/filters.go delete mode 100644 interfaces/log-parser/oop/group.go delete mode 100644 interfaces/log-parser/oop/groupers.go create mode 100644 interfaces/log-parser/oop/pipe/filter/domain.go create mode 100644 interfaces/log-parser/oop/pipe/filter/filter.go create mode 100644 interfaces/log-parser/oop/pipe/filter/noop.go create mode 100644 interfaces/log-parser/oop/pipe/filter/not.go create mode 100644 interfaces/log-parser/oop/pipe/group/domain.go create mode 100644 interfaces/log-parser/oop/pipe/group/group.go create mode 100644 interfaces/log-parser/oop/pipe/group/page.go rename interfaces/log-parser/oop/{ => pipe}/logcount.go (59%) rename interfaces/log-parser/oop/{readclose.go => pipe/parse/close.go} (83%) rename interfaces/log-parser/oop/{jsonlog.go => pipe/parse/json.go} (54%) create mode 100644 interfaces/log-parser/oop/pipe/parse/record.go rename interfaces/log-parser/oop/{textlog.go => pipe/parse/text.go} (56%) create mode 100644 interfaces/log-parser/oop/pipe/pipe.go create mode 100644 interfaces/log-parser/oop/pipe/pipeline.go create mode 100644 interfaces/log-parser/oop/pipe/record.go create mode 100644 interfaces/log-parser/oop/pipe/report/chart.go create mode 100644 interfaces/log-parser/oop/pipe/report/text.go delete mode 100644 interfaces/log-parser/oop/pipeline.go delete mode 100644 interfaces/log-parser/oop/record.go delete mode 100644 interfaces/log-parser/oop/textreport.go diff --git a/interfaces/log-parser/oop/chartreport.go b/interfaces/log-parser/oop/chartreport.go deleted file mode 100644 index a77ba78..0000000 --- a/interfaces/log-parser/oop/chartreport.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -// You need to run: -// go get -u github.com/wcharczuk/go-chart - -// type chartReport struct { -// title string -// width, height int -// } - -// func (s *chartReport) digest(records iterator) error { -// w := os.Stdout - -// donut := chart.DonutChart{ -// Title: s.title, -// TitleStyle: chart.Style{ -// FontSize: 35, -// Show: true, -// FontColor: chart.ColorAlternateGreen, -// }, -// Width: s.width, -// Height: s.height, -// } - -// records.each(func(r record) { -// v := chart.Value{ -// Label: r.domain + r.page + ": " + strconv.Itoa(r.visits), -// Value: float64(r.visits), -// Style: chart.Style{ -// FontSize: 14, -// }, -// } - -// donut.Values = append(donut.Values, v) -// }) - -// return donut.Render(chart.SVG, w) -// } diff --git a/interfaces/log-parser/oop/filepipe.go b/interfaces/log-parser/oop/filepipe.go new file mode 100644 index 0000000..557fb9a --- /dev/null +++ b/interfaces/log-parser/oop/filepipe.go @@ -0,0 +1,42 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package main + +import ( + "os" + "strings" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report" +) + +// fromFile generates a default pipeline. +// Detects the correct parser by the file extension. +// Uses a TextReport and groups by domain. +func fromFile(path string) (*pipe.Pipeline, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + var src pipe.Iterator + switch { + case strings.HasSuffix(path, ".txt"): + src = parse.FromText(f) + case strings.HasSuffix(path, ".jsonl"): + src = parse.FromJSON(f) + } + + return pipe.New( + src, + report.AsText(os.Stdout), + group.By(group.Domain), + ), nil +} diff --git a/interfaces/log-parser/oop/filter.go b/interfaces/log-parser/oop/filter.go deleted file mode 100644 index 031a556..0000000 --- a/interfaces/log-parser/oop/filter.go +++ /dev/null @@ -1,43 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -type filter struct { - src iterator - filters []filterFunc -} - -func filterBy(fn ...filterFunc) *filter { - return &filter{filters: fn} -} - -// transform the record -func (f *filter) digest(records iterator) error { - f.src = records - return nil -} - -// each yields only the filtered records -func (f *filter) each(yield recordFn) error { - return f.src.each(func(r record) { - if !f.check(r) { - return - } - yield(r) - }) -} - -// check all the filters against the record -func (f *filter) check(r record) bool { - for _, fi := range f.filters { - if !fi(r) { - return false - } - } - return true -} diff --git a/interfaces/log-parser/oop/filters.go b/interfaces/log-parser/oop/filters.go deleted file mode 100644 index ad00098..0000000 --- a/interfaces/log-parser/oop/filters.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import "strings" - -type filterFunc func(record) bool - -func noopFilter(r record) bool { - return true -} - -func notUsing(filter filterFunc) filterFunc { - return func(r record) bool { - return !filter(r) - } -} - -func domainExtFilter(domains ...string) filterFunc { - return func(r record) bool { - for _, domain := range domains { - if strings.HasSuffix(r.domain, "."+domain) { - return true - } - } - return false - } -} - -func domainFilter(domain string) filterFunc { - return func(r record) bool { - return strings.Contains(r.domain, domain) - } -} - -func orgDomainsFilter(r record) bool { - return strings.HasSuffix(r.domain, ".org") -} diff --git a/interfaces/log-parser/oop/group.go b/interfaces/log-parser/oop/group.go deleted file mode 100644 index 5d57955..0000000 --- a/interfaces/log-parser/oop/group.go +++ /dev/null @@ -1,49 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "sort" -) - -type group struct { - sum map[string]record // metrics per group key - keys []string // unique group keys - key groupFunc -} - -func groupBy(key groupFunc) *group { - return &group{ - sum: make(map[string]record), - key: key, - } -} - -// digest the records -func (g *group) digest(records iterator) error { - return records.each(func(r record) { - k := g.key(r) - - if _, ok := g.sum[k]; !ok { - g.keys = append(g.keys, k) - } - - g.sum[k] = r.sum(g.sum[k]) - }) -} - -// each yields the grouped records -func (g *group) each(yield recordFn) error { - sort.Strings(g.keys) - - for _, k := range g.keys { - yield(g.sum[k]) - } - - return nil -} diff --git a/interfaces/log-parser/oop/groupers.go b/interfaces/log-parser/oop/groupers.go deleted file mode 100644 index 79980e7..0000000 --- a/interfaces/log-parser/oop/groupers.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -type groupFunc func(record) string - -// domainGrouper groups by domain. -// but it keeps the other fields. -// for example: it returns pages as well, but you shouldn't use them. -// exercise: write a function that erases the unnecessary data. -func domainGrouper(r record) string { - return r.domain -} - -func pageGrouper(r record) string { - return r.domain + r.page -} diff --git a/interfaces/log-parser/oop/main.go b/interfaces/log-parser/oop/main.go index 285fdbb..7a30be0 100644 --- a/interfaces/log-parser/oop/main.go +++ b/interfaces/log-parser/oop/main.go @@ -10,35 +10,40 @@ package main import ( "log" "os" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/filter" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse" + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report" ) func main() { - // newGrouper(domainGrouper) - - // s := &chartReport{ - // title: "visits per domain", - // width: 1920, - // height: 800, - // } - - // pipe, err := fromFile("../logs/log.jsonl") - // if err != nil { - // log.Fatalln(err) - // } - - pipe := newPipeline( - newTextLog(os.Stdin), - // newJSONLog(os.Stdin), - newTextReport(), - filterBy(notUsing(domainExtFilter("com", "io"))), - groupBy(domainGrouper), + pipe := pipe.New( + parse.FromText(os.Stdin), + // parse.FromJSON(os.Stdin), + report.AsText(os.Stdout), + filter.By(filter.Not(filter.DomainExt("com", "io"))), + group.By(group.Domain), + new(logger), ) - if err := pipe.run(); err != nil { + if err := pipe.Run(); err != nil { log.Fatalln(err) } - - // if err := reportFromFile(os.Args[1]); err != nil { - // log.Fatalln(err) - // } +} + +type logger struct { + src pipe.Iterator +} + +func (l *logger) Digest(records pipe.Iterator) error { + l.src = records + return nil +} + +func (l *logger) Each(yield func(pipe.Record)) error { + return l.src.Each(func(r pipe.Record) { + yield(r) + }) } diff --git a/interfaces/log-parser/oop/pipe/filter/domain.go b/interfaces/log-parser/oop/pipe/filter/domain.go new file mode 100644 index 0000000..e3901c5 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/filter/domain.go @@ -0,0 +1,38 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package filter + +import ( + "strings" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" +) + +// DomainExt filters a set of domain extensions. +func DomainExt(domains ...string) Func { + return func(r pipe.Record) bool { + for _, domain := range domains { + if strings.HasSuffix(r.Str("domain"), "."+domain) { + return true + } + } + return false + } +} + +// Domain filters a domain if it contains the given text. +func Domain(text string) Func { + return func(r pipe.Record) bool { + return strings.Contains(r.Str("domain"), text) + } +} + +// DomainOrg filters only the ".org" domains. +func DomainOrg(r pipe.Record) bool { + return strings.HasSuffix(r.Str("domain"), ".org") +} diff --git a/interfaces/log-parser/oop/pipe/filter/filter.go b/interfaces/log-parser/oop/pipe/filter/filter.go new file mode 100644 index 0000000..c58bab0 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/filter/filter.go @@ -0,0 +1,50 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package filter + +import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + +// Func represents a filtering pipeline func. +type Func func(pipe.Record) (pass bool) + +// Filter the records. +type Filter struct { + src pipe.Iterator + filters []Func +} + +// By returns a new filter pipeline. +func By(fn ...Func) *Filter { + return &Filter{filters: fn} +} + +// Digest saves the iterator for later processing. +func (f *Filter) Digest(records pipe.Iterator) error { + f.src = records + return nil +} + +// Each yields only the filtered records. +func (f *Filter) Each(yield func(pipe.Record)) error { + return f.src.Each(func(r pipe.Record) { + if !f.check(r) { + return + } + yield(r) + }) +} + +// check all the filters against the record. +func (f *Filter) check(r pipe.Record) bool { + for _, fi := range f.filters { + if !fi(r) { + return false + } + } + return true +} diff --git a/interfaces/log-parser/oop/pipe/filter/noop.go b/interfaces/log-parser/oop/pipe/filter/noop.go new file mode 100644 index 0000000..cce6639 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/filter/noop.go @@ -0,0 +1,15 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package filter + +import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + +// Noop filter that does nothing. +func Noop(r pipe.Record) bool { + return true +} diff --git a/interfaces/log-parser/oop/pipe/filter/not.go b/interfaces/log-parser/oop/pipe/filter/not.go new file mode 100644 index 0000000..face064 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/filter/not.go @@ -0,0 +1,17 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package filter + +import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + +// Not reverses a filter. True becomes false, and vice versa. +func Not(filter Func) Func { + return func(r pipe.Record) bool { + return !filter(r) + } +} diff --git a/interfaces/log-parser/oop/pipe/group/domain.go b/interfaces/log-parser/oop/pipe/group/domain.go new file mode 100644 index 0000000..6bf650e --- /dev/null +++ b/interfaces/log-parser/oop/pipe/group/domain.go @@ -0,0 +1,18 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package group + +import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + +// Domain groups the records by domain. +// It keeps the other fields intact. +// For example: It returns the page field as well. +// Exercise: Write a solution that removes the unnecessary data. +func Domain(r pipe.Record) string { + return r.Str("domain") +} diff --git a/interfaces/log-parser/oop/pipe/group/group.go b/interfaces/log-parser/oop/pipe/group/group.go new file mode 100644 index 0000000..aed72fe --- /dev/null +++ b/interfaces/log-parser/oop/pipe/group/group.go @@ -0,0 +1,60 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package group + +import ( + "sort" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" +) + +// Func represents a grouping func that returns a grouping key. +type Func func(pipe.Record) (key string) + +// Group records by a key. +type Group struct { + sum map[string]pipe.Record // metrics per group key + keys []string // unique group keys + key Func +} + +// By returns a new Group. +// It takes a group func that returns a group key. +// The returned group will group the record using the key. +func By(key Func) *Group { + return &Group{ + sum: make(map[string]pipe.Record), + key: key, + } +} + +// Digest records for grouping. +func (g *Group) Digest(records pipe.Iterator) error { + return records.Each(func(r pipe.Record) { + k := g.key(r) + + if _, ok := g.sum[k]; !ok { + g.keys = append(g.keys, k) + } + + if r, ok := r.(pipe.Summer); ok { + g.sum[k] = r.Sum(g.sum[k]) + } + }) +} + +// Each sorts and yields the grouped records. +func (g *Group) Each(yield func(pipe.Record)) error { + sort.Strings(g.keys) + + for _, k := range g.keys { + yield(g.sum[k]) + } + + return nil +} diff --git a/interfaces/log-parser/oop/pipe/group/page.go b/interfaces/log-parser/oop/pipe/group/page.go new file mode 100644 index 0000000..1428d2e --- /dev/null +++ b/interfaces/log-parser/oop/pipe/group/page.go @@ -0,0 +1,15 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package group + +import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" + +// Page groups records by page. +func Page(r pipe.Record) string { + return r.Str("domain") + r.Str("page") +} diff --git a/interfaces/log-parser/oop/logcount.go b/interfaces/log-parser/oop/pipe/logcount.go similarity index 59% rename from interfaces/log-parser/oop/logcount.go rename to interfaces/log-parser/oop/pipe/logcount.go index 62f214a..44ab8e3 100644 --- a/interfaces/log-parser/oop/logcount.go +++ b/interfaces/log-parser/oop/pipe/logcount.go @@ -5,18 +5,20 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package pipe import "fmt" -// logCount counts the yielded records +// logCount counts the yielded records. type logCount struct { - iterator + Iterator n int } -func (lc *logCount) each(yield recordFn) error { - err := lc.iterator.each(func(r record) { +// Each yields to the inner iterator while counting the records. +// Reports the record number on an error. +func (lc *logCount) Each(yield func(Record)) error { + err := lc.Iterator.Each(func(r Record) { lc.n++ yield(r) }) @@ -28,6 +30,7 @@ func (lc *logCount) each(yield recordFn) error { return nil } +// count returns the last read record number. func (lc *logCount) count() int { return lc.n } diff --git a/interfaces/log-parser/oop/readclose.go b/interfaces/log-parser/oop/pipe/parse/close.go similarity index 83% rename from interfaces/log-parser/oop/readclose.go rename to interfaces/log-parser/oop/pipe/parse/close.go index 1aa7a30..be034c8 100644 --- a/interfaces/log-parser/oop/readclose.go +++ b/interfaces/log-parser/oop/pipe/parse/close.go @@ -5,12 +5,13 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package parse import ( "io" ) +// readClose the reader if it's a io.Closer. func readClose(r io.Reader) { if rc, ok := r.(io.Closer); ok { rc.Close() diff --git a/interfaces/log-parser/oop/jsonlog.go b/interfaces/log-parser/oop/pipe/parse/json.go similarity index 54% rename from interfaces/log-parser/oop/jsonlog.go rename to interfaces/log-parser/oop/pipe/parse/json.go index 1c3affe..96c214e 100644 --- a/interfaces/log-parser/oop/jsonlog.go +++ b/interfaces/log-parser/oop/pipe/parse/json.go @@ -5,26 +5,30 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package parse import ( - "bufio" "encoding/json" "io" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" ) -type jsonLog struct { +// JSON parses json records. +type JSON struct { reader io.Reader } -func newJSONLog(r io.Reader) *jsonLog { - return &jsonLog{reader: r} +// FromJSON creates a json parser. +func FromJSON(r io.Reader) *JSON { + return &JSON{reader: r} } -func (j *jsonLog) each(yield recordFn) error { +// Each yields records from a json reader. +func (j *JSON) Each(yield func(pipe.Record)) error { defer readClose(j.reader) - dec := json.NewDecoder(bufio.NewReader(j.reader)) + dec := json.NewDecoder(j.reader) for { var r record diff --git a/interfaces/log-parser/oop/pipe/parse/record.go b/interfaces/log-parser/oop/pipe/parse/record.go new file mode 100644 index 0000000..bcc16a3 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/parse/record.go @@ -0,0 +1,116 @@ +package parse + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" +) + +const fieldsLength = 4 + +// record stores fields of a log line. +type record struct { + Domain string + Page string + Visits int + Uniques int +} + +// Str gets a string field by name. +func (r record) Str(field string) string { + switch field { + case "domain": + return r.Domain + case "page": + return r.Page + } + panic(fieldErr(field)) +} + +// Int gets an integer field by name. +func (r record) Int(field string) int { + switch field { + case "visits": + return r.Visits + case "uniques": + return r.Uniques + } + panic(fieldErr(field)) +} + +// Sum the numeric fields with another record. +func (r record) Sum(other pipe.Record) pipe.Record { + if other == nil { + return r + } + r.Visits += other.(record).Visits + r.Uniques += other.(record).Uniques + return r +} + +// UnmarshalText to a *record. +func (r *record) UnmarshalText(p []byte) (err error) { + fields := strings.Fields(string(p)) + if len(fields) != fieldsLength { + return fmt.Errorf("wrong number of fields %q", fields) + } + + r.Domain, r.Page = fields[0], fields[1] + + if r.Visits, err = parseStr("visits", fields[2]); err != nil { + return err + } + if r.Uniques, err = parseStr("uniques", fields[3]); err != nil { + return err + } + return validate(*r) +} + +// UnmarshalJSON to a *record. +func (r *record) UnmarshalJSON(data []byte) error { + // `methodless` doesn't have any methods including UnmarshalJSON. + // This trick prevents the stack-overflow (infinite loop). + type methodless record + + var m methodless + if err := json.Unmarshal(data, &m); err != nil { + return err + } + + // Cast back to the record and save. + *r = record(m) + + return validate(*r) +} + +// parseStr helps UnmarshalText for string to positive int parsing. +func parseStr(name, v string) (int, error) { + n, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err) + } + return n, nil +} + +// validate whether a parsed record is valid or not. +func validate(r record) (err error) { + switch { + case r.Domain == "": + err = errors.New("record.domain cannot be empty") + case r.Page == "": + err = errors.New("record.page cannot be empty") + case r.Visits < 0: + err = errors.New("record.visits cannot be negative") + case r.Uniques < 0: + err = errors.New("record.uniques cannot be negative") + } + return +} + +func fieldErr(field string) error { + return fmt.Errorf("record field: %q does not exist", field) +} diff --git a/interfaces/log-parser/oop/textlog.go b/interfaces/log-parser/oop/pipe/parse/text.go similarity index 56% rename from interfaces/log-parser/oop/textlog.go rename to interfaces/log-parser/oop/pipe/parse/text.go index 9a5e1fa..c106c86 100644 --- a/interfaces/log-parser/oop/textlog.go +++ b/interfaces/log-parser/oop/pipe/parse/text.go @@ -5,22 +5,27 @@ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // -package main +package parse import ( "bufio" "io" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" ) -type textLog struct { +// Text parses text based log lines. +type Text struct { reader io.Reader } -func newTextLog(r io.Reader) *textLog { - return &textLog{reader: r} +// FromText creates a text parser. +func FromText(r io.Reader) *Text { + return &Text{reader: r} } -func (p *textLog) each(yield recordFn) error { +// Each yields records from a text log. +func (p *Text) Each(yield func(pipe.Record)) error { defer readClose(p.reader) in := bufio.NewScanner(p.reader) @@ -32,7 +37,7 @@ func (p *textLog) each(yield recordFn) error { return err } - yield(*r) + yield(r) } return in.Err() diff --git a/interfaces/log-parser/oop/pipe/pipe.go b/interfaces/log-parser/oop/pipe/pipe.go new file mode 100644 index 0000000..306055a --- /dev/null +++ b/interfaces/log-parser/oop/pipe/pipe.go @@ -0,0 +1,26 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +// Iterator yields a record. +type Iterator interface { + Each(func(Record)) error +} + +// Digester represents a record consumer. +type Digester interface { + Digest(Iterator) error +} + +// Transform represents both a record consumer and producer. +// It has an input and output. +// It takes a single record and provides an iterator for all the records. +type Transform interface { + Digester // consumer + Iterator // producer +} diff --git a/interfaces/log-parser/oop/pipe/pipeline.go b/interfaces/log-parser/oop/pipe/pipeline.go new file mode 100644 index 0000000..afac6e2 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/pipeline.go @@ -0,0 +1,48 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +import ( + "fmt" + "os" +) + +// Pipeline takes records from a source, transforms, and sends them to a destionation. +type Pipeline struct { + src Iterator + trans []Transform + dst Digester +} + +// New creates a new pipeline. +func New(src Iterator, dst Digester, t ...Transform) *Pipeline { + return &Pipeline{ + src: &logCount{Iterator: src}, + dst: dst, + trans: t, + } +} + +// Run the pipeline. +func (p *Pipeline) Run() error { + defer func() { + n := p.src.(*logCount).count() + fmt.Fprintf(os.Stderr, "%d records processed.\n", n) + }() + + last := p.src + + for _, t := range p.trans { + if err := t.Digest(last); err != nil { + return err + } + last = t + } + + return p.dst.Digest(last) +} diff --git a/interfaces/log-parser/oop/pipe/record.go b/interfaces/log-parser/oop/pipe/record.go new file mode 100644 index 0000000..5116615 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/record.go @@ -0,0 +1,19 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package pipe + +// Record provides a generic interface for any sort of records. +type Record interface { + Str(field string) string + Int(field string) int +} + +// Summer provides a method for summing the numeric fields. +type Summer interface { + Sum(Record) Record +} diff --git a/interfaces/log-parser/oop/pipe/report/chart.go b/interfaces/log-parser/oop/pipe/report/chart.go new file mode 100644 index 0000000..3322ef5 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/report/chart.go @@ -0,0 +1,49 @@ +package report + +/* +// You need to run: +// go get -u github.com/wcharczuk/go-chart + +// Chart renders a chart. +type Chart struct { + Title string + Width, Height int + + w io.Writer +} + +// AsChart returns a Chart report generator. +func AsChart(w io.Writer) *Chart { + return &Chart{w: w} +} + +// Digest generates a chart report. +func (c *Chart) Digest(records pipe.Iterator) error { + w := os.Stdout + + donut := chart.DonutChart{ + Title: c.Title, + TitleStyle: chart.Style{ + FontSize: 35, + Show: true, + FontColor: chart.ColorAlternateGreen, + }, + Width: c.Width, + Height: c.Height, + } + + records.Each(func(r pipe.Record) { + v := chart.Value{ + Label: r.Str("domain") + r.Str("page") + ": " + strconv.Itoa(r.Int("visits")), + Value: float64(r.Int("visits")), + Style: chart.Style{ + FontSize: 14, + }, + } + + donut.Values = append(donut.Values, v) + }) + + return donut.Render(chart.SVG, w) +} +*/ diff --git a/interfaces/log-parser/oop/pipe/report/text.go b/interfaces/log-parser/oop/pipe/report/text.go new file mode 100644 index 0000000..5db9b81 --- /dev/null +++ b/interfaces/log-parser/oop/pipe/report/text.go @@ -0,0 +1,64 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package report + +import ( + "fmt" + "io" + "text/tabwriter" + + "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" +) + +const ( + minWidth = 0 + tabWidth = 4 + padding = 4 + flags = 0 +) + +// Text report generator. +type Text struct { + w io.Writer +} + +// AsText returns a Text report generator. +func AsText(w io.Writer) *Text { + return &Text{w: w} +} + +// Digest generates a text report. +func (t *Text) Digest(records pipe.Iterator) error { + w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags) + + write := fmt.Fprintf + + write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") + write(w, "-------\t-----\t------\t-------\n") + + var total pipe.Record + + records.Each(func(r pipe.Record) { + if r, ok := r.(pipe.Summer); ok { + total = r.Sum(total) + } + + write(w, "%s\t%s\t%d\t%d\n", + r.Str("domain"), r.Str("page"), + r.Int("visits"), r.Int("uniques"), + ) + }) + + write(w, "\t\t\t\n") + write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", + total.Int("visits"), + total.Int("uniques"), + ) + + return w.Flush() +} diff --git a/interfaces/log-parser/oop/pipeline.go b/interfaces/log-parser/oop/pipeline.go deleted file mode 100644 index f2d2f7c..0000000 --- a/interfaces/log-parser/oop/pipeline.go +++ /dev/null @@ -1,78 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "fmt" - "os" - "strings" -) - -type recordFn func(record) - -type iterator interface{ each(recordFn) error } -type digester interface{ digest(iterator) error } - -type transform interface { - digester - iterator -} - -type pipeline struct { - src iterator - trans []transform - dst digester -} - -func (p *pipeline) run() error { - defer func() { - n := p.src.(*logCount).count() - fmt.Printf("%d records processed.\n", n) - }() - - last := p.src - - for _, t := range p.trans { - if err := t.digest(last); err != nil { - return err - } - last = t - } - - return p.dst.digest(last) -} - -func newPipeline(src iterator, dst digester, t ...transform) *pipeline { - return &pipeline{ - src: &logCount{iterator: src}, - dst: dst, - trans: t, - } -} - -// fromFile generates a default report -func fromFile(path string) (*pipeline, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - var src iterator - switch { - case strings.HasSuffix(path, ".txt"): - src = newTextLog(f) - case strings.HasSuffix(path, ".jsonl"): - src = newJSONLog(f) - } - - return newPipeline( - src, - newTextReport(), - groupBy(domainGrouper), - ), nil -} diff --git a/interfaces/log-parser/oop/record.go b/interfaces/log-parser/oop/record.go deleted file mode 100644 index 80b020f..0000000 --- a/interfaces/log-parser/oop/record.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" -) - -const fieldsLength = 4 - -type record struct { - domain string - page string - visits int - uniques int -} - -func (r record) sum(other record) record { - r.visits += other.visits - r.uniques += other.uniques - return r -} - -// UnmarshalText to a *record -func (r *record) UnmarshalText(p []byte) (err error) { - fields := strings.Fields(string(p)) - if len(fields) != fieldsLength { - return fmt.Errorf("wrong number of fields %q", fields) - } - - r.domain, r.page = fields[0], fields[1] - - if r.visits, err = parseStr("visits", fields[2]); err != nil { - return err - } - if r.uniques, err = parseStr("uniques", fields[3]); err != nil { - return err - } - return validate(*r) -} - -// UnmarshalJSON to a *record -func (r *record) UnmarshalJSON(data []byte) error { - var re struct { - Domain string - Page string - Visits int - Uniques int - } - - if err := json.Unmarshal(data, &re); err != nil { - return err - } - - *r = record{re.Domain, re.Page, re.Visits, re.Uniques} - return validate(*r) -} - -// parseStr helps UnmarshalText for string to positive int parsing -func parseStr(name, v string) (int, error) { - n, err := strconv.Atoi(v) - if err != nil { - return 0, fmt.Errorf("record.UnmarshalText %q: %v", name, err) - } - return n, nil -} - -func validate(r record) (err error) { - switch { - case r.domain == "": - err = errors.New("record.domain cannot be empty") - case r.page == "": - err = errors.New("record.page cannot be empty") - case r.visits < 0: - err = errors.New("record.visits cannot be negative") - case r.uniques < 0: - err = errors.New("record.uniques cannot be negative") - } - return -} diff --git a/interfaces/log-parser/oop/textreport.go b/interfaces/log-parser/oop/textreport.go deleted file mode 100644 index b0c42de..0000000 --- a/interfaces/log-parser/oop/textreport.go +++ /dev/null @@ -1,49 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "fmt" - "os" - "text/tabwriter" -) - -// TODO: make this configurable? or exercise? -const ( - minWidth = 0 - tabWidth = 4 - padding = 4 - flags = 0 -) - -type textReport struct{} - -func newTextReport() *textReport { - return new(textReport) -} - -func (s *textReport) digest(records iterator) error { - w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, ' ', flags) - - write := fmt.Fprintf - - write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") - write(w, "-------\t-----\t------\t-------\n") - - var total record - records.each(func(r record) { - total = total.sum(r) - - write(w, "%s\t%s\t%d\t%d\n", r.domain, r.page, r.visits, r.uniques) - }) - - write(w, "\t\t\t\n") - write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", total.visits, total.uniques) - - return w.Flush() -}