diff --git a/logparser/v6/Makefile b/logparser/v6/Makefile deleted file mode 100644 index a9f738d..0000000 --- a/logparser/v6/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -r: - go run . < ../../logs/log.txt - -t: - time go run . < ../../logs/log.txt \ No newline at end of file diff --git a/logparser/v6/filepipe.go b/logparser/v6/filepipe.go deleted file mode 100644 index c3d3207..0000000 --- a/logparser/v6/filepipe.go +++ /dev/null @@ -1,42 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "os" - "strings" - - "github.com/inancgumus/learngo/logparser/v6/pipe" - "github.com/inancgumus/learngo/logparser/v6/pipe/group" - "github.com/inancgumus/learngo/logparser/v6/pipe/parse" - "github.com/inancgumus/learngo/logparser/v6/pipe/report" -) - -// fromFile generates a default pipeline. -// Detects the correct parser by the file extension. -// Uses a TextReport and groups by domain. -func fromFile(path string) (*pipe.Pipeline, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - var src pipe.Iterator - switch { - case strings.HasSuffix(path, ".txt"): - src = parse.FromText(f) - case strings.HasSuffix(path, ".jsonl"): - src = parse.FromJSON(f) - } - - return pipe.New( - src, - report.AsText(os.Stdout), - group.By(group.Domain), - ), nil -} diff --git a/logparser/v6/main.go b/logparser/v6/main.go deleted file mode 100644 index 46e7641..0000000 --- a/logparser/v6/main.go +++ /dev/null @@ -1,49 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package main - -import ( - "log" - "os" - - "github.com/inancgumus/learngo/logparser/v6/pipe" - "github.com/inancgumus/learngo/logparser/v6/pipe/filter" - "github.com/inancgumus/learngo/logparser/v6/pipe/group" - "github.com/inancgumus/learngo/logparser/v6/pipe/parse" - "github.com/inancgumus/learngo/logparser/v6/pipe/report" -) - -func main() { - pipe := pipe.New( - parse.FromText(os.Stdin), - // parse.FromJSON(os.Stdin), - report.AsText(os.Stdout), - filter.By(filter.Not(filter.DomainExt("com", "io"))), - group.By(group.Domain), - new(logger), - ) - - if err := pipe.Run(); err != nil { - log.Fatalln(err) - } -} - -type logger struct { - src pipe.Iterator -} - -func (l *logger) Consume(records pipe.Iterator) error { - l.src = records - return nil -} - -func (l *logger) Each(yield func(pipe.Record)) error { - return l.src.Each(func(r pipe.Record) { - yield(r) - }) -} diff --git a/logparser/v6/pipe/filter/domain.go b/logparser/v6/pipe/filter/domain.go deleted file mode 100644 index 4ac10f7..0000000 --- a/logparser/v6/pipe/filter/domain.go +++ /dev/null @@ -1,38 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package filter - -import ( - "strings" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -// DomainExt filters a set of domain extensions. -func DomainExt(domains ...string) Func { - return func(r pipe.Record) bool { - for _, domain := range domains { - if strings.HasSuffix(r.Str("domain"), "."+domain) { - return true - } - } - return false - } -} - -// Domain filters a domain if it contains the given text. -func Domain(text string) Func { - return func(r pipe.Record) bool { - return strings.Contains(r.Str("domain"), text) - } -} - -// DomainOrg filters only the ".org" domains. -func DomainOrg(r pipe.Record) bool { - return strings.HasSuffix(r.Str("domain"), ".org") -} diff --git a/logparser/v6/pipe/filter/filter.go b/logparser/v6/pipe/filter/filter.go deleted file mode 100644 index 9c3e91f..0000000 --- a/logparser/v6/pipe/filter/filter.go +++ /dev/null @@ -1,50 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package filter - -import "github.com/inancgumus/learngo/logparser/v6/pipe" - -// Func represents a filtering pipeline func. -type Func func(pipe.Record) (pass bool) - -// Filter the records. -type Filter struct { - src pipe.Iterator - filters []Func -} - -// By returns a new filter pipeline. -func By(fn ...Func) *Filter { - return &Filter{filters: fn} -} - -// Consume saves the iterator for later processing. -func (f *Filter) Consume(records pipe.Iterator) error { - f.src = records - return nil -} - -// Each yields only the filtered records. -func (f *Filter) Each(yield func(pipe.Record)) error { - return f.src.Each(func(r pipe.Record) { - if !f.check(r) { - return - } - yield(r) - }) -} - -// check all the filters against the record. -func (f *Filter) check(r pipe.Record) bool { - for _, fi := range f.filters { - if !fi(r) { - return false - } - } - return true -} diff --git a/logparser/v6/pipe/filter/noop.go b/logparser/v6/pipe/filter/noop.go deleted file mode 100644 index c9862f2..0000000 --- a/logparser/v6/pipe/filter/noop.go +++ /dev/null @@ -1,15 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package filter - -import "github.com/inancgumus/learngo/logparser/v6/pipe" - -// Noop filter that does nothing. -func Noop(r pipe.Record) bool { - return true -} diff --git a/logparser/v6/pipe/filter/not.go b/logparser/v6/pipe/filter/not.go deleted file mode 100644 index 9f6176a..0000000 --- a/logparser/v6/pipe/filter/not.go +++ /dev/null @@ -1,17 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package filter - -import "github.com/inancgumus/learngo/logparser/v6/pipe" - -// Not reverses a filter. True becomes false, and vice versa. -func Not(filter Func) Func { - return func(r pipe.Record) bool { - return !filter(r) - } -} diff --git a/logparser/v6/pipe/group/domain.go b/logparser/v6/pipe/group/domain.go deleted file mode 100644 index a0e0e29..0000000 --- a/logparser/v6/pipe/group/domain.go +++ /dev/null @@ -1,18 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package group - -import "github.com/inancgumus/learngo/logparser/v6/pipe" - -// Domain groups the records by domain. -// It keeps the other fields intact. -// For example: It returns the page field as well. -// Exercise: Write a solution that removes the unnecessary data. -func Domain(r pipe.Record) string { - return r.Str("domain") -} diff --git a/logparser/v6/pipe/group/group.go b/logparser/v6/pipe/group/group.go deleted file mode 100644 index 8feefa4..0000000 --- a/logparser/v6/pipe/group/group.go +++ /dev/null @@ -1,60 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package group - -import ( - "sort" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -// Func represents a grouping func that returns a grouping key. -type Func func(pipe.Record) (key string) - -// Group records by a key. -type Group struct { - sum map[string]pipe.Record // metrics per group key - keys []string // unique group keys - key Func -} - -// By returns a new Group. -// It takes a group func that returns a group key. -// The returned group will group the record using the key. -func By(key Func) *Group { - return &Group{ - sum: make(map[string]pipe.Record), - key: key, - } -} - -// Consume records for grouping. -func (g *Group) Consume(records pipe.Iterator) error { - return records.Each(func(r pipe.Record) { - k := g.key(r) - - if _, ok := g.sum[k]; !ok { - g.keys = append(g.keys, k) - } - - if r, ok := r.(pipe.Summer); ok { - g.sum[k] = r.Sum(g.sum[k]) - } - }) -} - -// Each sorts and yields the grouped records. -func (g *Group) Each(yield func(pipe.Record)) error { - sort.Strings(g.keys) - - for _, k := range g.keys { - yield(g.sum[k]) - } - - return nil -} diff --git a/logparser/v6/pipe/group/page.go b/logparser/v6/pipe/group/page.go deleted file mode 100644 index 29f6736..0000000 --- a/logparser/v6/pipe/group/page.go +++ /dev/null @@ -1,15 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package group - -import "github.com/inancgumus/learngo/logparser/v6/pipe" - -// Page groups records by page. -func Page(r pipe.Record) string { - return r.Str("domain") + r.Str("page") -} diff --git a/logparser/v6/pipe/logcount.go b/logparser/v6/pipe/logcount.go deleted file mode 100644 index 44ab8e3..0000000 --- a/logparser/v6/pipe/logcount.go +++ /dev/null @@ -1,36 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package pipe - -import "fmt" - -// logCount counts the yielded records. -type logCount struct { - Iterator - n int -} - -// Each yields to the inner iterator while counting the records. -// Reports the record number on an error. -func (lc *logCount) Each(yield func(Record)) error { - err := lc.Iterator.Each(func(r Record) { - lc.n++ - yield(r) - }) - - if err != nil { - // lc.n+1: iterator.each won't call yield on err - return fmt.Errorf("record %d: %v", lc.n+1, err) - } - return nil -} - -// count returns the last read record number. -func (lc *logCount) count() int { - return lc.n -} diff --git a/logparser/v6/pipe/parse/close.go b/logparser/v6/pipe/parse/close.go deleted file mode 100644 index be034c8..0000000 --- a/logparser/v6/pipe/parse/close.go +++ /dev/null @@ -1,19 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package parse - -import ( - "io" -) - -// readClose the reader if it's a io.Closer. -func readClose(r io.Reader) { - if rc, ok := r.(io.Closer); ok { - rc.Close() - } -} diff --git a/logparser/v6/pipe/parse/json.go b/logparser/v6/pipe/parse/json.go deleted file mode 100644 index eff849d..0000000 --- a/logparser/v6/pipe/parse/json.go +++ /dev/null @@ -1,47 +0,0 @@ -// For more tutorials: https://bj.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package parse - -import ( - "encoding/json" - "io" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -// JSON parses json records. -type JSON struct { - reader io.Reader -} - -// FromJSON creates a json parser. -func FromJSON(r io.Reader) *JSON { - return &JSON{reader: r} -} - -// Each yields records from a json reader. -func (j *JSON) Each(yield func(pipe.Record)) error { - defer readClose(j.reader) - - dec := json.NewDecoder(j.reader) - - for { - var r record - - err := dec.Decode(&r) - if err == io.EOF { - break - } - if err != nil { - return err - } - - yield(r) - } - return nil -} diff --git a/logparser/v6/pipe/parse/record.go b/logparser/v6/pipe/parse/record.go deleted file mode 100644 index 8bcb576..0000000 --- a/logparser/v6/pipe/parse/record.go +++ /dev/null @@ -1,116 +0,0 @@ -package parse - -import ( - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -const fieldsLength = 4 - -// record stores fields of a log line. -type record struct { - Domain string - Page string - Visits int - Uniques int -} - -// Str gets a string field by name. -func (r record) Str(field string) string { - switch field { - case "domain": - return r.Domain - case "page": - return r.Page - } - panic(fieldErr(field)) -} - -// Int gets an integer field by name. -func (r record) Int(field string) int { - switch field { - case "visits": - return r.Visits - case "uniques": - return r.Uniques - } - panic(fieldErr(field)) -} - -// Sum the numeric fields with another record. -func (r record) Sum(other pipe.Record) pipe.Record { - if other == nil { - return r - } - r.Visits += other.(record).Visits - r.Uniques += other.(record).Uniques - return r -} - -// UnmarshalText to a *record. -func (r *record) UnmarshalText(p []byte) (err error) { - fields := strings.Fields(string(p)) - if len(fields) != fieldsLength { - return fmt.Errorf("wrong number of fields %q", fields) - } - - r.Domain, r.Page = fields[0], fields[1] - - if r.Visits, err = parseStr("visits", fields[2]); err != nil { - return err - } - if r.Uniques, err = parseStr("uniques", fields[3]); err != nil { - return err - } - return validate(*r) -} - -// UnmarshalJSON to a *record. -func (r *record) UnmarshalJSON(data []byte) error { - // `methodless` doesn't have any methods including UnmarshalJSON. - // This trick prevents the stack-overflow (infinite loop). - type methodless record - - var m methodless - if err := json.Unmarshal(data, &m); err != nil { - return err - } - - // Cast back to the record and save. - *r = record(m) - - return validate(*r) -} - -// parseStr helps UnmarshalText for string to positive int parsing. -func parseStr(name, v string) (int, error) { - n, err := strconv.Atoi(v) - if err != nil { - return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err) - } - return n, nil -} - -// validate whether a parsed record is valid or not. -func validate(r record) (err error) { - switch { - case r.Domain == "": - err = errors.New("record.domain cannot be empty") - case r.Page == "": - err = errors.New("record.page cannot be empty") - case r.Visits < 0: - err = errors.New("record.visits cannot be negative") - case r.Uniques < 0: - err = errors.New("record.uniques cannot be negative") - } - return -} - -func fieldErr(field string) error { - return fmt.Errorf("record field: %q does not exist", field) -} diff --git a/logparser/v6/pipe/parse/text.go b/logparser/v6/pipe/parse/text.go deleted file mode 100644 index 3763717..0000000 --- a/logparser/v6/pipe/parse/text.go +++ /dev/null @@ -1,44 +0,0 @@ -// For more tutorials: https://bp.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package parse - -import ( - "bufio" - "io" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -// Text parses text based log lines. -type Text struct { - reader io.Reader -} - -// FromText creates a text parser. -func FromText(r io.Reader) *Text { - return &Text{reader: r} -} - -// Each yields records from a text log. -func (p *Text) Each(yield func(pipe.Record)) error { - defer readClose(p.reader) - - in := bufio.NewScanner(p.reader) - - for in.Scan() { - r := new(record) - - if err := r.UnmarshalText(in.Bytes()); err != nil { - return err - } - - yield(r) - } - - return in.Err() -} diff --git a/logparser/v6/pipe/pipe.go b/logparser/v6/pipe/pipe.go deleted file mode 100644 index 69b0101..0000000 --- a/logparser/v6/pipe/pipe.go +++ /dev/null @@ -1,26 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package pipe - -// Iterator yields a record. -type Iterator interface { - Each(func(Record)) error -} - -// Consumer consumes records from an iterator. -type Consumer interface { - Consume(Iterator) error -} - -// Transform represents both a record consumer and producer. -// It has an input and output. -// It takes a single record and provides an iterator for all the records. -type Transform interface { - Consumer - Iterator // producer -} diff --git a/logparser/v6/pipe/pipeline.go b/logparser/v6/pipe/pipeline.go deleted file mode 100644 index 4c93f6f..0000000 --- a/logparser/v6/pipe/pipeline.go +++ /dev/null @@ -1,48 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package pipe - -import ( - "fmt" - "os" -) - -// Pipeline takes records from a source, transforms, and sends them to a destionation. -type Pipeline struct { - src Iterator - trans []Transform - dst Consumer -} - -// New creates a new pipeline. -func New(src Iterator, dst Consumer, t ...Transform) *Pipeline { - return &Pipeline{ - src: &logCount{Iterator: src}, - dst: dst, - trans: t, - } -} - -// Run the pipeline. -func (p *Pipeline) Run() error { - defer func() { - n := p.src.(*logCount).count() - fmt.Fprintf(os.Stderr, "%d records processed.\n", n) - }() - - last := p.src - - for _, t := range p.trans { - if err := t.Consume(last); err != nil { - return err - } - last = t - } - - return p.dst.Consume(last) -} diff --git a/logparser/v6/pipe/record.go b/logparser/v6/pipe/record.go deleted file mode 100644 index 5116615..0000000 --- a/logparser/v6/pipe/record.go +++ /dev/null @@ -1,19 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package pipe - -// Record provides a generic interface for any sort of records. -type Record interface { - Str(field string) string - Int(field string) int -} - -// Summer provides a method for summing the numeric fields. -type Summer interface { - Sum(Record) Record -} diff --git a/logparser/v6/pipe/report/chart.go b/logparser/v6/pipe/report/chart.go deleted file mode 100644 index 1302b93..0000000 --- a/logparser/v6/pipe/report/chart.go +++ /dev/null @@ -1,49 +0,0 @@ -package report - -/* -// You need to run: -// go get -u github.com/wcharczuk/go-chart - -// Chart renders a chart. -type Chart struct { - Title string - Width, Height int - - w io.Writer -} - -// AsChart returns a Chart report generator. -func AsChart(w io.Writer) *Chart { - return &Chart{w: w} -} - -// Consume generates a chart report. -func (c *Chart) Consume(records pipe.Iterator) error { - w := os.Stdout - - donut := chart.DonutChart{ - Title: c.Title, - TitleStyle: chart.Style{ - FontSize: 35, - Show: true, - FontColor: chart.ColorAlternateGreen, - }, - Width: c.Width, - Height: c.Height, - } - - records.Each(func(r pipe.Record) { - v := chart.Value{ - Label: r.Str("domain") + r.Str("page") + ": " + strconv.Itoa(r.Int("visits")), - Value: float64(r.Int("visits")), - Style: chart.Style{ - FontSize: 14, - }, - } - - donut.Values = append(donut.Values, v) - }) - - return donut.Render(chart.SVG, w) -} -*/ diff --git a/logparser/v6/pipe/report/text.go b/logparser/v6/pipe/report/text.go deleted file mode 100644 index ecbebf0..0000000 --- a/logparser/v6/pipe/report/text.go +++ /dev/null @@ -1,64 +0,0 @@ -// For more tutorials: https://blog.learngoprogramming.com -// -// Copyright © 2018 Inanc Gumus -// Learn Go Programming Course -// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ -// - -package report - -import ( - "fmt" - "io" - "text/tabwriter" - - "github.com/inancgumus/learngo/logparser/v6/pipe" -) - -const ( - minWidth = 0 - tabWidth = 4 - padding = 4 - flags = 0 -) - -// Text report generator. -type Text struct { - w io.Writer -} - -// AsText returns a Text report generator. -func AsText(w io.Writer) *Text { - return &Text{w: w} -} - -// Consume generates a text report. -func (t *Text) Consume(records pipe.Iterator) error { - w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags) - - write := fmt.Fprintf - - write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") - write(w, "-------\t-----\t------\t-------\n") - - var total pipe.Record - - records.Each(func(r pipe.Record) { - if r, ok := r.(pipe.Summer); ok { - total = r.Sum(total) - } - - write(w, "%s\t%s\t%d\t%d\n", - r.Str("domain"), r.Str("page"), - r.Int("visits"), r.Int("uniques"), - ) - }) - - write(w, "\t\t\t\n") - write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", - total.Int("visits"), - total.Int("uniques"), - ) - - return w.Flush() -}