From b5fbf7d10a6102bbb8e5c8515be4e0b1ded3b11e Mon Sep 17 00:00:00 2001 From: Inanc Gumus Date: Fri, 30 Aug 2019 21:51:14 +0300 Subject: [PATCH] add: experimental logparser v6 --- logparser/logs/log_err_missing.jsonl | 16 +++++ logparser/logs/log_err_negative.jsonl | 16 +++++ logparser/logs/log_err_str.jsonl | 16 +++++ logparser/v6/main.go | 56 +++++++++++++++++ logparser/v6/parse/count.go | 44 ++++++++++++++ logparser/v6/parse/json.go | 55 +++++++++++++++++ logparser/v6/parse/parser.go | 20 ++++++ logparser/v6/parse/record.go | 88 +++++++++++++++++++++++++++ logparser/v6/parse/text.go | 48 +++++++++++++++ logparser/v6/report/json.go | 37 +++++++++++ logparser/v6/report/text.go | 60 ++++++++++++++++++ 11 files changed, 456 insertions(+) create mode 100644 logparser/logs/log_err_missing.jsonl create mode 100644 logparser/logs/log_err_negative.jsonl create mode 100644 logparser/logs/log_err_str.jsonl create mode 100644 logparser/v6/main.go create mode 100644 logparser/v6/parse/count.go create mode 100644 logparser/v6/parse/json.go create mode 100644 logparser/v6/parse/parser.go create mode 100644 logparser/v6/parse/record.go create mode 100644 logparser/v6/parse/text.go create mode 100644 logparser/v6/report/json.go create mode 100644 logparser/v6/report/text.go diff --git a/logparser/logs/log_err_missing.jsonl b/logparser/logs/log_err_missing.jsonl new file mode 100644 index 0000000..668e8f7 --- /dev/null +++ b/logparser/logs/log_err_missing.jsonl @@ -0,0 +1,16 @@ +{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5} +{"domain": "learngoprogramming.com", "visits": 15, "uniques": 10} +{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2} +{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20} +{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10} +{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25} +{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5} +{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30} +{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20} +{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10} +{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35} +{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5} +{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15} +{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35} \ No newline at end of file diff --git a/logparser/logs/log_err_negative.jsonl b/logparser/logs/log_err_negative.jsonl new file mode 100644 index 0000000..40a2277 --- /dev/null +++ b/logparser/logs/log_err_negative.jsonl @@ -0,0 +1,16 @@ +{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5} +{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10} +{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": -15} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2} +{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20} +{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10} +{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25} +{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5} +{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30} +{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20} +{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10} +{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35} +{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5} +{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15} +{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35} \ No newline at end of file diff --git a/logparser/logs/log_err_str.jsonl b/logparser/logs/log_err_str.jsonl new file mode 100644 index 0000000..0540fec --- /dev/null +++ b/logparser/logs/log_err_str.jsonl @@ -0,0 +1,16 @@ +{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5} +{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10} +{"domain": "learngoprogramming.com", "page": "/courses", "visits": "TEN", "uniques": 5} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15} +{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2} +{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20} +{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10} +{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25} +{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5} +{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30} +{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20} +{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10} +{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35} +{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5} +{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15} +{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35} \ No newline at end of file diff --git a/logparser/v6/main.go b/logparser/v6/main.go new file mode 100644 index 0000000..f90936e --- /dev/null +++ b/logparser/v6/main.go @@ -0,0 +1,56 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package main + +import ( + "fmt" + "log" + "os" + + "github.com/inancgumus/learngo/logparser/v6/parse" + "github.com/inancgumus/learngo/logparser/v6/report" +) + +func main() { + // trace.Start(os.Stderr) + // defer trace.Stop() + + var p parse.Parser + // p = parse.Text(os.Stdin) + p = parse.JSON(os.Stdin) + p = parse.CountRecords(p) + + r := report.Text(os.Stdout) + + var out []parse.Record + for p.Parse() { + r := p.Value() + + // if !parse.Filter(r) { + // continue + // } + + // sum.group(r) + out = append(out, r) + } + + if err := p.Err(); err != nil { + log.Fatalln(err) + } + + // var out []parse.Record + // for sum.More() { + + // } + + if err := r.Generate(out); err != nil { + log.Fatalln(err) + } + + fmt.Println(p.(*parse.Count).Last(), "records are processed.") +} diff --git a/logparser/v6/parse/count.go b/logparser/v6/parse/count.go new file mode 100644 index 0000000..95582ae --- /dev/null +++ b/logparser/v6/parse/count.go @@ -0,0 +1,44 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package parse + +import "fmt" + +// Count the parsed records. +type Count struct { + // Parser is wrapped by Count to count the parsed records. + Parser + count int +} + +// CountRecords creates a record counter that wraps a parser. +func CountRecords(p Parser) *Count { + return &Count{Parser: p} +} + +// Last counted record number. +func (c *Count) Last() int { + return c.count - 1 +} + +// Parse increments the counter. +func (c *Count) Parse() bool { + c.count++ + return c.Parser.Parse() +} + +// Err returns the first error that was encountered by the Log. +func (c *Count) Err() (err error) { + if err = c.Parser.Err(); err != nil { + err = fmt.Errorf("record #%d: %v", c.Last()+1, err) + } + return +} + +// You don't need to implement the Value() method. +// Thanks to interface embedding. diff --git a/logparser/v6/parse/json.go b/logparser/v6/parse/json.go new file mode 100644 index 0000000..703283b --- /dev/null +++ b/logparser/v6/parse/json.go @@ -0,0 +1,55 @@ +// For more tutorials: https://bj.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package parse + +import ( + "encoding/json" + "io" +) + +// JSONParser parses json records. +type JSONParser struct { + in *json.Decoder + err error // last error + last *Record // last parsed record +} + +// JSON creates a json parser. +func JSON(r io.Reader) *JSONParser { + return &JSONParser{ + in: json.NewDecoder(r), + last: new(Record), + } +} + +// Parse the next line. +func (p *JSONParser) Parse() bool { + if p.err != nil { + return false + } + + p.last.Reset() + err := p.in.Decode(&p.last) + if err == io.EOF { + return false + } + + p.err = err + + return err == nil +} + +// Value returns the most recent record parsed by a call to Parse. +func (p *JSONParser) Value() Record { + return *p.last +} + +// Err returns the first error that was encountered by the Log. +func (p *JSONParser) Err() error { + return p.err +} diff --git a/logparser/v6/parse/parser.go b/logparser/v6/parse/parser.go new file mode 100644 index 0000000..251c372 --- /dev/null +++ b/logparser/v6/parse/parser.go @@ -0,0 +1,20 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package parse + +// Parser is an interface for the parsers. +type Parser interface { + // Parse the next record from the source. + Parse() bool + + // Value returns the last parsed record by a call to Parse. + Value() Record + + // Err returns the first error that was encountered. + Err() error +} diff --git a/logparser/v6/parse/record.go b/logparser/v6/parse/record.go new file mode 100644 index 0000000..e8c3d5e --- /dev/null +++ b/logparser/v6/parse/record.go @@ -0,0 +1,88 @@ +package parse + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" +) + +const fieldsLength = 4 + +// Record stores fields of a log line. +type Record struct { + Domain string + Page string + Visits int + Uniques int +} + +// Sum the numeric fields with another record. +func (r *Record) Sum(other Record) { + r.Visits += other.Visits + r.Uniques += other.Uniques +} + +// Reset all the fields of this record. +func (r *Record) Reset() { + r.Domain = "" + r.Page = "" + r.Visits = 0 + r.Uniques = 0 +} + +// FromText unmarshals the log line into this record. +func (r *Record) FromText(p []byte) (err error) { + fields := strings.Fields(string(p)) + + if len(fields) != fieldsLength { + return fmt.Errorf("wrong number of fields %q", fields) + } + + r.Domain = fields[0] + r.Page = fields[1] + + const msg = "record.UnmarshalText %q: %v" + if r.Visits, err = strconv.Atoi(fields[2]); err != nil { + return fmt.Errorf(msg, "visits", err) + } + if r.Uniques, err = strconv.Atoi(fields[3]); err != nil { + return fmt.Errorf(msg, "uniques", err) + } + + return r.validate() +} + +// UnmarshalJSON to a record. +func (r *Record) UnmarshalJSON(data []byte) error { + type rjson Record + + err := json.Unmarshal(data, (*rjson)(r)) + if err != nil { + return err + } + + return r.validate() +} + +// validate whether the current record is valid or not. +func (r *Record) validate() error { + var msg string + + switch { + case r.Domain == "": + msg = "record.domain cannot be empty" + case r.Page == "": + msg = "record.page cannot be empty" + case r.Visits < 0: + msg = "record.visits cannot be negative" + case r.Uniques < 0: + msg = "record.uniques cannot be negative" + } + + if msg != "" { + return errors.New(msg) + } + return nil +} diff --git a/logparser/v6/parse/text.go b/logparser/v6/parse/text.go new file mode 100644 index 0000000..df1d150 --- /dev/null +++ b/logparser/v6/parse/text.go @@ -0,0 +1,48 @@ +// For more tutorials: https://bp.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package parse + +import ( + "bufio" + "io" +) + +// TextParser parses text based log lines. +type TextParser struct { + in *bufio.Scanner + err error // last error + last *Record // last parsed record +} + +// Text creates a text parser. +func Text(r io.Reader) *TextParser { + return &TextParser{ + in: bufio.NewScanner(r), + last: new(Record), + } +} + +// Parse the next line. +func (p *TextParser) Parse() bool { + if p.err != nil || !p.in.Scan() { + return false + } + + p.err = p.last.FromText(p.in.Bytes()) + return true +} + +// Value returns the most recent record parsed by a call to Parse. +func (p *TextParser) Value() Record { + return *p.last +} + +// Err returns the first error that was encountered by the Log. +func (p *TextParser) Err() error { + return p.err +} diff --git a/logparser/v6/report/json.go b/logparser/v6/report/json.go new file mode 100644 index 0000000..4d1686d --- /dev/null +++ b/logparser/v6/report/json.go @@ -0,0 +1,37 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package report + +import ( + "encoding/json" + "io" + + "github.com/inancgumus/learngo/logparser/v6/parse" +) + +// JSONReport generates a JSON report. +type JSONReport struct { + w *json.Encoder +} + +// JSON returns a JSON report generator. +func JSON(w io.Writer) *JSONReport { + return &JSONReport{ + w: json.NewEncoder(w), + } +} + +// Generate the report from the records. +func (jr *JSONReport) Generate(rs []parse.Record) error { + for _, r := range rs { + if err := jr.w.Encode(&r); err != nil { + return err + } + } + return nil +} diff --git a/logparser/v6/report/text.go b/logparser/v6/report/text.go new file mode 100644 index 0000000..c347841 --- /dev/null +++ b/logparser/v6/report/text.go @@ -0,0 +1,60 @@ +// For more tutorials: https://blog.learngoprogramming.com +// +// Copyright © 2018 Inanc Gumus +// Learn Go Programming Course +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ +// + +package report + +import ( + "fmt" + "io" + "text/tabwriter" + + "github.com/inancgumus/learngo/logparser/v6/parse" +) + +// TextReport report generator. +type TextReport struct { + w *tabwriter.Writer + total parse.Record +} + +// Text creates a report generator. +func Text(w io.Writer) *TextReport { + tw := tabwriter.NewWriter(w, + 0, // minWidth + 4, // tabWidth + 4, // padding + ' ', // padChar + 0, // flags + ) + + return &TextReport{w: tw} +} + +// Generate the report from the records. +// tabwriter caches, the memory usage will be high +// if you send a large number of records to Generate. +func (tr *TextReport) Generate(rs []parse.Record) error { + fmt.Fprintf(tr.w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") + fmt.Fprintf(tr.w, "-------\t-----\t------\t-------\n") + + for _, r := range rs { + tr.total.Sum(r) + + fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n", + r.Domain, r.Page, + r.Visits, r.Uniques, + ) + } + + fmt.Fprintf(tr.w, "\t\t\t\n") + fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n", + "TOTAL", "", + tr.total.Visits, tr.total.Uniques, + ) + + return tr.w.Flush() +}