add: experimental logparser v6

logparser/logs/log_err_missing.jsonl

@@ -0,0 +1,16 @@
+{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
+{"domain": "learngoprogramming.com", "visits": 15, "uniques": 10}
+{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
+{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
+{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
+{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
+{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
+{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
+{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
+{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

logparser/logs/log_err_negative.jsonl

@@ -0,0 +1,16 @@
+{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
+{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10}
+{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": -15}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
+{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
+{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
+{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
+{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
+{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
+{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
+{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

logparser/logs/log_err_str.jsonl

@@ -0,0 +1,16 @@
+{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
+{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10}
+{"domain": "learngoprogramming.com", "page": "/courses", "visits": "TEN", "uniques": 5}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15}
+{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
+{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
+{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
+{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
+{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
+{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
+{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
+{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
+{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
+{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

logparser/v6/main.go

@@ -0,0 +1,56 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/inancgumus/learngo/logparser/v6/parse"
+	"github.com/inancgumus/learngo/logparser/v6/report"
+)
+
+func main() {
+	// trace.Start(os.Stderr)
+	// defer trace.Stop()
+
+	var p parse.Parser
+	// p = parse.Text(os.Stdin)
+	p = parse.JSON(os.Stdin)
+	p = parse.CountRecords(p)
+
+	r := report.Text(os.Stdout)
+
+	var out []parse.Record
+	for p.Parse() {
+		r := p.Value()
+
+		// if !parse.Filter(r) {
+		// 	continue
+		// }
+
+		// sum.group(r)
+		out = append(out, r)
+	}
+
+	if err := p.Err(); err != nil {
+		log.Fatalln(err)
+	}
+
+	// var out []parse.Record
+	// for sum.More() {
+
+	// }
+
+	if err := r.Generate(out); err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Println(p.(*parse.Count).Last(), "records are processed.")
+}

logparser/v6/parse/count.go

@@ -0,0 +1,44 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package parse
+
+import "fmt"
+
+// Count the parsed records.
+type Count struct {
+	// Parser is wrapped by Count to count the parsed records.
+	Parser
+	count int
+}
+
+// CountRecords creates a record counter that wraps a parser.
+func CountRecords(p Parser) *Count {
+	return &Count{Parser: p}
+}
+
+// Last counted record number.
+func (c *Count) Last() int {
+	return c.count - 1
+}
+
+// Parse increments the counter.
+func (c *Count) Parse() bool {
+	c.count++
+	return c.Parser.Parse()
+}
+
+// Err returns the first error that was encountered by the Log.
+func (c *Count) Err() (err error) {
+	if err = c.Parser.Err(); err != nil {
+		err = fmt.Errorf("record #%d: %v", c.Last()+1, err)
+	}
+	return
+}
+
+// You don't need to implement the Value() method.
+// Thanks to interface embedding.

logparser/v6/parse/json.go

@@ -0,0 +1,55 @@
+// For more tutorials: https://bj.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package parse
+
+import (
+	"encoding/json"
+	"io"
+)
+
+// JSONParser parses json records.
+type JSONParser struct {
+	in   *json.Decoder
+	err  error   // last error
+	last *Record // last parsed record
+}
+
+// JSON creates a json parser.
+func JSON(r io.Reader) *JSONParser {
+	return &JSONParser{
+		in:   json.NewDecoder(r),
+		last: new(Record),
+	}
+}
+
+// Parse the next line.
+func (p *JSONParser) Parse() bool {
+	if p.err != nil {
+		return false
+	}
+
+	p.last.Reset()
+	err := p.in.Decode(&p.last)
+	if err == io.EOF {
+		return false
+	}
+
+	p.err = err
+
+	return err == nil
+}
+
+// Value returns the most recent record parsed by a call to Parse.
+func (p *JSONParser) Value() Record {
+	return *p.last
+}
+
+// Err returns the first error that was encountered by the Log.
+func (p *JSONParser) Err() error {
+	return p.err
+}

logparser/v6/parse/parser.go

@@ -0,0 +1,20 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package parse
+
+// Parser is an interface for the parsers.
+type Parser interface {
+	// Parse the next record from the source.
+	Parse() bool
+
+	// Value returns the last parsed record by a call to Parse.
+	Value() Record
+
+	// Err returns the first error that was encountered.
+	Err() error
+}

logparser/v6/parse/record.go

@@ -0,0 +1,88 @@
+package parse
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const fieldsLength = 4
+
+// Record stores fields of a log line.
+type Record struct {
+	Domain  string
+	Page    string
+	Visits  int
+	Uniques int
+}
+
+// Sum the numeric fields with another record.
+func (r *Record) Sum(other Record) {
+	r.Visits += other.Visits
+	r.Uniques += other.Uniques
+}
+
+// Reset all the fields of this record.
+func (r *Record) Reset() {
+	r.Domain = ""
+	r.Page = ""
+	r.Visits = 0
+	r.Uniques = 0
+}
+
+// FromText unmarshals the log line into this record.
+func (r *Record) FromText(p []byte) (err error) {
+	fields := strings.Fields(string(p))
+
+	if len(fields) != fieldsLength {
+		return fmt.Errorf("wrong number of fields %q", fields)
+	}
+
+	r.Domain = fields[0]
+	r.Page = fields[1]
+
+	const msg = "record.UnmarshalText %q: %v"
+	if r.Visits, err = strconv.Atoi(fields[2]); err != nil {
+		return fmt.Errorf(msg, "visits", err)
+	}
+	if r.Uniques, err = strconv.Atoi(fields[3]); err != nil {
+		return fmt.Errorf(msg, "uniques", err)
+	}
+
+	return r.validate()
+}
+
+// UnmarshalJSON to a record.
+func (r *Record) UnmarshalJSON(data []byte) error {
+	type rjson Record
+
+	err := json.Unmarshal(data, (*rjson)(r))
+	if err != nil {
+		return err
+	}
+
+	return r.validate()
+}
+
+// validate whether the current record is valid or not.
+func (r *Record) validate() error {
+	var msg string
+
+	switch {
+	case r.Domain == "":
+		msg = "record.domain cannot be empty"
+	case r.Page == "":
+		msg = "record.page cannot be empty"
+	case r.Visits < 0:
+		msg = "record.visits cannot be negative"
+	case r.Uniques < 0:
+		msg = "record.uniques cannot be negative"
+	}
+
+	if msg != "" {
+		return errors.New(msg)
+	}
+	return nil
+}

logparser/v6/parse/text.go

@@ -0,0 +1,48 @@
+// For more tutorials: https://bp.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package parse
+
+import (
+	"bufio"
+	"io"
+)
+
+// TextParser parses text based log lines.
+type TextParser struct {
+	in   *bufio.Scanner
+	err  error   // last error
+	last *Record // last parsed record
+}
+
+// Text creates a text parser.
+func Text(r io.Reader) *TextParser {
+	return &TextParser{
+		in:   bufio.NewScanner(r),
+		last: new(Record),
+	}
+}
+
+// Parse the next line.
+func (p *TextParser) Parse() bool {
+	if p.err != nil || !p.in.Scan() {
+		return false
+	}
+
+	p.err = p.last.FromText(p.in.Bytes())
+	return true
+}
+
+// Value returns the most recent record parsed by a call to Parse.
+func (p *TextParser) Value() Record {
+	return *p.last
+}
+
+// Err returns the first error that was encountered by the Log.
+func (p *TextParser) Err() error {
+	return p.err
+}

logparser/v6/report/json.go

@@ -0,0 +1,37 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package report
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/inancgumus/learngo/logparser/v6/parse"
+)
+
+// JSONReport generates a JSON report.
+type JSONReport struct {
+	w *json.Encoder
+}
+
+// JSON returns a JSON report generator.
+func JSON(w io.Writer) *JSONReport {
+	return &JSONReport{
+		w: json.NewEncoder(w),
+	}
+}
+
+// Generate the report from the records.
+func (jr *JSONReport) Generate(rs []parse.Record) error {
+	for _, r := range rs {
+		if err := jr.w.Encode(&r); err != nil {
+			return err
+		}
+	}
+	return nil
+}

logparser/v6/report/text.go

@@ -0,0 +1,60 @@
+// For more tutorials: https://blog.learngoprogramming.com
+//
+// Copyright © 2018 Inanc Gumus
+// Learn Go Programming Course
+// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
+//
+
+package report
+
+import (
+	"fmt"
+	"io"
+	"text/tabwriter"
+
+	"github.com/inancgumus/learngo/logparser/v6/parse"
+)
+
+// TextReport report generator.
+type TextReport struct {
+	w     *tabwriter.Writer
+	total parse.Record
+}
+
+// Text creates a report generator.
+func Text(w io.Writer) *TextReport {
+	tw := tabwriter.NewWriter(w,
+		0,   // minWidth
+		4,   // tabWidth
+		4,   // padding
+		' ', // padChar
+		0,   // flags
+	)
+
+	return &TextReport{w: tw}
+}
+
+// Generate the report from the records.
+// tabwriter caches, the memory usage will be high
+// if you send a large number of records to Generate.
+func (tr *TextReport) Generate(rs []parse.Record) error {
+	fmt.Fprintf(tr.w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
+	fmt.Fprintf(tr.w, "-------\t-----\t------\t-------\n")
+
+	for _, r := range rs {
+		tr.total.Sum(r)
+
+		fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
+			r.Domain, r.Page,
+			r.Visits, r.Uniques,
+		)
+	}
+
+	fmt.Fprintf(tr.w, "\t\t\t\n")
+	fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
+		"TOTAL", "",
+		tr.total.Visits, tr.total.Uniques,
+	)
+
+	return tr.w.Flush()
+}