gaspersic/learngo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add: experimental logparser v6

Browse Source
This commit is contained in:
Inanc Gumus
2019-08-30 21:51:14 +03:00
parent ec39a882c8
commit b5fbf7d10a
11 changed files with 456 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
logparser/logs/log_err_missing.jsonl Normal file
View File

@ -0,0 +1,16 @@
{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "visits": 15, "uniques": 10}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

16
logparser/logs/log_err_negative.jsonl Normal file
View File

@ -0,0 +1,16 @@
{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": -15}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

16
logparser/logs/log_err_str.jsonl Normal file
View File

@ -0,0 +1,16 @@
{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": "TEN", "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

56
logparser/v6/main.go Normal file
View File

@ -0,0 +1,56 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"log"
"os"
"github.com/inancgumus/learngo/logparser/v6/parse"
"github.com/inancgumus/learngo/logparser/v6/report"
)
func main() {
// trace.Start(os.Stderr)
// defer trace.Stop()
var p parse.Parser
// p = parse.Text(os.Stdin)
p = parse.JSON(os.Stdin)
p = parse.CountRecords(p)
r := report.Text(os.Stdout)
var out []parse.Record
for p.Parse() {
r := p.Value()
// if !parse.Filter(r) {
// continue
// }
// sum.group(r)
out = append(out, r)
}
if err := p.Err(); err != nil {
log.Fatalln(err)
}
// var out []parse.Record
// for sum.More() {
// }
if err := r.Generate(out); err != nil {
log.Fatalln(err)
}
fmt.Println(p.(*parse.Count).Last(), "records are processed.")
}

44
logparser/v6/parse/count.go Normal file
View File

@ -0,0 +1,44 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import "fmt"
// Count the parsed records.
type Count struct {
// Parser is wrapped by Count to count the parsed records.
Parser
count int
}
// CountRecords creates a record counter that wraps a parser.
func CountRecords(p Parser) *Count {
return &Count{Parser: p}
}
// Last counted record number.
func (c *Count) Last() int {
return c.count - 1
}
// Parse increments the counter.
func (c *Count) Parse() bool {
c.count++
return c.Parser.Parse()
}
// Err returns the first error that was encountered by the Log.
func (c *Count) Err() (err error) {
if err = c.Parser.Err(); err != nil {
err = fmt.Errorf("record #%d: %v", c.Last()+1, err)
}
return
}
// You don't need to implement the Value() method.
// Thanks to interface embedding.

55
logparser/v6/parse/json.go Normal file
View File

@ -0,0 +1,55 @@
// For more tutorials: https://bj.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import (
"encoding/json"
"io"
)
// JSONParser parses json records.
type JSONParser struct {
in *json.Decoder
err error // last error
last *Record // last parsed record
}
// JSON creates a json parser.
func JSON(r io.Reader) *JSONParser {
return &JSONParser{
in: json.NewDecoder(r),
last: new(Record),
}
}
// Parse the next line.
func (p *JSONParser) Parse() bool {
if p.err != nil {
return false
}
p.last.Reset()
err := p.in.Decode(&p.last)
if err == io.EOF {
return false
}
p.err = err
return err == nil
}
// Value returns the most recent record parsed by a call to Parse.
func (p *JSONParser) Value() Record {
return *p.last
}
// Err returns the first error that was encountered by the Log.
func (p *JSONParser) Err() error {
return p.err
}

20
logparser/v6/parse/parser.go Normal file
View File

@ -0,0 +1,20 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
// Parser is an interface for the parsers.
type Parser interface {
// Parse the next record from the source.
Parse() bool
// Value returns the last parsed record by a call to Parse.
Value() Record
// Err returns the first error that was encountered.
Err() error
}

88
logparser/v6/parse/record.go Normal file
View File

@ -0,0 +1,88 @@
package parse
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// Record stores fields of a log line.
type Record struct {
Domain string
Page string
Visits int
Uniques int
}
// Sum the numeric fields with another record.
func (r *Record) Sum(other Record) {
r.Visits += other.Visits
r.Uniques += other.Uniques
}
// Reset all the fields of this record.
func (r *Record) Reset() {
r.Domain = ""
r.Page = ""
r.Visits = 0
r.Uniques = 0
}
// FromText unmarshals the log line into this record.
func (r *Record) FromText(p []byte) (err error) {
fields := strings.Fields(string(p))
if len(fields) != fieldsLength {
return fmt.Errorf("wrong number of fields %q", fields)
}
r.Domain = fields[0]
r.Page = fields[1]
const msg = "record.UnmarshalText %q: %v"
if r.Visits, err = strconv.Atoi(fields[2]); err != nil {
return fmt.Errorf(msg, "visits", err)
}
if r.Uniques, err = strconv.Atoi(fields[3]); err != nil {
return fmt.Errorf(msg, "uniques", err)
}
return r.validate()
}
// UnmarshalJSON to a record.
func (r *Record) UnmarshalJSON(data []byte) error {
type rjson Record
err := json.Unmarshal(data, (*rjson)(r))
if err != nil {
return err
}
return r.validate()
}
// validate whether the current record is valid or not.
func (r *Record) validate() error {
var msg string
switch {
case r.Domain == "":
msg = "record.domain cannot be empty"
case r.Page == "":
msg = "record.page cannot be empty"
case r.Visits < 0:
msg = "record.visits cannot be negative"
case r.Uniques < 0:
msg = "record.uniques cannot be negative"
}
if msg != "" {
return errors.New(msg)
}
return nil
}

48
logparser/v6/parse/text.go Normal file
View File

@ -0,0 +1,48 @@
// For more tutorials: https://bp.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import (
"bufio"
"io"
)
// TextParser parses text based log lines.
type TextParser struct {
in *bufio.Scanner
err error // last error
last *Record // last parsed record
}
// Text creates a text parser.
func Text(r io.Reader) *TextParser {
return &TextParser{
in: bufio.NewScanner(r),
last: new(Record),
}
}
// Parse the next line.
func (p *TextParser) Parse() bool {
if p.err != nil || !p.in.Scan() {
return false
}
p.err = p.last.FromText(p.in.Bytes())
return true
}
// Value returns the most recent record parsed by a call to Parse.
func (p *TextParser) Value() Record {
return *p.last
}
// Err returns the first error that was encountered by the Log.
func (p *TextParser) Err() error {
return p.err
}

37
logparser/v6/report/json.go Normal file
View File

@ -0,0 +1,37 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"encoding/json"
"io"
"github.com/inancgumus/learngo/logparser/v6/parse"
)
// JSONReport generates a JSON report.
type JSONReport struct {
w *json.Encoder
}
// JSON returns a JSON report generator.
func JSON(w io.Writer) *JSONReport {
return &JSONReport{
w: json.NewEncoder(w),
}
}
// Generate the report from the records.
func (jr *JSONReport) Generate(rs []parse.Record) error {
for _, r := range rs {
if err := jr.w.Encode(&r); err != nil {
return err
}
}
return nil
}

60
logparser/v6/report/text.go Normal file
View File

@ -0,0 +1,60 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
"io"
"text/tabwriter"
"github.com/inancgumus/learngo/logparser/v6/parse"
)
// TextReport report generator.
type TextReport struct {
w *tabwriter.Writer
total parse.Record
}
// Text creates a report generator.
func Text(w io.Writer) *TextReport {
tw := tabwriter.NewWriter(w,
0, // minWidth
4, // tabWidth
4, // padding
' ', // padChar
0, // flags
)
return &TextReport{w: tw}
}
// Generate the report from the records.
// tabwriter caches, the memory usage will be high
// if you send a large number of records to Generate.
func (tr *TextReport) Generate(rs []parse.Record) error {
fmt.Fprintf(tr.w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
fmt.Fprintf(tr.w, "-------\t-----\t------\t-------\n")
for _, r := range rs {
tr.total.Sum(r)
fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
r.Domain, r.Page,
r.Visits, r.Uniques,
)
}
fmt.Fprintf(tr.w, "\t\t\t\n")
fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
"TOTAL", "",
tr.total.Visits, tr.total.Uniques,
)
return tr.w.Flush()
}