gaspersic/learngo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: logparser v6

Browse Source
This commit is contained in:
Inanc Gumus
2019-08-31 12:38:50 +03:00
parent b5fbf7d10a
commit 81bd060e1b
15 changed files with 261 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
logparser/v6/parse/count.go → logparser/v6/logly/parse/count.go
View File

@ -21,11 +21,6 @@ func CountRecords(p Parser) *Count {
return &Count{Parser: p}
}
// Last counted record number.
func (c *Count) Last() int {
return c.count - 1
}
// Parse increments the counter.
func (c *Count) Parse() bool {
c.count++
@ -34,8 +29,9 @@ func (c *Count) Parse() bool {
// Err returns the first error that was encountered by the Log.
func (c *Count) Err() (err error) {
if err = c.Parser.Err(); err != nil {
err = fmt.Errorf("record #%d: %v", c.Last()+1, err)
err = c.Parser.Err()
if err != nil {
err = fmt.Errorf("record #%d: %v", c.count, err)
}
return
}

9
logparser/v6/parse/json.go → logparser/v6/logly/parse/json.go
View File

@ -10,20 +10,22 @@ package parse
import (
"encoding/json"
"io"
"github.com/inancgumus/learngo/logparser/v6/logly/record"
)
// JSONParser parses json records.
type JSONParser struct {
in *json.Decoder
err error // last error
last *Record // last parsed record
last *record.Record // last parsed record
}
// JSON creates a json parser.
func JSON(r io.Reader) *JSONParser {
return &JSONParser{
in: json.NewDecoder(r),
last: new(Record),
last: new(record.Record),
}
}
@ -34,6 +36,7 @@ func (p *JSONParser) Parse() bool {
}
p.last.Reset()
err := p.in.Decode(&p.last)
if err == io.EOF {
return false
@ -45,7 +48,7 @@ func (p *JSONParser) Parse() bool {
}
// Value returns the most recent record parsed by a call to Parse.
func (p *JSONParser) Value() Record {
func (p *JSONParser) Value() record.Record {
return *p.last
}

4
logparser/v6/parse/parser.go → logparser/v6/logly/parse/parser.go
View File

@ -7,13 +7,15 @@
package parse
import "github.com/inancgumus/learngo/logparser/v6/logly/record"
// Parser is an interface for the parsers.
type Parser interface {
// Parse the next record from the source.
Parse() bool
// Value returns the last parsed record by a call to Parse.
Value() Record
Value() record.Record
// Err returns the first error that was encountered.
Err() error

14
logparser/v6/parse/text.go → logparser/v6/logly/parse/text.go
View File

@ -10,35 +10,41 @@ package parse
import (
"bufio"
"io"
"github.com/inancgumus/learngo/logparser/v6/logly/record"
)
// TextParser parses text based log lines.
type TextParser struct {
in *bufio.Scanner
err error // last error
last *Record // last parsed record
last *record.Record // last parsed record
}
// Text creates a text parser.
func Text(r io.Reader) *TextParser {
return &TextParser{
in: bufio.NewScanner(r),
last: new(Record),
last: new(record.Record),
}
}
// Parse the next line.
func (p *TextParser) Parse() bool {
if p.err != nil || !p.in.Scan() {
if p.err != nil {
return false
}
if !p.in.Scan() {
return false
}
p.err = p.last.FromText(p.in.Bytes())
return true
}
// Value returns the most recent record parsed by a call to Parse.
func (p *TextParser) Value() Record {
func (p *TextParser) Value() record.Record {
return *p.last
}

22
logparser/v6/logly/record/json.go Normal file
View File

@ -0,0 +1,22 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package record
import "encoding/json"
// UnmarshalJSON to a record.
func (r *Record) UnmarshalJSON(data []byte) error {
type rjson Record
err := json.Unmarshal(data, (*rjson)(r))
if err != nil {
return err
}
return r.validate()
}

25
logparser/v6/logly/record/record.go Normal file
View File

@ -0,0 +1,25 @@
package record
const fieldsLength = 4
// Record stores fields of a log line.
type Record struct {
Domain string
Page string
Visits int
Uniques int
}
// Sum the numeric fields with another record.
func (r *Record) Sum(other Record) {
r.Visits += other.Visits
r.Uniques += other.Uniques
}
// Reset all the fields of this record.
func (r *Record) Reset() {
r.Domain = ""
r.Page = ""
r.Visits = 0
r.Uniques = 0
}

36
logparser/v6/logly/record/sum.go Normal file
View File

@ -0,0 +1,36 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package record
// Sum groups the records by summing their numeric fields.
type Sum struct {
sum map[string]Record
}
// SumGroup the records by domain.
func SumGroup() *Sum {
return &Sum{
sum: make(map[string]Record),
}
}
// Group the record.
func (s *Sum) Group(r Record) {
k := r.Domain
r.Sum(s.sum[k])
s.sum[k] = r
}
// Records returns the grouped records.
func (s *Sum) Records() []Record {
var out []Record
for _, res := range s.sum {
out = append(out, res)
}
return out
}

36
logparser/v6/logly/record/text.go Normal file
View File

@ -0,0 +1,36 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package record
import (
"fmt"
"strconv"
"strings"
)
// FromText unmarshals the log line into this record.
func (r *Record) FromText(p []byte) (err error) {
fields := strings.Fields(string(p))
if len(fields) != fieldsLength {
return fmt.Errorf("wrong number of fields %q", fields)
}
r.Domain = fields[0]
r.Page = fields[1]
const msg = "record.UnmarshalText %q: %v"
if r.Visits, err = strconv.Atoi(fields[2]); err != nil {
return fmt.Errorf(msg, "visits", err)
}
if r.Uniques, err = strconv.Atoi(fields[3]); err != nil {
return fmt.Errorf(msg, "uniques", err)
}
return r.validate()
}

31
logparser/v6/logly/record/validate.go Normal file
View File

@ -0,0 +1,31 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package record
import "errors"
// validate whether the current record is valid or not.
func (r *Record) validate() error {
var msg string
switch {
case r.Domain == "":
msg = "record.domain cannot be empty"
case r.Page == "":
msg = "record.page cannot be empty"
case r.Visits < 0:
msg = "record.visits cannot be negative"
case r.Uniques < 0:
msg = "record.uniques cannot be negative"
}
if msg != "" {
return errors.New(msg)
}
return nil
}

29
logparser/v6/logly/report/json.go Normal file
View File

@ -0,0 +1,29 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"encoding/json"
"io"
"github.com/inancgumus/learngo/logparser/v6/logly/record"
)
// JSON generates a json report.
func JSON(w io.Writer, rs []record.Record) error {
enc := json.NewEncoder(w)
for _, r := range rs {
err := enc.Encode(&r)
if err != nil {
return err
}
}
return nil
}

49
logparser/v6/logly/report/text.go Normal file
View File

@ -0,0 +1,49 @@
// For more tutorials: https://brecord.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
"io"
"text/tabwriter"
"github.com/inancgumus/learngo/logparser/v6/logly/record"
)
// Text generates a text report.
func Text(w io.Writer, rs []record.Record) error {
tw := tabwriter.NewWriter(
w,
0, // minWidth
4, // tabWidth
4, // padding
' ', // padChar
0, // flags
)
fmt.Fprintf(tw, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
fmt.Fprintf(tw, "-------\t-----\t------\t-------\n")
var total record.Record
for _, r := range rs {
total.Sum(r)
fmt.Fprintf(tw, "%s\t%s\t%d\t%d\n",
r.Domain, r.Page,
r.Visits, r.Uniques,
)
}
fmt.Fprintf(tw, "\t\t\t\n")
fmt.Fprintf(tw, "%s\t%s\t%d\t%d\n",
"TOTAL", "",
total.Visits, total.Uniques,
)
return tw.Flush()
}

39
logparser/v6/main.go
View File

@ -8,49 +8,28 @@
package main
import (
"fmt"
"log"
"os"
"github.com/inancgumus/learngo/logparser/v6/parse"
"github.com/inancgumus/learngo/logparser/v6/report"
"github.com/inancgumus/learngo/logparser/v6/logly/parse"
"github.com/inancgumus/learngo/logparser/v6/logly/record"
"github.com/inancgumus/learngo/logparser/v6/logly/report"
)
func main() {
// trace.Start(os.Stderr)
// defer trace.Stop()
var (
p = parse.CountRecords(parse.Text(os.Stdin))
g = record.SumGroup()
)
var p parse.Parser
// p = parse.Text(os.Stdin)
p = parse.JSON(os.Stdin)
p = parse.CountRecords(p)
r := report.Text(os.Stdout)
var out []parse.Record
for p.Parse() {
r := p.Value()
// if !parse.Filter(r) {
// continue
// }
// sum.group(r)
out = append(out, r)
g.Group(p.Value())
}
if err := p.Err(); err != nil {
log.Fatalln(err)
}
// var out []parse.Record
// for sum.More() {
// }
if err := r.Generate(out); err != nil {
if err := report.Text(os.Stdout, g.Records()); err != nil {
log.Fatalln(err)
}
fmt.Println(p.(*parse.Count).Last(), "records are processed.")
}

88
logparser/v6/parse/record.go
View File

@ -1,88 +0,0 @@
package parse
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// Record stores fields of a log line.
type Record struct {
Domain string
Page string
Visits int
Uniques int
}
// Sum the numeric fields with another record.
func (r *Record) Sum(other Record) {
r.Visits += other.Visits
r.Uniques += other.Uniques
}
// Reset all the fields of this record.
func (r *Record) Reset() {
r.Domain = ""
r.Page = ""
r.Visits = 0
r.Uniques = 0
}
// FromText unmarshals the log line into this record.
func (r *Record) FromText(p []byte) (err error) {
fields := strings.Fields(string(p))
if len(fields) != fieldsLength {
return fmt.Errorf("wrong number of fields %q", fields)
}
r.Domain = fields[0]
r.Page = fields[1]
const msg = "record.UnmarshalText %q: %v"
if r.Visits, err = strconv.Atoi(fields[2]); err != nil {
return fmt.Errorf(msg, "visits", err)
}
if r.Uniques, err = strconv.Atoi(fields[3]); err != nil {
return fmt.Errorf(msg, "uniques", err)
}
return r.validate()
}
// UnmarshalJSON to a record.
func (r *Record) UnmarshalJSON(data []byte) error {
type rjson Record
err := json.Unmarshal(data, (*rjson)(r))
if err != nil {
return err
}
return r.validate()
}
// validate whether the current record is valid or not.
func (r *Record) validate() error {
var msg string
switch {
case r.Domain == "":
msg = "record.domain cannot be empty"
case r.Page == "":
msg = "record.page cannot be empty"
case r.Visits < 0:
msg = "record.visits cannot be negative"
case r.Uniques < 0:
msg = "record.uniques cannot be negative"
}
if msg != "" {
return errors.New(msg)
}
return nil
}

37
logparser/v6/report/json.go
View File

@ -1,37 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"encoding/json"
"io"
"github.com/inancgumus/learngo/logparser/v6/parse"
)
// JSONReport generates a JSON report.
type JSONReport struct {
w *json.Encoder
}
// JSON returns a JSON report generator.
func JSON(w io.Writer) *JSONReport {
return &JSONReport{
w: json.NewEncoder(w),
}
}
// Generate the report from the records.
func (jr *JSONReport) Generate(rs []parse.Record) error {
for _, r := range rs {
if err := jr.w.Encode(&r); err != nil {
return err
}
}
return nil
}

60
logparser/v6/report/text.go
View File

@ -1,60 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
"io"
"text/tabwriter"
"github.com/inancgumus/learngo/logparser/v6/parse"
)
// TextReport report generator.
type TextReport struct {
w *tabwriter.Writer
total parse.Record
}
// Text creates a report generator.
func Text(w io.Writer) *TextReport {
tw := tabwriter.NewWriter(w,
0, // minWidth
4, // tabWidth
4, // padding
' ', // padChar
0, // flags
)
return &TextReport{w: tw}
}
// Generate the report from the records.
// tabwriter caches, the memory usage will be high
// if you send a large number of records to Generate.
func (tr *TextReport) Generate(rs []parse.Record) error {
fmt.Fprintf(tr.w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
fmt.Fprintf(tr.w, "-------\t-----\t------\t-------\n")
for _, r := range rs {
tr.total.Sum(r)
fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
r.Domain, r.Page,
r.Visits, r.Uniques,
)
}
fmt.Fprintf(tr.w, "\t\t\t\n")
fmt.Fprintf(tr.w, "%s\t%s\t%d\t%d\n",
"TOTAL", "",
tr.total.Visits, tr.total.Uniques,
)
return tr.w.Flush()
}