gaspersic/learngo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update: methods log parser

Browse Source
This commit is contained in:
Inanc Gumus
2019-05-10 14:20:58 +03:00
parent 80cf86da6d
commit d979da2d44
21 changed files with 241 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
28-methods/xxx-log-parser-methods/log.txt
View File

@ -1,6 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 10
golang.org 4
golang.org 6
blog.golang.org 20
blog.golang.org 10

6
28-methods/xxx-log-parser-methods/log_err_missing.txt
View File

@ -1,6 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 10
golang.org
golang.org 6
blog.golang.org 20
blog.golang.org 10

6
28-methods/xxx-log-parser-methods/log_err_negative.txt
View File

@ -1,6 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 10
golang.org -100
golang.org 6
blog.golang.org 20
blog.golang.org 10

6
28-methods/xxx-log-parser-methods/log_err_str.txt
View File

@ -1,6 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 10
golang.org FOUR
golang.org 6
blog.golang.org 20
blog.golang.org 10

29
28-methods/xxx-log-parser-methods/packaged/main.go
View File

@ -1,29 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"os"
"github.com/inancgumus/learngo/28-methods/xxx-log-parser-methods/packaged/metrics"
)
func main() {
in := bufio.NewScanner(os.Stdin)
parser, report := metrics.NewParser(), metrics.NewReport()
for in.Scan() {
report.Update(parser.Parse(in.Text()))
}
summarize(report, parser.Err(), in.Err())
// s, _ := json.Marshal(report)
// fmt.Println(string(s))
}

52
28-methods/xxx-log-parser-methods/packaged/metrics/parser.go
View File

@ -1,52 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package metrics
import (
"fmt"
)
// Parser keep tracks of the parsing
type Parser struct {
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
}
// Parsed wraps a result for generating a parser error
type Parsed struct {
result // use struct embedding
err error // inject an error
}
// NewParser returns a new parser
func NewParser() *Parser {
return new(Parser)
}
// Parse parses a log line and returns a result with an injected error
func (p *Parser) Parse(line string) (parsed Parsed) {
// always set the error
defer func() { parsed.err = p.lerr }()
// if there was an error do not continue
if p.lerr != nil {
return
}
// chain the parser's error to the result's
res, err := parse(line)
if p.lines++; err != nil {
p.lerr = fmt.Errorf("%s: (line #%d)", err, p.lines)
}
return Parsed{result: res}
}
// Err returns the last error encountered
func (p *Parser) Err() error {
return p.lerr
}

90
28-methods/xxx-log-parser-methods/packaged/metrics/report.go
View File

@ -1,90 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package metrics
import (
"encoding/json"
"sort"
)
// Report aggregates the final report
type Report struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
total result // total visits for all domains
}
// NewReport constructs and initializes a new report
// You can't use its methods without pointer mechanics
func NewReport() *Report {
return &Report{sum: make(map[string]result)}
}
// Update updates the report for the given parsing result
func (r *Report) Update(p Parsed) {
// do not update the report if the result has an error
if p.err != nil {
return
}
domain := p.Domain
if _, ok := r.sum[domain]; !ok {
r.domains = append(r.domains, domain)
}
// let the result handle the addition
// this allows us to manage the result in once place
// and this way it becomes easily extendable
r.total = r.total.add(p.result)
r.sum[domain] = p.add(r.sum[domain])
}
// Iterator returns `next()` to detect when the iteration ends,
// and a `cur()` to return the current result.
// iterator iterates sorted by domains.
func (r *Report) Iterator() (next func() bool, cur func() result) {
sort.Strings(r.domains)
// remember the last iterated result
var last int
next = func() bool {
defer func() { last++ }()
return len(r.domains) > last
}
cur = func() result {
// returns a copy so the caller cannot change it
name := r.domains[last-1]
return r.sum[name]
}
return
}
// Total returns the total metrics
func (r *Report) Total() Parsed {
return Parsed{result: r.total}
}
// MarshalJSON marshals a report to JSON
// Alternative: unexported embedding
func (r *Report) MarshalJSON() ([]byte, error) {
type total struct {
*result
IgnoreDomain *string `json:"Domain,omitempty"`
}
return json.Marshal(struct {
Sum map[string]result
Domains []string
Total total
}{
Sum: r.sum, Domains: r.domains, Total: total{result: &r.total},
})
}

48
28-methods/xxx-log-parser-methods/packaged/summarize.go
View File

@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strings"
"github.com/inancgumus/learngo/28-methods/xxx-log-parser-methods/packaged/metrics"
)
// summarize prints the report and errors if any
func summarize(rep *metrics.Report, errs ...error) {
// TODO: make it strings.Builder
const format = "%-30s %10s %20s\n"
const formatValue = "%-30s %10d %20d\n"
fmt.Printf(format, "DOMAIN", "VISITS", "TIME SPENT")
fmt.Println(strings.Repeat("-", 65))
next, cur := rep.Iterator()
for next() {
rec := cur()
fmt.Printf(formatValue, rec.Domain, rec.Visits, rec.TimeSpent)
}
fmt.Printf("\n"+formatValue, "TOTAL",
rep.Total().Visits, rep.Total().TimeSpent,
)
// only handle the errors once
dumpErrs(errs...)
}
// this variadic func simplifies the multiple error handling
func dumpErrs(errs ...error) {
for _, err := range errs {
if err != nil {
fmt.Printf("> Err: %s\n", err)
}
}
}

40
28-methods/xxx-log-parser-methods/parser.go
View File

@ -1,40 +0,0 @@
package main
import (
"fmt"
)
// parser keep tracks of the parsing
type parser struct {
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
}
// parsed wraps a result for generating parser error
type parsed struct {
result // use struct embedding
err error // inject an error
}
// parse parses a log line and returns a result with an injected error
func (p *parser) parse(line string) (pv parsed) {
// always set the error
defer func() { pv.err = p.lerr }()
// if there was an error do not continue
if p.lerr != nil {
return
}
// chain the parser's error to the result's
res, err := parseLine(line)
if p.lines++; err != nil {
p.lerr = fmt.Errorf("%s: (line #%d)", err, p.lines)
}
return parsed{result: res}
}
// err returns the last error encountered
func (p *parser) err() error {
return p.lerr
}

64
28-methods/xxx-log-parser-methods/report.go
View File

@ -1,64 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import "sort"
// report aggregates the final report
type report struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
total result // total visits for all domains
}
// newReport constructs and initializes a new report
func newReport() *report {
return &report{sum: make(map[string]result)}
}
// update updates the errors for the given parsing result
func (r *report) update(p parsed) {
// do not update the report if the result has an error
if p.err != nil {
return
}
domain := p.domain
if _, ok := r.sum[domain]; !ok {
r.domains = append(r.domains, domain)
}
// let the result handle the addition
// this allows us to manage the result in once place
// and this way it becomes easily extendable
r.total = r.total.add(p.result)
r.sum[domain] = p.add(r.sum[domain])
}
// iterator returns `next()` to detect when the iteration ends,
// and a `cur()` to return the current result.
// iterator iterates sorted by domains.
func (r *report) iterator() (next func() bool, cur func() result) {
sort.Strings(r.domains)
// remember the last iterated result
var last int
next = func() bool {
defer func() { last++ }()
return len(r.domains) > last
}
cur = func() result {
// returns a copy so the caller cannot change it
name := r.domains[last-1]
return r.sum[name]
}
return
}

51
28-methods/xxx-log-parser-methods/result.go
View File

@ -1,51 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
// always put all the related things together as in here
// result stores metrics for a domain
// it uses the value mechanics,
// because it doesn't have to update anything
type result struct {
domain string
visits int
// add more metrics if needed
}
// add adds the metrics of another result to itself and returns a new result
func (r result) add(other result) result {
return result{
domain: r.domain,
visits: r.visits + other.visits,
}
}
// parseLine parses a single result line
func parseLine(line string) (p result, err error) {
fields := strings.Fields(line)
if len(fields) != 2 {
err = fmt.Errorf("wrong input: %v", fields)
return
}
p.domain = fields[0]
p.visits, err = strconv.Atoi(fields[1])
if p.visits < 0 || err != nil {
err = fmt.Errorf("wrong input: %q", fields[1])
}
return
}

38
28-methods/xxx-log-parser-methods/summarize.go
View File

@ -1,38 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strings"
)
// summarize prints the report and errors if any
func summarize(rep *report, errs ...error) {
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
fmt.Println(strings.Repeat("-", 45))
next, cur := rep.iterator()
for next() {
rec := cur()
fmt.Printf("%-30s %10d\n", rec.domain, rec.visits)
}
fmt.Printf("\n%-30s %10d\n", "TOTAL", rep.total.visits)
// only handle the errors once
dumpErrs(errs...)
}
// this variadic func simplifies the multiple error handling
func dumpErrs(errs ...error) {
for _, err := range errs {
if err != nil {
fmt.Printf("> Err: %s\n", err)
}
}
}

0
28-methods/xxx-log-parser-methods/packaged/log.txt → 28-methods/xxx-log-parser/log.txt
View File

0
28-methods/xxx-log-parser-methods/packaged/log_err_missing.txt → 28-methods/xxx-log-parser/log_err_missing.txt
View File

2
28-methods/xxx-log-parser-methods/packaged/log_err_negative.txt → 28-methods/xxx-log-parser/log_err_negative.txt
View File

@ -1,5 +1,5 @@
learngoprogramming.com 10 200 learngoprogramming.com 10 200
learngoprogramming.com 10 -500 learngoprogramming.com 10 300
golang.org -100 50 golang.org -100 50
golang.org 6 100 golang.org 6 100
blog.golang.org 20 25 blog.golang.org 20 25

0
28-methods/xxx-log-parser-methods/packaged/log_err_str.txt → 28-methods/xxx-log-parser/log_err_str.txt
View File

8
28-methods/xxx-log-parser-methods/main.go → 28-methods/xxx-log-parser/main.go
View File

@ -10,15 +10,17 @@ package main
import ( import (
"bufio" "bufio"
"os" "os"
"github.com/inancgumus/learngo/28-methods/xxx-log-parser/report"
) )
func main() { func main() {
in := bufio.NewScanner(os.Stdin) in := bufio.NewScanner(os.Stdin)
p, report := new(parser), newReport() r := report.New()
for in.Scan() { for in.Scan() {
report.update(p.parse(in.Text())) r.Parse(in.Text())
} }
summarize(report, p.err(), in.Err()) summarize(r.Summarize(), r.Err(), in.Err())
} }

52
28-methods/xxx-log-parser/report/parser.go Normal file
View File

@ -0,0 +1,52 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
)
// Parser parses the log file and generates a summary report.
type Parser struct {
summary *Summary // summarizes the parsing results
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
}
// New returns a new parsing state.
func New() *Parser {
return &Parser{summary: newSummary()}
}
// Parse parses a log line and adds it to the summary.
func (p *Parser) Parse(line string) {
// if there was an error do not continue
if p.lerr != nil {
return
}
// chain the parser's error to the result's
res, err := parse(line)
if p.lines++; err != nil {
p.lerr = fmt.Errorf("line #%d: %s", p.lines, err)
return
}
p.summary.update(res)
}
// Summarize summarizes the parsing results.
// Only use it after the parsing is done.
func (p *Parser) Summarize() *Summary {
return p.summary
}
// Err returns the last error encountered
func (p *Parser) Err() error {
return p.lerr
}

48
28-methods/xxx-log-parser-methods/packaged/metrics/result.go → 28-methods/xxx-log-parser/report/result.go
View File

@ -5,7 +5,7 @@
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ // License: https://creativecommons.org/licenses/by-nc-sa/4.0/
// //
package metrics package report
import ( import (
"fmt" "fmt"
@ -15,19 +15,19 @@ import (
// always put all the related things together as in here // always put all the related things together as in here
// result stores metrics for a domain // Result stores metrics for a domain
// it uses the value mechanics, // it uses the value mechanics,
// because it doesn't have to update anything // because it doesn't have to update anything
type result struct { type Result struct {
Domain string Domain string `json:"domain"`
Visits int Visits int `json:"visits"`
TimeSpent int TimeSpent int `json:"time_spent"`
// add more metrics if needed // add more metrics if needed
} }
// add adds the metrics of another Result to itself and returns a new Result // add adds the metrics of another Result to itself and returns a new Result
func (r result) add(other result) result { func (r Result) add(other Result) Result {
return result{ return Result{
Domain: r.Domain, Domain: r.Domain,
Visits: r.Visits + other.Visits, Visits: r.Visits + other.Visits,
TimeSpent: r.TimeSpent + other.TimeSpent, TimeSpent: r.TimeSpent + other.TimeSpent,
@ -35,24 +35,26 @@ func (r result) add(other result) result {
} }
// parse parses a single log line // parse parses a single log line
func parse(line string) (r result, err error) { func parse(line string) (r Result, err error) {
fields := strings.Fields(line) fields := strings.Fields(line)
if len(fields) != 3 { if len(fields) != 3 {
err = fmt.Errorf("wrong input: %v", fields) return r, fmt.Errorf("missing fields: %v", fields)
return
} }
f := new(field)
r.Domain = fields[0] r.Domain = fields[0]
r.Visits = f.atoi("visits", fields[1])
r.Visits, err = strconv.Atoi(fields[1]) r.TimeSpent = f.atoi("time spent", fields[2])
if r.Visits < 0 || err != nil { return r, f.err
err = fmt.Errorf("wrong input: %q", fields[1]) }
}
// field helps for field parsing
r.TimeSpent, err = strconv.Atoi(fields[2]) type field struct{ err error }
if r.TimeSpent < 0 || err != nil {
err = fmt.Errorf("wrong input: %q", fields[2]) func (f *field) atoi(name, val string) int {
} n, err := strconv.Atoi(val)
if n < 0 || err != nil {
return f.err = fmt.Errorf("incorrect %s: %q", name, val)
}
return n
} }

85
28-methods/xxx-log-parser/report/summary.go Normal file
View File

@ -0,0 +1,85 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"encoding/json"
"sort"
)
// Summary aggregates the parsing results
type Summary struct {
sum map[string]Result // metrics per domain
domains []string // unique domain names
total Result // total visits for all domains
}
// newSummary constructs and initializes a new summary
// You can't use its methods without pointer mechanics
func newSummary() *Summary {
return &Summary{sum: make(map[string]Result)}
}
// Update updates the report for the given parsing result
func (s *Summary) update(parsed Result) {
domain := parsed.Domain
if _, ok := s.sum[domain]; !ok {
s.domains = append(s.domains, domain)
}
// let the result handle the addition
// this allows us to manage the result in once place
// and this way it becomes easily extendable
s.total = s.total.add(parsed)
s.sum[domain] = parsed.add(s.sum[domain])
}
// Iterator returns `next()` to detect when the iteration ends,
// and a `cur()` to return the current result.
// iterator iterates sorted by domains.
func (s *Summary) Iterator() (next func() bool, cur func() Result) {
sort.Strings(s.domains)
// remember the last iterated result
var last int
next = func() bool {
defer func() { last++ }()
return len(s.domains) > last
}
cur = func() Result {
// returns a copy so the caller cannot change it
name := s.domains[last-1]
return s.sum[name]
}
return
}
// Total returns the total metrics
func (s *Summary) Total() Result {
return s.total
}
// MarshalJSON marshals a report to JSON
// Alternative: unexported embedding
func (s *Summary) MarshalJSON() ([]byte, error) {
type total struct {
*Result
IgnoreDomain *string `json:"domain,omitempty"`
}
return json.Marshal(struct {
Sum map[string]Result `json:"summary"`
Domains []string `json:"domains"`
Total total `json:"total"`
}{
Sum: s.sum, Domains: s.domains, Total: total{Result: &s.total},
})
}

73
28-methods/xxx-log-parser/summarize.go Normal file
View File

@ -0,0 +1,73 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/inancgumus/learngo/28-methods/xxx-log-parser/report"
)
// summarize prints the parsing results.
//
// it prints the errors and returns if there are any.
//
// --json flag encodes to json and prints.
func summarize(sum *report.Summary, errors ...error) {
if errs(errors...) {
return
}
if args := os.Args[1:]; len(args) == 1 && args[0] == "--json" {
encode(sum)
return
}
stdout(sum)
}
// encodes the summary to json
func encode(sum *report.Summary) {
out, err := json.MarshalIndent(sum, "", "\t")
if err != nil {
panic(err)
}
os.Stdout.Write(out)
}
// prints the summary to standard out
func stdout(sum *report.Summary) {
const format = "%-30s %10s %20s\n"
const formatValue = "%-30s %10d %20d\n"
fmt.Printf(format, "DOMAIN", "VISITS", "TIME SPENT")
fmt.Println(strings.Repeat("-", 65))
next, cur := sum.Iterator()
for next() {
rec := cur()
fmt.Printf(formatValue, rec.Domain, rec.Visits, rec.TimeSpent)
}
fmt.Printf("\n"+formatValue, "TOTAL",
sum.Total().Visits, sum.Total().TimeSpent,
)
}
// this variadic func simplifies the multiple error handling
func errs(errs ...error) (wasErr bool) {
for _, err := range errs {
if err != nil {
fmt.Printf("> Err: %s\n", err)
wasErr = true
}
}
return
}