gaspersic/learngo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move: log parsers

Browse Source
This commit is contained in:
Inanc Gumus
2019-08-28 20:23:38 +03:00
parent 0a121cd911
commit 9afbe8f350
123 changed files with 1018 additions and 1515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
interfaces/log-parser/logs/Makefile
View File

@@ -1,33 +0,0 @@
SHELL := /bin/bash
LINES = $$(wc -l log.txt | cut -f1 -d' ')
ECHO_LINES = echo -e ">> log.txt has $(LINES) lines"
n ?= 18
load: restore
@echo "enlarging the file with itself, please wait..."
@for i in {1..$(n)}; do \
awk 1 log.txt log.txt > log_.txt; \
mv log_.txt log.txt; \
rm -f log_.txt; \
done
@$(ECHO_LINES)
restore:
@echo "restoring the file..."
# @git checkout log.txt
@$(ECHO_LINES)
multiply: remove
@echo "creating 20 log files..."
@for i in {1..20}; do \
echo log$${i}.txt; \
cp log.txt log$${i}.txt; \
done
remove:
rm -f log{1..20}.txt
lines:
@$(ECHO_LINES)

16
interfaces/log-parser/logs/log.jsonl
View File

@@ -1,16 +0,0 @@
{"domain": "learngoprogramming.com", "page": "/", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 15, "uniques": 10}
{"domain": "learngoprogramming.com", "page": "/courses", "visits": 10, "uniques": 5}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 20, "uniques": 15}
{"domain": "learngoprogramming.com", "page": "/articles", "visits": 5, "uniques": 2}
{"domain": "golang.org", "page": "/", "visits": 40, "uniques": 20}
{"domain": "golang.org", "page": "/", "visits": 20, "uniques": 10}
{"domain": "golang.org", "page": "/blog", "visits": 45, "uniques": 25}
{"domain": "golang.org", "page": "/blog", "visits": 15, "uniques": 5}
{"domain": "blog.golang.org", "page": "/courses", "visits": 60, "uniques": 30}
{"domain": "blog.golang.org", "page": "/courses", "visits": 30, "uniques": 20}
{"domain": "blog.golang.org", "page": "/updates", "visits": 20, "uniques": 10}
{"domain": "blog.golang.org", "page": "/reference", "visits": 65, "uniques": 35}
{"domain": "blog.golang.org", "page": "/reference", "visits": 15, "uniques": 5}
{"domain": "inanc.io", "page": "/about", "visits": 30, "uniques": 15}
{"domain": "inanc.io", "page": "/about","visits": 70, "uniques": 35}

16
interfaces/log-parser/logs/log.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com / 10 5
learngoprogramming.com /courses 15 10
learngoprogramming.com /courses 10 5
learngoprogramming.com /articles 20 15
learngoprogramming.com /articles 5 2
golang.org / 40 20
golang.org / 20 10
golang.org /blog 45 25
golang.org /blog 15 5
blog.golang.org /courses 60 30
blog.golang.org /courses 30 20
blog.golang.org /updates 20 10
blog.golang.org /reference 65 35
blog.golang.org /reference 15 5
inanc.io /about 30 15
inanc.io /about 70 35

16
interfaces/log-parser/logs/log_err_missing.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com / 10 5
learngoprogramming.com /courses 15 10
learngoprogramming.com /courses 10 5
learngoprogramming.com /articles 20 15
learngoprogramming.com /articles 5 2
golang.org / 40 20
golang.org / 20 10
golang.org /blog 45 25
golang.org /blog 15 5
blog.golang.org /updates
blog.golang.org /updates 30 20
blog.golang.org /updates 20 10
blog.golang.org /reference 65 35
blog.golang.org /reference 15 5
inanc.io /about 30 15
inanc.io /about 70 35

16
interfaces/log-parser/logs/log_err_negative.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com / 10 5
learngoprogramming.com /courses 15 10
learngoprogramming.com /courses 10 5
learngoprogramming.com /articles 20 15
learngoprogramming.com /articles 5 2
golang.org / 40 20
golang.org / 20 10
golang.org /blog 45 -250
golang.org /blog 15 5
blog.golang.org /updates 60 30
blog.golang.org /updates 30 20
blog.golang.org /updates 20 10
blog.golang.org /reference 65 35
blog.golang.org /reference 15 5
inanc.io /about 30 15
inanc.io /about 70 35

16
interfaces/log-parser/logs/log_err_str.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com / 10 5
learngoprogramming.com /courses 15 10
learngoprogramming.com /courses 10 5
learngoprogramming.com /articles 20 15
learngoprogramming.com /articles 5 2
golang.org / 40 TWENTY
golang.org / 20 10
golang.org /blog 45 25
golang.org /blog 15 5
blog.golang.org /updates 60 30
blog.golang.org /updates 30 20
blog.golang.org /updates 20 10
blog.golang.org /reference 65 35
blog.golang.org /reference 15 5
inanc.io /about 30 15
inanc.io /about 70 35

5
interfaces/log-parser/oop/Makefile
View File

@@ -1,5 +0,0 @@
r:
go run . < ../../logs/log.txt
t:
time go run . < ../../logs/log.txt

42
interfaces/log-parser/oop/filepipe.go
View File

@@ -1,42 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"os"
"strings"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
)
// fromFile generates a default pipeline.
// Detects the correct parser by the file extension.
// Uses a TextReport and groups by domain.
func fromFile(path string) (*pipe.Pipeline, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
var src pipe.Iterator
switch {
case strings.HasSuffix(path, ".txt"):
src = parse.FromText(f)
case strings.HasSuffix(path, ".jsonl"):
src = parse.FromJSON(f)
}
return pipe.New(
src,
report.AsText(os.Stdout),
group.By(group.Domain),
), nil
}

49
interfaces/log-parser/oop/main.go
View File

@@ -1,49 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"log"
"os"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/filter"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
)
func main() {
pipe := pipe.New(
parse.FromText(os.Stdin),
// parse.FromJSON(os.Stdin),
report.AsText(os.Stdout),
filter.By(filter.Not(filter.DomainExt("com", "io"))),
group.By(group.Domain),
new(logger),
)
if err := pipe.Run(); err != nil {
log.Fatalln(err)
}
}
type logger struct {
src pipe.Iterator
}
func (l *logger) Digest(records pipe.Iterator) error {
l.src = records
return nil
}
func (l *logger) Each(yield func(pipe.Record)) error {
return l.src.Each(func(r pipe.Record) {
yield(r)
})
}

38
interfaces/log-parser/oop/pipe/filter/domain.go
View File

@@ -1,38 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package filter
import (
"strings"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
// DomainExt filters a set of domain extensions.
func DomainExt(domains ...string) Func {
return func(r pipe.Record) bool {
for _, domain := range domains {
if strings.HasSuffix(r.Str("domain"), "."+domain) {
return true
}
}
return false
}
}
// Domain filters a domain if it contains the given text.
func Domain(text string) Func {
return func(r pipe.Record) bool {
return strings.Contains(r.Str("domain"), text)
}
}
// DomainOrg filters only the ".org" domains.
func DomainOrg(r pipe.Record) bool {
return strings.HasSuffix(r.Str("domain"), ".org")
}

50
interfaces/log-parser/oop/pipe/filter/filter.go
View File

@@ -1,50 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package filter
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
// Func represents a filtering pipeline func.
type Func func(pipe.Record) (pass bool)
// Filter the records.
type Filter struct {
src pipe.Iterator
filters []Func
}
// By returns a new filter pipeline.
func By(fn ...Func) *Filter {
return &Filter{filters: fn}
}
// Digest saves the iterator for later processing.
func (f *Filter) Digest(records pipe.Iterator) error {
f.src = records
return nil
}
// Each yields only the filtered records.
func (f *Filter) Each(yield func(pipe.Record)) error {
return f.src.Each(func(r pipe.Record) {
if !f.check(r) {
return
}
yield(r)
})
}
// check all the filters against the record.
func (f *Filter) check(r pipe.Record) bool {
for _, fi := range f.filters {
if !fi(r) {
return false
}
}
return true
}

15
interfaces/log-parser/oop/pipe/filter/noop.go
View File

@@ -1,15 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package filter
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
// Noop filter that does nothing.
func Noop(r pipe.Record) bool {
return true
}

17
interfaces/log-parser/oop/pipe/filter/not.go
View File

@@ -1,17 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package filter
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
// Not reverses a filter. True becomes false, and vice versa.
func Not(filter Func) Func {
return func(r pipe.Record) bool {
return !filter(r)
}
}

18
interfaces/log-parser/oop/pipe/group/domain.go
View File

@@ -1,18 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package group
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
// Domain groups the records by domain.
// It keeps the other fields intact.
// For example: It returns the page field as well.
// Exercise: Write a solution that removes the unnecessary data.
func Domain(r pipe.Record) string {
return r.Str("domain")
}

60
interfaces/log-parser/oop/pipe/group/group.go
View File

@@ -1,60 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package group
import (
"sort"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
// Func represents a grouping func that returns a grouping key.
type Func func(pipe.Record) (key string)
// Group records by a key.
type Group struct {
sum map[string]pipe.Record // metrics per group key
keys []string // unique group keys
key Func
}
// By returns a new Group.
// It takes a group func that returns a group key.
// The returned group will group the record using the key.
func By(key Func) *Group {
return &Group{
sum: make(map[string]pipe.Record),
key: key,
}
}
// Digest records for grouping.
func (g *Group) Digest(records pipe.Iterator) error {
return records.Each(func(r pipe.Record) {
k := g.key(r)
if _, ok := g.sum[k]; !ok {
g.keys = append(g.keys, k)
}
if r, ok := r.(pipe.Summer); ok {
g.sum[k] = r.Sum(g.sum[k])
}
})
}
// Each sorts and yields the grouped records.
func (g *Group) Each(yield func(pipe.Record)) error {
sort.Strings(g.keys)
for _, k := range g.keys {
yield(g.sum[k])
}
return nil
}

15
interfaces/log-parser/oop/pipe/group/page.go
View File

@@ -1,15 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package group
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
// Page groups records by page.
func Page(r pipe.Record) string {
return r.Str("domain") + r.Str("page")
}

36
interfaces/log-parser/oop/pipe/logcount.go
View File

@@ -1,36 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package pipe
import "fmt"
// logCount counts the yielded records.
type logCount struct {
Iterator
n int
}
// Each yields to the inner iterator while counting the records.
// Reports the record number on an error.
func (lc *logCount) Each(yield func(Record)) error {
err := lc.Iterator.Each(func(r Record) {
lc.n++
yield(r)
})
if err != nil {
// lc.n+1: iterator.each won't call yield on err
return fmt.Errorf("record %d: %v", lc.n+1, err)
}
return nil
}
// count returns the last read record number.
func (lc *logCount) count() int {
return lc.n
}

19
interfaces/log-parser/oop/pipe/parse/close.go
View File

@@ -1,19 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import (
"io"
)
// readClose the reader if it's a io.Closer.
func readClose(r io.Reader) {
if rc, ok := r.(io.Closer); ok {
rc.Close()
}
}

47
interfaces/log-parser/oop/pipe/parse/json.go
View File

@@ -1,47 +0,0 @@
// For more tutorials: https://bj.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import (
"encoding/json"
"io"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
// JSON parses json records.
type JSON struct {
reader io.Reader
}
// FromJSON creates a json parser.
func FromJSON(r io.Reader) *JSON {
return &JSON{reader: r}
}
// Each yields records from a json reader.
func (j *JSON) Each(yield func(pipe.Record)) error {
defer readClose(j.reader)
dec := json.NewDecoder(j.reader)
for {
var r record
err := dec.Decode(&r)
if err == io.EOF {
break
}
if err != nil {
return err
}
yield(r)
}
return nil
}

116
interfaces/log-parser/oop/pipe/parse/record.go
View File

@@ -1,116 +0,0 @@
package parse
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
const fieldsLength = 4
// record stores fields of a log line.
type record struct {
Domain string
Page string
Visits int
Uniques int
}
// Str gets a string field by name.
func (r record) Str(field string) string {
switch field {
case "domain":
return r.Domain
case "page":
return r.Page
}
panic(fieldErr(field))
}
// Int gets an integer field by name.
func (r record) Int(field string) int {
switch field {
case "visits":
return r.Visits
case "uniques":
return r.Uniques
}
panic(fieldErr(field))
}
// Sum the numeric fields with another record.
func (r record) Sum(other pipe.Record) pipe.Record {
if other == nil {
return r
}
r.Visits += other.(record).Visits
r.Uniques += other.(record).Uniques
return r
}
// UnmarshalText to a *record.
func (r *record) UnmarshalText(p []byte) (err error) {
fields := strings.Fields(string(p))
if len(fields) != fieldsLength {
return fmt.Errorf("wrong number of fields %q", fields)
}
r.Domain, r.Page = fields[0], fields[1]
if r.Visits, err = parseStr("visits", fields[2]); err != nil {
return err
}
if r.Uniques, err = parseStr("uniques", fields[3]); err != nil {
return err
}
return validate(*r)
}
// UnmarshalJSON to a *record.
func (r *record) UnmarshalJSON(data []byte) error {
// `methodless` doesn't have any methods including UnmarshalJSON.
// This trick prevents the stack-overflow (infinite loop).
type methodless record
var m methodless
if err := json.Unmarshal(data, &m); err != nil {
return err
}
// Cast back to the record and save.
*r = record(m)
return validate(*r)
}
// parseStr helps UnmarshalText for string to positive int parsing.
func parseStr(name, v string) (int, error) {
n, err := strconv.Atoi(v)
if err != nil {
return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err)
}
return n, nil
}
// validate whether a parsed record is valid or not.
func validate(r record) (err error) {
switch {
case r.Domain == "":
err = errors.New("record.domain cannot be empty")
case r.Page == "":
err = errors.New("record.page cannot be empty")
case r.Visits < 0:
err = errors.New("record.visits cannot be negative")
case r.Uniques < 0:
err = errors.New("record.uniques cannot be negative")
}
return
}
func fieldErr(field string) error {
return fmt.Errorf("record field: %q does not exist", field)
}

44
interfaces/log-parser/oop/pipe/parse/text.go
View File

@@ -1,44 +0,0 @@
// For more tutorials: https://bp.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package parse
import (
"bufio"
"io"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
// Text parses text based log lines.
type Text struct {
reader io.Reader
}
// FromText creates a text parser.
func FromText(r io.Reader) *Text {
return &Text{reader: r}
}
// Each yields records from a text log.
func (p *Text) Each(yield func(pipe.Record)) error {
defer readClose(p.reader)
in := bufio.NewScanner(p.reader)
for in.Scan() {
r := new(record)
if err := r.UnmarshalText(in.Bytes()); err != nil {
return err
}
yield(r)
}
return in.Err()
}

26
interfaces/log-parser/oop/pipe/pipe.go
View File

@@ -1,26 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package pipe
// Iterator yields a record.
type Iterator interface {
Each(func(Record)) error
}
// Digester represents a record consumer.
type Digester interface {
Digest(Iterator) error
}
// Transform represents both a record consumer and producer.
// It has an input and output.
// It takes a single record and provides an iterator for all the records.
type Transform interface {
Digester // consumer
Iterator // producer
}

48
interfaces/log-parser/oop/pipe/pipeline.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package pipe
import (
"fmt"
"os"
)
// Pipeline takes records from a source, transforms, and sends them to a destionation.
type Pipeline struct {
src Iterator
trans []Transform
dst Digester
}
// New creates a new pipeline.
func New(src Iterator, dst Digester, t ...Transform) *Pipeline {
return &Pipeline{
src: &logCount{Iterator: src},
dst: dst,
trans: t,
}
}
// Run the pipeline.
func (p *Pipeline) Run() error {
defer func() {
n := p.src.(*logCount).count()
fmt.Fprintf(os.Stderr, "%d records processed.\n", n)
}()
last := p.src
for _, t := range p.trans {
if err := t.Digest(last); err != nil {
return err
}
last = t
}
return p.dst.Digest(last)
}

19
interfaces/log-parser/oop/pipe/record.go
View File

@@ -1,19 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package pipe
// Record provides a generic interface for any sort of records.
type Record interface {
Str(field string) string
Int(field string) int
}
// Summer provides a method for summing the numeric fields.
type Summer interface {
Sum(Record) Record
}

49
interfaces/log-parser/oop/pipe/report/chart.go
View File

@@ -1,49 +0,0 @@
package report
/*
// You need to run:
// go get -u github.com/wcharczuk/go-chart
// Chart renders a chart.
type Chart struct {
Title string
Width, Height int
w io.Writer
}
// AsChart returns a Chart report generator.
func AsChart(w io.Writer) *Chart {
return &Chart{w: w}
}
// Digest generates a chart report.
func (c *Chart) Digest(records pipe.Iterator) error {
w := os.Stdout
donut := chart.DonutChart{
Title: c.Title,
TitleStyle: chart.Style{
FontSize: 35,
Show: true,
FontColor: chart.ColorAlternateGreen,
},
Width: c.Width,
Height: c.Height,
}
records.Each(func(r pipe.Record) {
v := chart.Value{
Label: r.Str("domain") + r.Str("page") + ": " + strconv.Itoa(r.Int("visits")),
Value: float64(r.Int("visits")),
Style: chart.Style{
FontSize: 14,
},
}
donut.Values = append(donut.Values, v)
})
return donut.Render(chart.SVG, w)
}
*/

64
interfaces/log-parser/oop/pipe/report/text.go
View File

@@ -1,64 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
"io"
"text/tabwriter"
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
)
const (
minWidth = 0
tabWidth = 4
padding = 4
flags = 0
)
// Text report generator.
type Text struct {
w io.Writer
}
// AsText returns a Text report generator.
func AsText(w io.Writer) *Text {
return &Text{w: w}
}
// Digest generates a text report.
func (t *Text) Digest(records pipe.Iterator) error {
w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags)
write := fmt.Fprintf
write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
write(w, "-------\t-----\t------\t-------\n")
var total pipe.Record
records.Each(func(r pipe.Record) {
if r, ok := r.(pipe.Summer); ok {
total = r.Sum(total)
}
write(w, "%s\t%s\t%d\t%d\n",
r.Str("domain"), r.Str("page"),
r.Int("visits"), r.Int("uniques"),
)
})
write(w, "\t\t\t\n")
write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "",
total.Int("visits"),
total.Int("uniques"),
)
return w.Flush()
}

6
interfaces/log-parser/testing/log.txt
View File

@@ -1,6 +0,0 @@
learngoprogramming.com 10 200
learngoprogramming.com 10 300
golang.org 4 50
golang.org 6 100
blog.golang.org 20 25
blog.golang.org 10 1

6
interfaces/log-parser/testing/log_err_missing.txt
View File

@@ -1,6 +0,0 @@
learngoprogramming.com 10 200
learngoprogramming.com 10
golang.org 4 50
golang.org 6 100
blog.golang.org 20 25
blog.golang.org 10 1

6
interfaces/log-parser/testing/log_err_negative.txt
View File

@@ -1,6 +0,0 @@
learngoprogramming.com 10 200
learngoprogramming.com 10 300
golang.org -100 50
golang.org 6 100
blog.golang.org 20 25
blog.golang.org 10 1

6
interfaces/log-parser/testing/log_err_str.txt
View File

@@ -1,6 +0,0 @@
learngoprogramming.com 10 200
learngoprogramming.com 10 THREE-HUNDRED
golang.org FOUR 50
golang.org 6 100
blog.golang.org 20 25
blog.golang.org 10 1

26
interfaces/log-parser/testing/main.go
View File

@@ -1,26 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"os"
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
)
func main() {
p := report.New()
in := bufio.NewScanner(os.Stdin)
for in.Scan() {
p.Parse(in.Text())
}
summarize(p.Summarize(), p.Err(), in.Err())
}

59
interfaces/log-parser/testing/main_test.go
View File

@@ -1,59 +0,0 @@
// +build integration
// go test -tags=integration
package main_test
import (
"bytes"
"os/exec"
"strings"
"testing"
)
const (
okIn = `
a.com 1 2
b.com 3 4
a.com 4 5
b.com 6 7`
okOut = `
DOMAIN VISITS TIME SPENT
-----------------------------------------------------------------
a.com 5 7
b.com 9 11
TOTAL 14 18`
)
func TestSummary(t *testing.T) {
tests := []struct {
name, in, out string
}{
{"valid input", okIn, okOut},
{"missing fields", "a.com 1 2\nb.com 3", "> Err: line #2: missing fields: [b.com 3]"},
{"incorrect visits", "a.com 1 2\nb.com -1 1", `> Err: line #2: incorrect visits: "-1"`},
{"incorrect time spent", "a.com 1 2\nb.com 3 -1", `> Err: line #2: incorrect time spent: "-1"`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
run(t, strings.TrimSpace(tt.in), strings.TrimSpace(tt.out))
})
}
}
func run(t *testing.T, in, out string) {
cmd := exec.Command("go", "run", ".")
cmd.Stdin = strings.NewReader(in)
got, err := cmd.CombinedOutput()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, []byte(out+"\n")) {
t.Fatalf("\nwant:\n%s\n\ngot:\n%s", out, got)
}
}

52
interfaces/log-parser/testing/report/parser.go
View File

@@ -1,52 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
)
// Parser parses the log file and generates a summary report.
type Parser struct {
summary *Summary // summarizes the parsing results
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
}
// New returns a new parsing state.
func New() *Parser {
return &Parser{summary: newSummary()}
}
// Parse parses a log line and adds it to the summary.
func (p *Parser) Parse(line string) {
// if there was an error do not continue
if p.lerr != nil {
return
}
// chain the parser's error to the result's
res, err := parse(line)
if p.lines++; err != nil {
p.lerr = fmt.Errorf("line #%d: %s", p.lines, err)
return
}
p.summary.update(res)
}
// Summarize summarizes the parsing results.
// Only use it after the parsing is done.
func (p *Parser) Summarize() *Summary {
return p.summary
}
// Err returns the last error encountered
func (p *Parser) Err() error {
return p.lerr
}

55
interfaces/log-parser/testing/report/parser_test.go
View File

@@ -1,55 +0,0 @@
package report_test
import (
"strings"
"testing"
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
)
func newParser(lines string) *report.Parser {
p := report.New()
p.Parse(lines)
return p
}
func TestParserLineErrs(t *testing.T) {
p := newParser("a.com 1 2")
p.Parse("b.com -1 -1")
want := "#2"
err := p.Err().Error()
if !strings.Contains(err, want) {
t.Errorf("want: %q; got: %q", want, err)
}
}
func TestParserStopsOnErr(t *testing.T) {
p := newParser("a.com 10 20")
p.Parse("b.com -1 -1")
p.Parse("neverparses.com 30 40")
s := p.Summarize()
if want, got := 10, s.Total().Visits; want != got {
t.Errorf("want: %d; got: %d", want, got)
}
}
func TestParserIncorrectFields(t *testing.T) {
tests := []struct {
in, name string
}{
{"a.com", "missing fields"},
{"a.com -1 2", "incorrect visits"},
{"a.com 1 -1", "incorrect time spent"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if p := newParser(tt.in); p.Err() == nil {
t.Errorf("in: %q; got: nil err", tt.in)
}
})
}
}

60
interfaces/log-parser/testing/report/result.go
View File

@@ -1,60 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"fmt"
"strconv"
"strings"
)
// always put all the related things together as in here
// Result stores metrics for a domain
// it uses the value mechanics,
// because it doesn't have to update anything
type Result struct {
Domain string `json:"domain"`
Visits int `json:"visits"`
TimeSpent int `json:"time_spent"`
// add more metrics if needed
}
// add adds the metrics of another Result to itself and returns a new Result
func (r Result) add(other Result) Result {
return Result{
Domain: r.Domain,
Visits: r.Visits + other.Visits,
TimeSpent: r.TimeSpent + other.TimeSpent,
}
}
// parse parses a single log line
func parse(line string) (r Result, err error) {
fields := strings.Fields(line)
if len(fields) != 3 {
return r, fmt.Errorf("missing fields: %v", fields)
}
f := new(field)
r.Domain = fields[0]
r.Visits = f.atoi("visits", fields[1])
r.TimeSpent = f.atoi("time spent", fields[2])
return r, f.err
}
// field helps for field parsing
type field struct{ err error }
func (f *field) atoi(name, val string) int {
n, err := strconv.Atoi(val)
if n < 0 || err != nil {
f.err = fmt.Errorf("incorrect %s: %q", name, val)
}
return n
}

86
interfaces/log-parser/testing/report/summary.go
View File

@@ -1,86 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package report
import (
"sort"
)
// Summary aggregates the parsing results
type Summary struct {
sum map[string]Result // metrics per domain
domains []string // unique domain names
total Result // total visits for all domains
}
// newSummary constructs and initializes a new summary
// You can't use its methods without pointer mechanics
func newSummary() *Summary {
return &Summary{sum: make(map[string]Result)}
}
// Update updates the report for the given parsing result
func (s *Summary) update(r Result) {
domain := r.Domain
if _, ok := s.sum[domain]; !ok {
s.domains = append(s.domains, domain)
}
// let the result handle the addition
// this allows us to manage the result in once place
// and this way it becomes easily extendable
s.total = s.total.add(r)
s.sum[domain] = r.add(s.sum[domain])
}
// Iterator returns `next()` to detect when the iteration ends,
// and a `cur()` to return the current result.
// iterator iterates sorted by domains.
func (s *Summary) Iterator() (next func() bool, cur func() Result) {
sort.Strings(s.domains)
// remember the last iterated result
var last int
next = func() bool {
defer func() { last++ }()
return len(s.domains) > last
}
cur = func() Result {
// returns a copy so the caller cannot change it
name := s.domains[last-1]
return s.sum[name]
}
return
}
// Total returns the total metrics
func (s *Summary) Total() Result {
return s.total
}
// For the interfaces section
//
// MarshalJSON marshals a report to JSON
// Alternative: unexported embedding
// func (s *Summary) MarshalJSON() ([]byte, error) {
// type total struct {
// *Result
// IgnoreDomain *string `json:"domain,omitempty"`
// }
// return json.Marshal(struct {
// Sum map[string]Result `json:"summary"`
// Domains []string `json:"domains"`
// Total total `json:"total"`
// }{
// Sum: s.sum, Domains: s.domains, Total: total{Result: &s.total},
// })
// }

44
interfaces/log-parser/testing/report/summary_test.go
View File

@@ -1,44 +0,0 @@
package report_test
import (
"testing"
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
)
func TestSummaryTotal(t *testing.T) {
p := newParser("a.com 1 2")
p.Parse("b.com 3 4")
s := p.Summarize()
want := report.Result{Domain: "", Visits: 4, TimeSpent: 6}
if got := s.Total(); want != got {
t.Errorf("want: %+v; got: %+v", want, got)
}
}
func TestSummaryIterator(t *testing.T) {
p := newParser("a.com 1 2")
p.Parse("a.com 3 4")
p.Parse("b.com 5 6")
s := p.Summarize()
next, cur := s.Iterator()
wants := []report.Result{
{Domain: "a.com", Visits: 4, TimeSpent: 6},
{Domain: "b.com", Visits: 5, TimeSpent: 6},
}
for _, want := range wants {
t.Run(want.Domain, func(t *testing.T) {
if got := next(); !got {
t.Errorf("next(): want: %t; got: %t", true, got)
}
if got := cur(); want != got {
t.Errorf("cur(): want: %+v; got: %+v", want, got)
}
})
}
}

73
interfaces/log-parser/testing/summarize.go
View File

@@ -1,73 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
)
// summarize prints the parsing results.
//
// it prints the errors and returns if there are any.
//
// --json flag encodes to json and prints.
func summarize(sum *report.Summary, errors ...error) {
if errs(errors...) {
return
}
if args := os.Args[1:]; len(args) == 1 && args[0] == "--json" {
encode(sum)
return
}
stdout(sum)
}
// encodes the summary to json
func encode(sum *report.Summary) {
out, err := json.MarshalIndent(sum, "", "\t")
if err != nil {
panic(err)
}
os.Stdout.Write(out)
}
// prints the summary to standard out
func stdout(sum *report.Summary) {
const (
head = "%-30s %10s %20s\n"
val = "%-30s %10d %20d\n"
)
fmt.Printf(head, "DOMAIN", "VISITS", "TIME SPENT")
fmt.Println(strings.Repeat("-", 65))
for next, cur := sum.Iterator(); next(); {
r := cur()
fmt.Printf(val, r.Domain, r.Visits, r.TimeSpent)
}
t := sum.Total()
fmt.Printf("\n"+val, "TOTAL", t.Visits, t.TimeSpent)
}
// this variadic func simplifies the multiple error handling
func errs(errs ...error) (wasErr bool) {
for _, err := range errs {
if err != nil {
fmt.Printf("> Err: %s\n", err)
wasErr = true
}
}
return
}