refactor: oop log parser to pkgs
This commit is contained in:
@ -1,38 +0,0 @@
|
||||
package main
|
||||
|
||||
// You need to run:
|
||||
// go get -u github.com/wcharczuk/go-chart
|
||||
|
||||
// type chartReport struct {
|
||||
// title string
|
||||
// width, height int
|
||||
// }
|
||||
|
||||
// func (s *chartReport) digest(records iterator) error {
|
||||
// w := os.Stdout
|
||||
|
||||
// donut := chart.DonutChart{
|
||||
// Title: s.title,
|
||||
// TitleStyle: chart.Style{
|
||||
// FontSize: 35,
|
||||
// Show: true,
|
||||
// FontColor: chart.ColorAlternateGreen,
|
||||
// },
|
||||
// Width: s.width,
|
||||
// Height: s.height,
|
||||
// }
|
||||
|
||||
// records.each(func(r record) {
|
||||
// v := chart.Value{
|
||||
// Label: r.domain + r.page + ": " + strconv.Itoa(r.visits),
|
||||
// Value: float64(r.visits),
|
||||
// Style: chart.Style{
|
||||
// FontSize: 14,
|
||||
// },
|
||||
// }
|
||||
|
||||
// donut.Values = append(donut.Values, v)
|
||||
// })
|
||||
|
||||
// return donut.Render(chart.SVG, w)
|
||||
// }
|
42
interfaces/log-parser/oop/filepipe.go
Normal file
42
interfaces/log-parser/oop/filepipe.go
Normal file
@ -0,0 +1,42 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
|
||||
)
|
||||
|
||||
// fromFile generates a default pipeline.
|
||||
// Detects the correct parser by the file extension.
|
||||
// Uses a TextReport and groups by domain.
|
||||
func fromFile(path string) (*pipe.Pipeline, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var src pipe.Iterator
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".txt"):
|
||||
src = parse.FromText(f)
|
||||
case strings.HasSuffix(path, ".jsonl"):
|
||||
src = parse.FromJSON(f)
|
||||
}
|
||||
|
||||
return pipe.New(
|
||||
src,
|
||||
report.AsText(os.Stdout),
|
||||
group.By(group.Domain),
|
||||
), nil
|
||||
}
|
@ -1,43 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
type filter struct {
|
||||
src iterator
|
||||
filters []filterFunc
|
||||
}
|
||||
|
||||
func filterBy(fn ...filterFunc) *filter {
|
||||
return &filter{filters: fn}
|
||||
}
|
||||
|
||||
// transform the record
|
||||
func (f *filter) digest(records iterator) error {
|
||||
f.src = records
|
||||
return nil
|
||||
}
|
||||
|
||||
// each yields only the filtered records
|
||||
func (f *filter) each(yield recordFn) error {
|
||||
return f.src.each(func(r record) {
|
||||
if !f.check(r) {
|
||||
return
|
||||
}
|
||||
yield(r)
|
||||
})
|
||||
}
|
||||
|
||||
// check all the filters against the record
|
||||
func (f *filter) check(r record) bool {
|
||||
for _, fi := range f.filters {
|
||||
if !fi(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
@ -1,36 +0,0 @@
|
||||
package main
|
||||
|
||||
import "strings"
|
||||
|
||||
type filterFunc func(record) bool
|
||||
|
||||
func noopFilter(r record) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func notUsing(filter filterFunc) filterFunc {
|
||||
return func(r record) bool {
|
||||
return !filter(r)
|
||||
}
|
||||
}
|
||||
|
||||
func domainExtFilter(domains ...string) filterFunc {
|
||||
return func(r record) bool {
|
||||
for _, domain := range domains {
|
||||
if strings.HasSuffix(r.domain, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func domainFilter(domain string) filterFunc {
|
||||
return func(r record) bool {
|
||||
return strings.Contains(r.domain, domain)
|
||||
}
|
||||
}
|
||||
|
||||
func orgDomainsFilter(r record) bool {
|
||||
return strings.HasSuffix(r.domain, ".org")
|
||||
}
|
@ -1,49 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
type group struct {
|
||||
sum map[string]record // metrics per group key
|
||||
keys []string // unique group keys
|
||||
key groupFunc
|
||||
}
|
||||
|
||||
func groupBy(key groupFunc) *group {
|
||||
return &group{
|
||||
sum: make(map[string]record),
|
||||
key: key,
|
||||
}
|
||||
}
|
||||
|
||||
// digest the records
|
||||
func (g *group) digest(records iterator) error {
|
||||
return records.each(func(r record) {
|
||||
k := g.key(r)
|
||||
|
||||
if _, ok := g.sum[k]; !ok {
|
||||
g.keys = append(g.keys, k)
|
||||
}
|
||||
|
||||
g.sum[k] = r.sum(g.sum[k])
|
||||
})
|
||||
}
|
||||
|
||||
// each yields the grouped records
|
||||
func (g *group) each(yield recordFn) error {
|
||||
sort.Strings(g.keys)
|
||||
|
||||
for _, k := range g.keys {
|
||||
yield(g.sum[k])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
@ -1,15 +0,0 @@
|
||||
package main
|
||||
|
||||
type groupFunc func(record) string
|
||||
|
||||
// domainGrouper groups by domain.
|
||||
// but it keeps the other fields.
|
||||
// for example: it returns pages as well, but you shouldn't use them.
|
||||
// exercise: write a function that erases the unnecessary data.
|
||||
func domainGrouper(r record) string {
|
||||
return r.domain
|
||||
}
|
||||
|
||||
func pageGrouper(r record) string {
|
||||
return r.domain + r.page
|
||||
}
|
@ -10,35 +10,40 @@ package main
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/filter"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// newGrouper(domainGrouper)
|
||||
|
||||
// s := &chartReport{
|
||||
// title: "visits per domain",
|
||||
// width: 1920,
|
||||
// height: 800,
|
||||
// }
|
||||
|
||||
// pipe, err := fromFile("../logs/log.jsonl")
|
||||
// if err != nil {
|
||||
// log.Fatalln(err)
|
||||
// }
|
||||
|
||||
pipe := newPipeline(
|
||||
newTextLog(os.Stdin),
|
||||
// newJSONLog(os.Stdin),
|
||||
newTextReport(),
|
||||
filterBy(notUsing(domainExtFilter("com", "io"))),
|
||||
groupBy(domainGrouper),
|
||||
pipe := pipe.New(
|
||||
parse.FromText(os.Stdin),
|
||||
// parse.FromJSON(os.Stdin),
|
||||
report.AsText(os.Stdout),
|
||||
filter.By(filter.Not(filter.DomainExt("com", "io"))),
|
||||
group.By(group.Domain),
|
||||
new(logger),
|
||||
)
|
||||
|
||||
if err := pipe.run(); err != nil {
|
||||
if err := pipe.Run(); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// if err := reportFromFile(os.Args[1]); err != nil {
|
||||
// log.Fatalln(err)
|
||||
// }
|
||||
}
|
||||
|
||||
type logger struct {
|
||||
src pipe.Iterator
|
||||
}
|
||||
|
||||
func (l *logger) Digest(records pipe.Iterator) error {
|
||||
l.src = records
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *logger) Each(yield func(pipe.Record)) error {
|
||||
return l.src.Each(func(r pipe.Record) {
|
||||
yield(r)
|
||||
})
|
||||
}
|
||||
|
38
interfaces/log-parser/oop/pipe/filter/domain.go
Normal file
38
interfaces/log-parser/oop/pipe/filter/domain.go
Normal file
@ -0,0 +1,38 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package filter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
// DomainExt filters a set of domain extensions.
|
||||
func DomainExt(domains ...string) Func {
|
||||
return func(r pipe.Record) bool {
|
||||
for _, domain := range domains {
|
||||
if strings.HasSuffix(r.Str("domain"), "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Domain filters a domain if it contains the given text.
|
||||
func Domain(text string) Func {
|
||||
return func(r pipe.Record) bool {
|
||||
return strings.Contains(r.Str("domain"), text)
|
||||
}
|
||||
}
|
||||
|
||||
// DomainOrg filters only the ".org" domains.
|
||||
func DomainOrg(r pipe.Record) bool {
|
||||
return strings.HasSuffix(r.Str("domain"), ".org")
|
||||
}
|
50
interfaces/log-parser/oop/pipe/filter/filter.go
Normal file
50
interfaces/log-parser/oop/pipe/filter/filter.go
Normal file
@ -0,0 +1,50 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package filter
|
||||
|
||||
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
|
||||
// Func represents a filtering pipeline func.
|
||||
type Func func(pipe.Record) (pass bool)
|
||||
|
||||
// Filter the records.
|
||||
type Filter struct {
|
||||
src pipe.Iterator
|
||||
filters []Func
|
||||
}
|
||||
|
||||
// By returns a new filter pipeline.
|
||||
func By(fn ...Func) *Filter {
|
||||
return &Filter{filters: fn}
|
||||
}
|
||||
|
||||
// Digest saves the iterator for later processing.
|
||||
func (f *Filter) Digest(records pipe.Iterator) error {
|
||||
f.src = records
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each yields only the filtered records.
|
||||
func (f *Filter) Each(yield func(pipe.Record)) error {
|
||||
return f.src.Each(func(r pipe.Record) {
|
||||
if !f.check(r) {
|
||||
return
|
||||
}
|
||||
yield(r)
|
||||
})
|
||||
}
|
||||
|
||||
// check all the filters against the record.
|
||||
func (f *Filter) check(r pipe.Record) bool {
|
||||
for _, fi := range f.filters {
|
||||
if !fi(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
15
interfaces/log-parser/oop/pipe/filter/noop.go
Normal file
15
interfaces/log-parser/oop/pipe/filter/noop.go
Normal file
@ -0,0 +1,15 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package filter
|
||||
|
||||
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
|
||||
// Noop filter that does nothing.
|
||||
func Noop(r pipe.Record) bool {
|
||||
return true
|
||||
}
|
17
interfaces/log-parser/oop/pipe/filter/not.go
Normal file
17
interfaces/log-parser/oop/pipe/filter/not.go
Normal file
@ -0,0 +1,17 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package filter
|
||||
|
||||
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
|
||||
// Not reverses a filter. True becomes false, and vice versa.
|
||||
func Not(filter Func) Func {
|
||||
return func(r pipe.Record) bool {
|
||||
return !filter(r)
|
||||
}
|
||||
}
|
18
interfaces/log-parser/oop/pipe/group/domain.go
Normal file
18
interfaces/log-parser/oop/pipe/group/domain.go
Normal file
@ -0,0 +1,18 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package group
|
||||
|
||||
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
|
||||
// Domain groups the records by domain.
|
||||
// It keeps the other fields intact.
|
||||
// For example: It returns the page field as well.
|
||||
// Exercise: Write a solution that removes the unnecessary data.
|
||||
func Domain(r pipe.Record) string {
|
||||
return r.Str("domain")
|
||||
}
|
60
interfaces/log-parser/oop/pipe/group/group.go
Normal file
60
interfaces/log-parser/oop/pipe/group/group.go
Normal file
@ -0,0 +1,60 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package group
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
// Func represents a grouping func that returns a grouping key.
|
||||
type Func func(pipe.Record) (key string)
|
||||
|
||||
// Group records by a key.
|
||||
type Group struct {
|
||||
sum map[string]pipe.Record // metrics per group key
|
||||
keys []string // unique group keys
|
||||
key Func
|
||||
}
|
||||
|
||||
// By returns a new Group.
|
||||
// It takes a group func that returns a group key.
|
||||
// The returned group will group the record using the key.
|
||||
func By(key Func) *Group {
|
||||
return &Group{
|
||||
sum: make(map[string]pipe.Record),
|
||||
key: key,
|
||||
}
|
||||
}
|
||||
|
||||
// Digest records for grouping.
|
||||
func (g *Group) Digest(records pipe.Iterator) error {
|
||||
return records.Each(func(r pipe.Record) {
|
||||
k := g.key(r)
|
||||
|
||||
if _, ok := g.sum[k]; !ok {
|
||||
g.keys = append(g.keys, k)
|
||||
}
|
||||
|
||||
if r, ok := r.(pipe.Summer); ok {
|
||||
g.sum[k] = r.Sum(g.sum[k])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Each sorts and yields the grouped records.
|
||||
func (g *Group) Each(yield func(pipe.Record)) error {
|
||||
sort.Strings(g.keys)
|
||||
|
||||
for _, k := range g.keys {
|
||||
yield(g.sum[k])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
15
interfaces/log-parser/oop/pipe/group/page.go
Normal file
15
interfaces/log-parser/oop/pipe/group/page.go
Normal file
@ -0,0 +1,15 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package group
|
||||
|
||||
import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
|
||||
// Page groups records by page.
|
||||
func Page(r pipe.Record) string {
|
||||
return r.Str("domain") + r.Str("page")
|
||||
}
|
@ -5,18 +5,20 @@
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
package pipe
|
||||
|
||||
import "fmt"
|
||||
|
||||
// logCount counts the yielded records
|
||||
// logCount counts the yielded records.
|
||||
type logCount struct {
|
||||
iterator
|
||||
Iterator
|
||||
n int
|
||||
}
|
||||
|
||||
func (lc *logCount) each(yield recordFn) error {
|
||||
err := lc.iterator.each(func(r record) {
|
||||
// Each yields to the inner iterator while counting the records.
|
||||
// Reports the record number on an error.
|
||||
func (lc *logCount) Each(yield func(Record)) error {
|
||||
err := lc.Iterator.Each(func(r Record) {
|
||||
lc.n++
|
||||
yield(r)
|
||||
})
|
||||
@ -28,6 +30,7 @@ func (lc *logCount) each(yield recordFn) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// count returns the last read record number.
|
||||
func (lc *logCount) count() int {
|
||||
return lc.n
|
||||
}
|
@ -5,12 +5,13 @@
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
package parse
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// readClose the reader if it's a io.Closer.
|
||||
func readClose(r io.Reader) {
|
||||
if rc, ok := r.(io.Closer); ok {
|
||||
rc.Close()
|
@ -5,26 +5,30 @@
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
package parse
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
type jsonLog struct {
|
||||
// JSON parses json records.
|
||||
type JSON struct {
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func newJSONLog(r io.Reader) *jsonLog {
|
||||
return &jsonLog{reader: r}
|
||||
// FromJSON creates a json parser.
|
||||
func FromJSON(r io.Reader) *JSON {
|
||||
return &JSON{reader: r}
|
||||
}
|
||||
|
||||
func (j *jsonLog) each(yield recordFn) error {
|
||||
// Each yields records from a json reader.
|
||||
func (j *JSON) Each(yield func(pipe.Record)) error {
|
||||
defer readClose(j.reader)
|
||||
|
||||
dec := json.NewDecoder(bufio.NewReader(j.reader))
|
||||
dec := json.NewDecoder(j.reader)
|
||||
|
||||
for {
|
||||
var r record
|
116
interfaces/log-parser/oop/pipe/parse/record.go
Normal file
116
interfaces/log-parser/oop/pipe/parse/record.go
Normal file
@ -0,0 +1,116 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// record stores fields of a log line.
|
||||
type record struct {
|
||||
Domain string
|
||||
Page string
|
||||
Visits int
|
||||
Uniques int
|
||||
}
|
||||
|
||||
// Str gets a string field by name.
|
||||
func (r record) Str(field string) string {
|
||||
switch field {
|
||||
case "domain":
|
||||
return r.Domain
|
||||
case "page":
|
||||
return r.Page
|
||||
}
|
||||
panic(fieldErr(field))
|
||||
}
|
||||
|
||||
// Int gets an integer field by name.
|
||||
func (r record) Int(field string) int {
|
||||
switch field {
|
||||
case "visits":
|
||||
return r.Visits
|
||||
case "uniques":
|
||||
return r.Uniques
|
||||
}
|
||||
panic(fieldErr(field))
|
||||
}
|
||||
|
||||
// Sum the numeric fields with another record.
|
||||
func (r record) Sum(other pipe.Record) pipe.Record {
|
||||
if other == nil {
|
||||
return r
|
||||
}
|
||||
r.Visits += other.(record).Visits
|
||||
r.Uniques += other.(record).Uniques
|
||||
return r
|
||||
}
|
||||
|
||||
// UnmarshalText to a *record.
|
||||
func (r *record) UnmarshalText(p []byte) (err error) {
|
||||
fields := strings.Fields(string(p))
|
||||
if len(fields) != fieldsLength {
|
||||
return fmt.Errorf("wrong number of fields %q", fields)
|
||||
}
|
||||
|
||||
r.Domain, r.Page = fields[0], fields[1]
|
||||
|
||||
if r.Visits, err = parseStr("visits", fields[2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Uniques, err = parseStr("uniques", fields[3]); err != nil {
|
||||
return err
|
||||
}
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// UnmarshalJSON to a *record.
|
||||
func (r *record) UnmarshalJSON(data []byte) error {
|
||||
// `methodless` doesn't have any methods including UnmarshalJSON.
|
||||
// This trick prevents the stack-overflow (infinite loop).
|
||||
type methodless record
|
||||
|
||||
var m methodless
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Cast back to the record and save.
|
||||
*r = record(m)
|
||||
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// parseStr helps UnmarshalText for string to positive int parsing.
|
||||
func parseStr(name, v string) (int, error) {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// validate whether a parsed record is valid or not.
|
||||
func validate(r record) (err error) {
|
||||
switch {
|
||||
case r.Domain == "":
|
||||
err = errors.New("record.domain cannot be empty")
|
||||
case r.Page == "":
|
||||
err = errors.New("record.page cannot be empty")
|
||||
case r.Visits < 0:
|
||||
err = errors.New("record.visits cannot be negative")
|
||||
case r.Uniques < 0:
|
||||
err = errors.New("record.uniques cannot be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func fieldErr(field string) error {
|
||||
return fmt.Errorf("record field: %q does not exist", field)
|
||||
}
|
@ -5,22 +5,27 @@
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
package parse
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
type textLog struct {
|
||||
// Text parses text based log lines.
|
||||
type Text struct {
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func newTextLog(r io.Reader) *textLog {
|
||||
return &textLog{reader: r}
|
||||
// FromText creates a text parser.
|
||||
func FromText(r io.Reader) *Text {
|
||||
return &Text{reader: r}
|
||||
}
|
||||
|
||||
func (p *textLog) each(yield recordFn) error {
|
||||
// Each yields records from a text log.
|
||||
func (p *Text) Each(yield func(pipe.Record)) error {
|
||||
defer readClose(p.reader)
|
||||
|
||||
in := bufio.NewScanner(p.reader)
|
||||
@ -32,7 +37,7 @@ func (p *textLog) each(yield recordFn) error {
|
||||
return err
|
||||
}
|
||||
|
||||
yield(*r)
|
||||
yield(r)
|
||||
}
|
||||
|
||||
return in.Err()
|
26
interfaces/log-parser/oop/pipe/pipe.go
Normal file
26
interfaces/log-parser/oop/pipe/pipe.go
Normal file
@ -0,0 +1,26 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package pipe
|
||||
|
||||
// Iterator yields a record.
|
||||
type Iterator interface {
|
||||
Each(func(Record)) error
|
||||
}
|
||||
|
||||
// Digester represents a record consumer.
|
||||
type Digester interface {
|
||||
Digest(Iterator) error
|
||||
}
|
||||
|
||||
// Transform represents both a record consumer and producer.
|
||||
// It has an input and output.
|
||||
// It takes a single record and provides an iterator for all the records.
|
||||
type Transform interface {
|
||||
Digester // consumer
|
||||
Iterator // producer
|
||||
}
|
48
interfaces/log-parser/oop/pipe/pipeline.go
Normal file
48
interfaces/log-parser/oop/pipe/pipeline.go
Normal file
@ -0,0 +1,48 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package pipe
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Pipeline takes records from a source, transforms, and sends them to a destionation.
|
||||
type Pipeline struct {
|
||||
src Iterator
|
||||
trans []Transform
|
||||
dst Digester
|
||||
}
|
||||
|
||||
// New creates a new pipeline.
|
||||
func New(src Iterator, dst Digester, t ...Transform) *Pipeline {
|
||||
return &Pipeline{
|
||||
src: &logCount{Iterator: src},
|
||||
dst: dst,
|
||||
trans: t,
|
||||
}
|
||||
}
|
||||
|
||||
// Run the pipeline.
|
||||
func (p *Pipeline) Run() error {
|
||||
defer func() {
|
||||
n := p.src.(*logCount).count()
|
||||
fmt.Fprintf(os.Stderr, "%d records processed.\n", n)
|
||||
}()
|
||||
|
||||
last := p.src
|
||||
|
||||
for _, t := range p.trans {
|
||||
if err := t.Digest(last); err != nil {
|
||||
return err
|
||||
}
|
||||
last = t
|
||||
}
|
||||
|
||||
return p.dst.Digest(last)
|
||||
}
|
19
interfaces/log-parser/oop/pipe/record.go
Normal file
19
interfaces/log-parser/oop/pipe/record.go
Normal file
@ -0,0 +1,19 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package pipe
|
||||
|
||||
// Record provides a generic interface for any sort of records.
|
||||
type Record interface {
|
||||
Str(field string) string
|
||||
Int(field string) int
|
||||
}
|
||||
|
||||
// Summer provides a method for summing the numeric fields.
|
||||
type Summer interface {
|
||||
Sum(Record) Record
|
||||
}
|
49
interfaces/log-parser/oop/pipe/report/chart.go
Normal file
49
interfaces/log-parser/oop/pipe/report/chart.go
Normal file
@ -0,0 +1,49 @@
|
||||
package report
|
||||
|
||||
/*
|
||||
// You need to run:
|
||||
// go get -u github.com/wcharczuk/go-chart
|
||||
|
||||
// Chart renders a chart.
|
||||
type Chart struct {
|
||||
Title string
|
||||
Width, Height int
|
||||
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
// AsChart returns a Chart report generator.
|
||||
func AsChart(w io.Writer) *Chart {
|
||||
return &Chart{w: w}
|
||||
}
|
||||
|
||||
// Digest generates a chart report.
|
||||
func (c *Chart) Digest(records pipe.Iterator) error {
|
||||
w := os.Stdout
|
||||
|
||||
donut := chart.DonutChart{
|
||||
Title: c.Title,
|
||||
TitleStyle: chart.Style{
|
||||
FontSize: 35,
|
||||
Show: true,
|
||||
FontColor: chart.ColorAlternateGreen,
|
||||
},
|
||||
Width: c.Width,
|
||||
Height: c.Height,
|
||||
}
|
||||
|
||||
records.Each(func(r pipe.Record) {
|
||||
v := chart.Value{
|
||||
Label: r.Str("domain") + r.Str("page") + ": " + strconv.Itoa(r.Int("visits")),
|
||||
Value: float64(r.Int("visits")),
|
||||
Style: chart.Style{
|
||||
FontSize: 14,
|
||||
},
|
||||
}
|
||||
|
||||
donut.Values = append(donut.Values, v)
|
||||
})
|
||||
|
||||
return donut.Render(chart.SVG, w)
|
||||
}
|
||||
*/
|
64
interfaces/log-parser/oop/pipe/report/text.go
Normal file
64
interfaces/log-parser/oop/pipe/report/text.go
Normal file
@ -0,0 +1,64 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package report
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
)
|
||||
|
||||
const (
|
||||
minWidth = 0
|
||||
tabWidth = 4
|
||||
padding = 4
|
||||
flags = 0
|
||||
)
|
||||
|
||||
// Text report generator.
|
||||
type Text struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
// AsText returns a Text report generator.
|
||||
func AsText(w io.Writer) *Text {
|
||||
return &Text{w: w}
|
||||
}
|
||||
|
||||
// Digest generates a text report.
|
||||
func (t *Text) Digest(records pipe.Iterator) error {
|
||||
w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags)
|
||||
|
||||
write := fmt.Fprintf
|
||||
|
||||
write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
|
||||
write(w, "-------\t-----\t------\t-------\n")
|
||||
|
||||
var total pipe.Record
|
||||
|
||||
records.Each(func(r pipe.Record) {
|
||||
if r, ok := r.(pipe.Summer); ok {
|
||||
total = r.Sum(total)
|
||||
}
|
||||
|
||||
write(w, "%s\t%s\t%d\t%d\n",
|
||||
r.Str("domain"), r.Str("page"),
|
||||
r.Int("visits"), r.Int("uniques"),
|
||||
)
|
||||
})
|
||||
|
||||
write(w, "\t\t\t\n")
|
||||
write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "",
|
||||
total.Int("visits"),
|
||||
total.Int("uniques"),
|
||||
)
|
||||
|
||||
return w.Flush()
|
||||
}
|
@ -1,78 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type recordFn func(record)
|
||||
|
||||
type iterator interface{ each(recordFn) error }
|
||||
type digester interface{ digest(iterator) error }
|
||||
|
||||
type transform interface {
|
||||
digester
|
||||
iterator
|
||||
}
|
||||
|
||||
type pipeline struct {
|
||||
src iterator
|
||||
trans []transform
|
||||
dst digester
|
||||
}
|
||||
|
||||
func (p *pipeline) run() error {
|
||||
defer func() {
|
||||
n := p.src.(*logCount).count()
|
||||
fmt.Printf("%d records processed.\n", n)
|
||||
}()
|
||||
|
||||
last := p.src
|
||||
|
||||
for _, t := range p.trans {
|
||||
if err := t.digest(last); err != nil {
|
||||
return err
|
||||
}
|
||||
last = t
|
||||
}
|
||||
|
||||
return p.dst.digest(last)
|
||||
}
|
||||
|
||||
func newPipeline(src iterator, dst digester, t ...transform) *pipeline {
|
||||
return &pipeline{
|
||||
src: &logCount{iterator: src},
|
||||
dst: dst,
|
||||
trans: t,
|
||||
}
|
||||
}
|
||||
|
||||
// fromFile generates a default report
|
||||
func fromFile(path string) (*pipeline, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var src iterator
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".txt"):
|
||||
src = newTextLog(f)
|
||||
case strings.HasSuffix(path, ".jsonl"):
|
||||
src = newJSONLog(f)
|
||||
}
|
||||
|
||||
return newPipeline(
|
||||
src,
|
||||
newTextReport(),
|
||||
groupBy(domainGrouper),
|
||||
), nil
|
||||
}
|
@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
type record struct {
|
||||
domain string
|
||||
page string
|
||||
visits int
|
||||
uniques int
|
||||
}
|
||||
|
||||
func (r record) sum(other record) record {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
|
||||
// UnmarshalText to a *record
|
||||
func (r *record) UnmarshalText(p []byte) (err error) {
|
||||
fields := strings.Fields(string(p))
|
||||
if len(fields) != fieldsLength {
|
||||
return fmt.Errorf("wrong number of fields %q", fields)
|
||||
}
|
||||
|
||||
r.domain, r.page = fields[0], fields[1]
|
||||
|
||||
if r.visits, err = parseStr("visits", fields[2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.uniques, err = parseStr("uniques", fields[3]); err != nil {
|
||||
return err
|
||||
}
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// UnmarshalJSON to a *record
|
||||
func (r *record) UnmarshalJSON(data []byte) error {
|
||||
var re struct {
|
||||
Domain string
|
||||
Page string
|
||||
Visits int
|
||||
Uniques int
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &re); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*r = record{re.Domain, re.Page, re.Visits, re.Uniques}
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// parseStr helps UnmarshalText for string to positive int parsing
|
||||
func parseStr(name, v string) (int, error) {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("record.UnmarshalText %q: %v", name, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func validate(r record) (err error) {
|
||||
switch {
|
||||
case r.domain == "":
|
||||
err = errors.New("record.domain cannot be empty")
|
||||
case r.page == "":
|
||||
err = errors.New("record.page cannot be empty")
|
||||
case r.visits < 0:
|
||||
err = errors.New("record.visits cannot be negative")
|
||||
case r.uniques < 0:
|
||||
err = errors.New("record.uniques cannot be negative")
|
||||
}
|
||||
return
|
||||
}
|
@ -1,49 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
)
|
||||
|
||||
// TODO: make this configurable? or exercise?
|
||||
const (
|
||||
minWidth = 0
|
||||
tabWidth = 4
|
||||
padding = 4
|
||||
flags = 0
|
||||
)
|
||||
|
||||
type textReport struct{}
|
||||
|
||||
func newTextReport() *textReport {
|
||||
return new(textReport)
|
||||
}
|
||||
|
||||
func (s *textReport) digest(records iterator) error {
|
||||
w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, ' ', flags)
|
||||
|
||||
write := fmt.Fprintf
|
||||
|
||||
write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
|
||||
write(w, "-------\t-----\t------\t-------\n")
|
||||
|
||||
var total record
|
||||
records.each(func(r record) {
|
||||
total = total.sum(r)
|
||||
|
||||
write(w, "%s\t%s\t%d\t%d\n", r.domain, r.page, r.visits, r.uniques)
|
||||
})
|
||||
|
||||
write(w, "\t\t\t\n")
|
||||
write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", total.visits, total.uniques)
|
||||
|
||||
return w.Flush()
|
||||
}
|
Reference in New Issue
Block a user