gaspersic/learngo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move: log parsers

Browse Source
This commit is contained in:
Inanc Gumus
2019-08-28 20:23:38 +03:00
parent 0a121cd911
commit 9afbe8f350
123 changed files with 1018 additions and 1515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
advfuncs/logparser/01/errdump.go
View File

@@ -1,19 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import "fmt"
// dumpErrs together to simplify multiple error handling
func dumpErrs(errs []error) {
for _, err := range errs {
if err != nil {
fmt.Println("> Err:", err)
}
}
}

38
advfuncs/logparser/01/filters.go
View File

@@ -1,38 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import "strings"
func filter(r result) bool {
return filterOrg(r)
}
func filterOrg(r result) bool {
return strings.HasSuffix(r.domain, ".org")
}
func filterBlogs(r result) bool {
return strings.HasPrefix(r.domain, "blog.")
}
// type filterFunc func(result) bool
// func filter(r result, process filterFunc) bool {
// return process(r)
// }
// func noopFilter(r result) bool {
// return true
// }
// func notUsing(filter filterFunc) filterFunc {
// return func(r result) bool {
// return !filter(r)
// }
// }

16
advfuncs/logparser/01/log.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 15
learngoprogramming.com 10
learngoprogramming.com 20
learngoprogramming.com 5
golang.org 40
golang.org 20
golang.org 45
golang.org 15
blog.golang.org 60
blog.golang.org 30
blog.golang.org 20
blog.golang.org 65
blog.golang.org 15
inanc.io 30
inanc.io 70

16
advfuncs/logparser/01/log_err_missing.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 15
learngoprogramming.com 10
learngoprogramming.com 20
learngoprogramming.com
golang.org 40
golang.org 20
golang.org 45
golang.org 15
blog.golang.org 60
blog.golang.org 30
blog.golang.org 20
blog.golang.org 65
blog.golang.org 15
inanc.io 30
inanc.io 70

16
advfuncs/logparser/01/log_err_negative.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 15
learngoprogramming.com 10
learngoprogramming.com 20
learngoprogramming.com 5
golang.org 40
golang.org 20
golang.org -50
golang.org 15
blog.golang.org 60
blog.golang.org 30
blog.golang.org 20
blog.golang.org 65
blog.golang.org 15
inanc.io 30
inanc.io 70

16
advfuncs/logparser/01/log_err_str.txt
View File

@@ -1,16 +0,0 @@
learngoprogramming.com 10
learngoprogramming.com 15
learngoprogramming.com 10
learngoprogramming.com 20
learngoprogramming.com 5
golang.org FORTY
golang.org 20
golang.org 45
golang.org 15
blog.golang.org 60
blog.golang.org 30
blog.golang.org 20
blog.golang.org 65
blog.golang.org 15
inanc.io 30
inanc.io 70

38
advfuncs/logparser/01/main.go
View File

@@ -1,38 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"os"
)
/*
fmt.Println(strings.Map(unpunct, "hello!!! HOW ARE YOU???? :))"))
fmt.Println(strings.Map(unpunct, "TIME IS UP!"))
func unpunct(r rune) rune {
if unicode.IsPunct(r) {
return -1
}
return unicode.ToLower(r)
}
*/
func main() {
p := newParser()
in := bufio.NewScanner(os.Stdin)
for in.Scan() {
r := parse(p, in.Text()) // TODO: parsed -> r
update(p, r)
}
summarize(p)
dumpErrs([]error{in.Err(), err(p)})
}

27
advfuncs/logparser/01/output.go
View File

@@ -1,27 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
// summarize the parsing results
func summarize(p *parser) {
sort.Strings(p.domains)
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
fmt.Println(strings.Repeat("-", 45))
for _, domain := range p.domains {
fmt.Printf("%-30s %10d\n", domain, p.sum[domain].visits)
}
fmt.Printf("\n%-30s %10d\n", "TOTAL", p.total)
}

65
advfuncs/logparser/01/parser.go
View File

@@ -1,65 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
// result stores details about a log line
type result struct {
domain string
visits int
// add more metrics when needed
}
// parser keep tracks of the parsing
type parser struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
total int // total visits for all domains
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
}
// newParser creates and returns a new parser
func newParser() *parser {
return &parser{sum: make(map[string]result)}
}
// update the parsing results
func parse(p *parser, line string) (r result) {
if p.lerr != nil {
return
}
p.lines++
fields := strings.Fields(line)
if len(fields) != 2 {
p.lerr = fmt.Errorf("wrong input: %v (line #%d)", fields, p.lines)
return
}
var err error
r.domain = fields[0]
r.visits, err = strconv.Atoi(fields[1])
if r.visits < 0 || err != nil {
p.lerr = fmt.Errorf("wrong input: %q (line #%d)", fields[1], p.lines)
}
return
}
// err returns the last error encountered
func err(p *parser) error {
return p.lerr
}

33
advfuncs/logparser/01/update.go
View File

@@ -1,33 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
// update updates all the parsing results using the given parsing result
func update(p *parser, r result) {
if p.lerr != nil {
return
}
if !filter(r) {
return
}
// Collect the unique domains
if _, ok := p.sum[r.domain]; !ok {
p.domains = append(p.domains, r.domain)
}
// Keep track of total and per domain visits
p.total += r.visits
// create and assign a new copy of `visit`
p.sum[r.domain] = result{
domain: r.domain,
visits: r.visits + p.sum[r.domain].visits,
}
}

7
advfuncs/logparser/functional/Makefile
View File

@@ -1,7 +0,0 @@
SHELL := /bin/bash
r:
go run . < ../logs/log.txt
t:
time go run . < ../logs/log.txt

38
advfuncs/logparser/functional/chartwriter.go
View File

@@ -1,38 +0,0 @@
package main
// func chartWriter(w io.Writer) outputFn {
// return func(res []result) error {
// return chartWrite(w, res)
// }
// }
// func chartWrite(w io.Writer, res []result) error {
// sort.Slice(res, func(i, j int) bool {
// return res[i].domain > res[j].domain
// })
// donut := chart.DonutChart{
// Title: "Total Visits Per Domain",
// TitleStyle: chart.Style{
// FontSize: 35,
// Show: true,
// FontColor: chart.ColorAlternateGreen,
// },
// Width: 1920,
// Height: 800,
// }
// for _, r := range res {
// v := chart.Value{
// Label: r.domain + r.page + ": " + strconv.Itoa(r.visits),
// Value: float64(r.visits),
// Style: chart.Style{
// FontSize: 14,
// },
// }
// donut.Values = append(donut.Values, v)
// }
// return donut.Render(chart.SVG, w)
// }

33
advfuncs/logparser/functional/field.go
View File

@@ -1,33 +0,0 @@
package main
import (
"errors"
"fmt"
"strconv"
)
// field helps for field parsing
type field struct{ err error }
// uatoi parses an unsigned integer string and saves the error.
// it assumes that the val is unsigned.
// for ease of usability: it returns an int instead of uint.
func (f *field) uatoi(name, val string) int {
n, err := strconv.Atoi(val)
if err != nil || n < 0 {
f.err = fmt.Errorf("incorrect field -> %q = %q", name, val)
}
return n
}
func atoi(input []byte) (int, error) {
val := 0
for i := 0; i < len(input); i++ {
char := input[i]
if char < '0' || char > '9' {
return 0, errors.New("invalid number")
}
val = val*10 + int(char) - '0'
}
return val, nil
}

34
advfuncs/logparser/functional/filters.go
View File

@@ -1,34 +0,0 @@
package main
import "strings"
func noopFilter(r result) bool {
return true
}
func notUsing(filter filterFn) filterFn {
return func(r result) bool {
return !filter(r)
}
}
func domainExtFilter(domains ...string) filterFn {
return func(r result) bool {
for _, domain := range domains {
if strings.HasSuffix(r.domain, "."+domain) {
return true
}
}
return false
}
}
func domainFilter(domain string) filterFn {
return func(r result) bool {
return strings.Contains(r.domain, domain)
}
}
func orgDomainsFilter(r result) bool {
return strings.HasSuffix(r.domain, ".org")
}

20
advfuncs/logparser/functional/groupers.go
View File

@@ -1,20 +0,0 @@
package main
// domainGrouper groups by domain.
// but it keeps the other fields.
// for example: it returns pages as well, but you shouldn't use them.
// exercise: write a function that erases superfluous data.
func domainGrouper(r result) string {
return r.domain
}
func pageGrouper(r result) string {
return r.domain + r.page
}
// groupBy allocates map unnecessarily
func noopGrouper(r result) string {
// with something like:
// return randomStrings()
return ""
}

44
advfuncs/logparser/functional/main.go
View File

@@ -1,44 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"os"
)
func main() {
p := pipeline{
read: textReader(os.Stdin),
write: textWriter(os.Stdout),
filter: notUsing(domainExtFilter("io", "com")),
group: domainGrouper,
}
// var p pipeline
// p.
// filterBy(notUsing(domainExtFilter("io", "com"))).
// groupBy(domainGrouper)
if err := p.start(); err != nil {
fmt.Println("> Err:", err)
}
}
// []outputter{textFile("results.txt"), chartFile("graph.png")}
// func outputs(w io.Writer) outputFn {
// tw := textWriter(w)
// cw := chartWriter(w)
// return func(rs []result) error {
// err := tw(rs)
// err = cw(rs)
// return err
// }
// }

78
advfuncs/logparser/functional/pipeline.go
View File

@@ -1,78 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import "os"
type (
processFn func(r result)
inputFn func(processFn) error
outputFn func([]result) error
filterFn func(result) (include bool)
groupFn func(result) (key string)
)
type pipeline struct {
read inputFn
write outputFn
filter filterFn
group groupFn
}
func (p *pipeline) filterBy(f filterFn) *pipeline { p.filter = f; return p }
func (p *pipeline) groupBy(f groupFn) *pipeline { p.group = f; return p }
func (p *pipeline) from(f inputFn) *pipeline { p.read = f; return p }
func (p *pipeline) to(f outputFn) *pipeline { p.write = f; return p }
func (p *pipeline) defaults() {
if p.filter == nil {
p.filter = noopFilter
}
if p.group == nil {
p.group = domainGrouper
}
if p.read == nil {
p.read = textReader(os.Stdin)
}
if p.write == nil {
p.write = textWriter(os.Stdout)
}
}
func (p *pipeline) start() error {
p.defaults()
// retrieve and process the lines
sum := make(map[string]result)
process := func(r result) {
if !p.filter(r) {
return
}
k := p.group(r)
sum[k] = r.add(sum[k])
}
// return err from input reader
if err := p.read(process); err != nil {
return err
}
// prepare the results for outputting
var out []result
for _, res := range sum {
out = append(out, res)
}
// return err from output reader
return p.write(out)
}

83
advfuncs/logparser/functional/result.go
View File

@@ -1,83 +0,0 @@
package main
import (
"fmt"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain string
page string
visits int
uniques int
}
// add adds the metrics of another result
func (r result) add(other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}
// parseFields parses and returns the parsing result
func parseFields(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong number of fields %q", fields)
}
r.domain = fields[0]
r.page = fields[1]
f := new(field)
r.visits = f.uatoi("visits", fields[2])
r.uniques = f.uatoi("uniques", fields[3])
return r, f.err
}
func fastParseFields(data []byte) (res result, err error) {
const separator = ' '
var findex int
for i, j := 0, 0; i < len(data); i++ {
c := data[i]
last := len(data) == i+1
if c != separator && !last {
continue
}
if last {
i = len(data)
}
switch fval := data[j:i]; findex {
case 0:
res.domain = string(fval)
case 1:
res.page = string(fval)
case 2:
res.visits, err = atoi(fval)
case 3:
res.uniques, err = atoi(fval)
}
if err != nil {
return res, err
}
j = i + 1
findex++
}
if findex != fieldsLength {
err = fmt.Errorf("wrong number of fields %q", data)
}
return res, err
}

39
advfuncs/logparser/functional/textreader.go
View File

@@ -1,39 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"io"
)
func textReader(r io.Reader) inputFn {
return func(process processFn) error {
var (
l = 1
in = bufio.NewScanner(r)
)
for in.Scan() {
r, err := fastParseFields(in.Bytes())
// r, err := parseFields(in.Text())
if err != nil {
return fmt.Errorf("line %d: %v", l, err)
}
process(r)
l++
}
if c, ok := r.(io.Closer); ok {
c.Close()
}
return in.Err()
}
}

50
advfuncs/logparser/functional/textwriter.go
View File

@@ -1,50 +0,0 @@
package main
import (
"fmt"
"io"
"sort"
"strings"
)
// TODO: sort by result key interfaces section
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
func textWriter(w io.Writer) outputFn {
return func(res []result) error {
sort.Slice(res, func(i, j int) bool {
return res[i].domain > res[j].domain
})
var total result
fmt.Fprintf(w, header, "DOMAINS", "PAGES", "VISITS", "UNIQUES")
fmt.Fprintf(w, strings.Repeat(dash, dashLength)+"\n")
for _, r := range res {
total = total.add(r)
fmt.Fprintf(w, line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Fprintf(w, footer, "", total.visits, total.uniques)
return nil
}
}
func noWhere() outputFn {
return func(res []result) error {
return nil
}
}

34
advfuncs/logparser/refactor-notes/refactor-00/changes.md
View File

@@ -1,34 +0,0 @@
## CHANGES
### PROBLEM
+ adding new fields makes the code complex
+ needs to update: `result`, `parser`, `summarizer`
+ needs to add new fields to `parser`: `totalVisits` + `totalUniques`
+ in `parse()`: repeating line errors
+ if we parsing out of it we'd need to have *parser — superfluous
### SOLUTION
+ move all the result related logic to result.go
+ move `parser.go/result` -> `result.go`
+ move `parser.go/parsing` logic -> `result.go`
+ add `addResult` -> `result.go`
+ remove `parser struct`'s: `totalVisits`, `totalUniques`
+ change `update()`'s last line: `p.sum[r.domain] = addResult`
+ remove `(line #d)` errors from `result.go`
+ add: `return r, err` — named params are error prone
+ always check for the error first
+ `if r.visits < 0 || err != nil` -> `if err != nil || r.visits < 0`
+ `parser.go`: check the `parseFields()`:
```golang
r, err := parseFields(line)
if err != nil {
p.lerr = fmt.Errorf("line %d: %v", p.lines, err)
}```
+ - `parser.go` and `summarize.go`
- remove `total int`
- let `summarize()` calculate the totals

36
advfuncs/logparser/refactor-notes/refactor-00/main.go
View File

@@ -1,36 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"os"
)
func main() {
p := newParser()
in := bufio.NewScanner(os.Stdin)
for in.Scan() {
parsed := parse(p, in.Text())
update(p, parsed)
}
summarize(p)
dumpErrs([]error{in.Err(), err(p)})
}
// dumpErrs simplifies handling multiple errors
func dumpErrs(errs []error) {
for _, err := range errs {
if err != nil {
fmt.Println("> Err:", err)
}
}
}

74
advfuncs/logparser/refactor-notes/refactor-00/parser.go
View File

@@ -1,74 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
)
// parser keeps track of the parsing
type parser struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
lines int // number of parsed lines (for the error messages)
lerr error // the last error occurred
// totalVisits int // total visits for all domains
// totalUniques int // total uniques for all domains
}
// newParser constructs, initializes and returns a new parser
func newParser() *parser {
return &parser{sum: make(map[string]result)}
}
// parse a log line and return the result
func parse(p *parser, line string) (r result) {
if p.lerr != nil {
return
}
p.lines++
r, err := parseResult(line)
if err != nil {
p.lerr = fmt.Errorf("line %d: %v", p.lines, err)
}
return r
}
// update the parsing results
func update(p *parser, r result) {
if p.lerr != nil {
return
}
// Collect the unique domains
cur, ok := p.sum[r.domain]
if !ok {
p.domains = append(p.domains, r.domain)
}
// Keep track of total and per domain visits
// p.totalVisits += r.visits
// p.totalUniques += r.uniques
// create and assign a new copy of `visit`
// p.sum[r.domain] = result{
// domain: r.domain,
// visits: r.visits + cur.visits,
// uniques: r.uniques + cur.uniques,
// }
p.sum[r.domain] = addResult(r, cur)
}
// err returns the last error encountered
func err(p *parser) error {
return p.lerr
}

53
advfuncs/logparser/refactor-notes/refactor-00/result.go
View File

@@ -1,53 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain, page string
visits, uniques int
// add more metrics if needed
}
// parseResult from a log line
func parseResult(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong input: %v", fields)
}
r.domain = fields[0]
r.page = fields[1]
r.visits, err = strconv.Atoi(fields[2])
if err != nil || r.visits < 0 {
return r, fmt.Errorf("wrong input: %q", fields[2])
}
r.uniques, err = strconv.Atoi(fields[3])
if err != nil || r.uniques < 0 {
return r, fmt.Errorf("wrong input: %q", fields[3])
}
return r, nil
}
// addResult to another one
func addResult(r, other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}

43
advfuncs/logparser/refactor-notes/refactor-00/summarize.go
View File

@@ -1,43 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
// summarize summarizes and prints the parsing result
func summarize(p *parser) {
sort.Strings(p.domains)
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
fmt.Println(strings.Repeat("-", dashLength))
var total result
for _, domain := range p.domains {
r := p.sum[domain]
total = addResult(total, r)
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
}

61
advfuncs/logparser/refactor-notes/refactor-01/changes.md
View File

@@ -1,61 +0,0 @@
### PROBLEM
+ `main.go` (api client) does a lot of things:
+ read the log input
+ parse line by line
+ updates the results
+ display the results
+ inflexible:
+ filter by extension (can change)
+ group by domain (can change) — group by page?
## SOLUTION
+ hide the parsing api from the client
+ move `main.go/scanner` -> `parser.go/parse()`
+ add `main.go`: err handling from `parse()`
+ `parser.go/parse()` -> return err directly
+ remove: `if p.lerr != nil { return }` from parse() and update()
+ remove: `dumpErrs`
+ remove: `parser.go/err()`
+ remove `parser.go/lerr`
+ return `in.Err()` from `parse()`
+ remove: `p.lines++`
+ `return r, fmt.Errorf("line %d: %v", p.lines, err)`
+ remove: `lines int`
+ `parse()` and `parse()` becomes:
```golang
func parse(p *parser, line string) (result, error) {
return parseFields(line)
}
func parse(p *parser) {
// ...
r, err := parse(p, in.Text())
if err != nil {
return fmt.Errorf("line %d: %v", p.lines, err)
}
// ...
}
```
+ remove `parse()`
+ call `parseFields` directly in `parse()`:
```go
var (
l = 1
in = bufio.NewScanner(os.Stdin)
)
for in.Scan() {
r, err := parseFields(in.Text())
if err != nil {
return fmt.Errorf("line %d: %v", l, err)
}
update(p, r)
l++
}
```

23
advfuncs/logparser/refactor-notes/refactor-01/main.go
View File

@@ -1,23 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
)
func main() {
p := newParser()
if err := parse(p); err != nil {
fmt.Println("> Err:", err)
return
}
summarize(p)
}

57
advfuncs/logparser/refactor-notes/refactor-01/parser.go
View File

@@ -1,57 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"os"
)
// parser keeps track of the parsing
type parser struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
}
// newParser constructs, initializes and returns a new parser
func newParser() *parser {
return &parser{sum: make(map[string]result)}
}
// parse the log lines and return results
func parse(p *parser) error {
var (
l = 1
in = bufio.NewScanner(os.Stdin)
)
for in.Scan() {
r, err := parseResult(in.Text())
if err != nil {
return fmt.Errorf("line %d: %v", l, err)
}
l++
update(p, r)
}
return in.Err()
}
// update the parsing results
func update(p *parser, r result) {
// Collect the unique domains
if _, ok := p.sum[r.domain]; !ok {
p.domains = append(p.domains, r.domain)
}
// create and assign a new copy of `visit`
p.sum[r.domain] = addResult(r, p.sum[r.domain])
}

53
advfuncs/logparser/refactor-notes/refactor-01/result.go
View File

@@ -1,53 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain, page string
visits, uniques int
// add more metrics if needed
}
// parseResult from a log line
func parseResult(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong input: %v", fields)
}
r.domain = fields[0]
r.page = fields[1]
r.visits, err = strconv.Atoi(fields[2])
if err != nil || r.visits < 0 {
return r, fmt.Errorf("wrong input: %q", fields[2])
}
r.uniques, err = strconv.Atoi(fields[3])
if err != nil || r.uniques < 0 {
return r, fmt.Errorf("wrong input: %q", fields[3])
}
return r, nil
}
// addResult to another one
func addResult(r, other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}

48
advfuncs/logparser/refactor-notes/refactor-01/summarize.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
// summarize summarizes and prints the parsing result
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
// + give it the []result only.
// + let it calculate the total.
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
// summarize summarizes and prints the parsing result
func summarize(p *parser) {
sort.Strings(p.domains)
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
fmt.Println(strings.Repeat("-", dashLength))
var total result
for _, domain := range p.domains {
r := p.sum[domain]
total = addResult(total, r)
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
}

20
advfuncs/logparser/refactor-notes/refactor-02/changes.md
View File

@@ -1,20 +0,0 @@
### PROBLEM
+ `summarize()` knows a lot about the internals of the `parser`.
+ coupled to the `parser`.
## SOLUTION
+ remove: `parser.go` `sum` and `domains` fields
+ remove: `parser.go/newParser()`
+ change: `parser.go/parse(p *parser) error` -> `parse() ([]result, error)`
+ initialize: `sum` inside `parse()`
+ remove: `update()`
+ call: `sum` update in the `parse()`
+ collect the grouped results and return them from `parser()`
+ `summarize(p *parser)` -> `summarize([]result)`
+ in `summarize()`
+ `sort.Slice`
+ range over `[]result`
+ `main.go`
+ just: `res, err := parse()`

22
advfuncs/logparser/refactor-notes/refactor-02/main.go
View File

@@ -1,22 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
)
func main() {
res, err := parse()
if err != nil {
fmt.Println("> Err:", err)
return
}
summarize(res)
}

48
advfuncs/logparser/refactor-notes/refactor-02/parser.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"os"
)
// parser keeps track of the parsing
type parser struct {
}
// parse the log lines and return results
func parse() ([]result, error) {
var (
l = 1
in = bufio.NewScanner(os.Stdin)
sum = make(map[string]result)
)
// parse the log lines
for in.Scan() {
r, err := parseResult(in.Text())
if err != nil {
return nil, fmt.Errorf("line %d: %v", l, err)
}
l++
// group the log lines by domain
sum[r.domain] = addResult(r, sum[r.domain])
}
// collect the grouped results
res := make([]result, 0, len(sum))
for _, r := range sum {
res = append(res, r)
}
return res, in.Err()
}

53
advfuncs/logparser/refactor-notes/refactor-02/result.go
View File

@@ -1,53 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain, page string
visits, uniques int
// add more metrics if needed
}
// parseResult from a log line
func parseResult(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong input: %v", fields)
}
r.domain = fields[0]
r.page = fields[1]
r.visits, err = strconv.Atoi(fields[2])
if err != nil || r.visits < 0 {
return r, fmt.Errorf("wrong input: %q", fields[2])
}
r.uniques, err = strconv.Atoi(fields[3])
if err != nil || r.uniques < 0 {
return r, fmt.Errorf("wrong input: %q", fields[3])
}
return r, nil
}
// addResult to another one
func addResult(r, other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}

49
advfuncs/logparser/refactor-notes/refactor-02/summarize.go
View File

@@ -1,49 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
// summarize summarizes and prints the parsing result
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
// + give it the []result only.
// + let it calculate the total.
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
// summarize summarizes and prints the parsing result
func summarize(res []result) {
// sort.Strings(p.domains)
sort.Slice(res, func(i, j int) bool {
return res[i].domain <= res[j].domain
})
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
fmt.Println(strings.Repeat("-", dashLength))
var total result
for _, r := range res {
total = addResult(total, r)
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
}

14
advfuncs/logparser/refactor-notes/refactor-03-deprecated/changes.md
View File

@@ -1,14 +0,0 @@
### PROBLEM
+ `parser.go/parse()` also does updating. back to square one.
+ we need to extract the reusable behavior: scanning.
+ inflexible:
+ adding a filter is hard. needs to change the `scan()` code.
+ adding a grouper is also hard. domain grouping is hardcoded.
## SOLUTION
+
## IDEAS:
+ make domain filter accept variadic args

36
advfuncs/logparser/refactor-notes/refactor-03-deprecated/main.go
View File

@@ -1,36 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
)
/*
p := pipeline{
read: textReader(os.Stdin),
write: textWriter(os.Stdout),
filterBy: notUsing(domainExtFilter("io")),
groupBy: domainGrouper,
}
if err := start(p); err != nil {
fmt.Println("> Err:", err)
}
*/
func main() {
p := newParser()
if err := parse(p); err != nil {
fmt.Println("> Err:", err)
return
}
summarize(p)
}

40
advfuncs/logparser/refactor-notes/refactor-03-deprecated/parser.go
View File

@@ -1,40 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
// parser keeps track of the parsing
type parser struct {
sum map[string]result // metrics per domain
domains []string // unique domain names
}
// newParser constructs, initializes and returns a new parser
func newParser() *parser {
return &parser{sum: make(map[string]result)}
}
// parse all the log lines and update the results
func parse(p *parser) error {
process := func(r result) {
update(p, r)
}
err := scan(process)
return err
}
func update(p *parser, r result) {
// Collect the unique domains
if _, ok := p.sum[r.domain]; !ok {
p.domains = append(p.domains, r.domain)
}
// create and assign a new copy of `visit`
p.sum[r.domain] = addResult(r, p.sum[r.domain])
}

53
advfuncs/logparser/refactor-notes/refactor-03-deprecated/result.go
View File

@@ -1,53 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain, page string
visits, uniques int
// add more metrics if needed
}
// parseResult from a log line
func parseResult(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong input: %v", fields)
}
r.domain = fields[0]
r.page = fields[1]
r.visits, err = strconv.Atoi(fields[2])
if err != nil || r.visits < 0 {
return r, fmt.Errorf("wrong input: %q", fields[2])
}
r.uniques, err = strconv.Atoi(fields[3])
if err != nil || r.uniques < 0 {
return r, fmt.Errorf("wrong input: %q", fields[3])
}
return r, nil
}
// addResult to another one
func addResult(r, other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}

35
advfuncs/logparser/refactor-notes/refactor-03-deprecated/scanner.go
View File

@@ -1,35 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"os"
)
type processFn func(r result)
func scan(process processFn) error {
var (
l = 1
in = bufio.NewScanner(os.Stdin)
)
for in.Scan() {
r, err := parseResult(in.Text())
if err != nil {
return fmt.Errorf("line %d: %v", l, err)
}
l++
process(r)
}
return in.Err()
}

48
advfuncs/logparser/refactor-notes/refactor-03-deprecated/summarize.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
// summarize summarizes and prints the parsing result
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
// + give it the []result only.
// + let it calculate the total.
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
// summarize summarizes and prints the parsing result
func summarize(p *parser) {
sort.Strings(p.domains)
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
fmt.Println(strings.Repeat("-", dashLength))
var total result
for _, domain := range p.domains {
r := p.sum[domain]
total = addResult(total, r)
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
}

7
advfuncs/logparser/refactor-notes/refactor-03/changes.md
View File

@@ -1,7 +0,0 @@
### PROBLEM
+ ...
## SOLUTION
+ `parser struct` -> `pipeline struct`
+ `parse()` -> `pipe(pipeline)`

29
advfuncs/logparser/refactor-notes/refactor-03/main.go
View File

@@ -1,29 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
)
func main() {
pipes := pipeline{
// read: textParser(),
// write: textSummary(),
// filterBy: notUsing(domainExtFilter("io", "com")),
// groupBy: domainGrouper,
}
res, err := pipe(pipes)
if err != nil {
fmt.Println("> Err:", err)
return
}
summarize(res)
}

48
advfuncs/logparser/refactor-notes/refactor-03/parser.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"bufio"
"fmt"
"os"
)
// pipeline determines the behavior of log processing
type pipeline struct {
}
// pipe the log lines through funcs and produce a result
func pipe(opts pipeline) ([]result, error) {
var (
l = 1
in = bufio.NewScanner(os.Stdin)
sum = make(map[string]result)
)
// parse the log lines
for in.Scan() {
r, err := parseResult(in.Text())
if err != nil {
return nil, fmt.Errorf("line %d: %v", l, err)
}
l++
// group the log lines by domain
sum[r.domain] = addResult(r, sum[r.domain])
}
// collect the grouped results
res := make([]result, 0, len(sum))
for _, r := range sum {
res = append(res, r)
}
return res, in.Err()
}

53
advfuncs/logparser/refactor-notes/refactor-03/result.go
View File

@@ -1,53 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"strconv"
"strings"
)
const fieldsLength = 4
// result stores the parsed result for a domain
type result struct {
domain, page string
visits, uniques int
// add more metrics if needed
}
// parseResult from a log line
func parseResult(line string) (r result, err error) {
fields := strings.Fields(line)
if len(fields) != fieldsLength {
return r, fmt.Errorf("wrong input: %v", fields)
}
r.domain = fields[0]
r.page = fields[1]
r.visits, err = strconv.Atoi(fields[2])
if err != nil || r.visits < 0 {
return r, fmt.Errorf("wrong input: %q", fields[2])
}
r.uniques, err = strconv.Atoi(fields[3])
if err != nil || r.uniques < 0 {
return r, fmt.Errorf("wrong input: %q", fields[3])
}
return r, nil
}
// addResult to another one
func addResult(r, other result) result {
r.visits += other.visits
r.uniques += other.uniques
return r
}

48
advfuncs/logparser/refactor-notes/refactor-03/summarize.go
View File

@@ -1,48 +0,0 @@
// For more tutorials: https://blog.learngoprogramming.com
//
// Copyright © 2018 Inanc Gumus
// Learn Go Programming Course
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
//
package main
import (
"fmt"
"sort"
"strings"
)
// summarize summarizes and prints the parsing result
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
// + give it the []result only.
// + let it calculate the total.
const (
// DOMAINS PAGES VISITS UNIQUES
// ^ ^ ^ ^
// | | | |
header = "%-25s %-10s %10s %10s\n"
line = "%-25s %-10s %10d %10d\n"
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
dash = "-"
dashLength = 58
)
// summarize summarizes and prints the parsing result
func summarize(res []result) {
sort.Slice(res, func(i, j int) bool {
return res[i].domain <= res[j].domain
})
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
fmt.Println(strings.Repeat("-", dashLength))
var total result
for _, r := range res {
total = addResult(total, r)
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
}
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
}