move: log parsers
This commit is contained in:
Inanc Gumus
@ -1,19 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
// dumpErrs together to simplify multiple error handling
|
||||
func dumpErrs(errs []error) {
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import "strings"
|
||||
|
||||
func filter(r result) bool {
|
||||
return filterOrg(r)
|
||||
}
|
||||
|
||||
func filterOrg(r result) bool {
|
||||
return strings.HasSuffix(r.domain, ".org")
|
||||
}
|
||||
|
||||
func filterBlogs(r result) bool {
|
||||
return strings.HasPrefix(r.domain, "blog.")
|
||||
}
|
||||
|
||||
// type filterFunc func(result) bool
|
||||
|
||||
// func filter(r result, process filterFunc) bool {
|
||||
// return process(r)
|
||||
// }
|
||||
|
||||
// func noopFilter(r result) bool {
|
||||
// return true
|
||||
// }
|
||||
|
||||
// func notUsing(filter filterFunc) filterFunc {
|
||||
// return func(r result) bool {
|
||||
// return !filter(r)
|
||||
// }
|
||||
// }
|
||||
@ -1,16 +0,0 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 15
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 20
|
||||
learngoprogramming.com 5
|
||||
golang.org 40
|
||||
golang.org 20
|
||||
golang.org 45
|
||||
golang.org 15
|
||||
blog.golang.org 60
|
||||
blog.golang.org 30
|
||||
blog.golang.org 20
|
||||
blog.golang.org 65
|
||||
blog.golang.org 15
|
||||
inanc.io 30
|
||||
inanc.io 70
|
||||
@ -1,16 +0,0 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 15
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 20
|
||||
learngoprogramming.com
|
||||
golang.org 40
|
||||
golang.org 20
|
||||
golang.org 45
|
||||
golang.org 15
|
||||
blog.golang.org 60
|
||||
blog.golang.org 30
|
||||
blog.golang.org 20
|
||||
blog.golang.org 65
|
||||
blog.golang.org 15
|
||||
inanc.io 30
|
||||
inanc.io 70
|
||||
@ -1,16 +0,0 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 15
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 20
|
||||
learngoprogramming.com 5
|
||||
golang.org 40
|
||||
golang.org 20
|
||||
golang.org -50
|
||||
golang.org 15
|
||||
blog.golang.org 60
|
||||
blog.golang.org 30
|
||||
blog.golang.org 20
|
||||
blog.golang.org 65
|
||||
blog.golang.org 15
|
||||
inanc.io 30
|
||||
inanc.io 70
|
||||
@ -1,16 +0,0 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 15
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 20
|
||||
learngoprogramming.com 5
|
||||
golang.org FORTY
|
||||
golang.org 20
|
||||
golang.org 45
|
||||
golang.org 15
|
||||
blog.golang.org 60
|
||||
blog.golang.org 30
|
||||
blog.golang.org 20
|
||||
blog.golang.org 65
|
||||
blog.golang.org 15
|
||||
inanc.io 30
|
||||
inanc.io 70
|
||||
@ -1,38 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
)
|
||||
|
||||
/*
|
||||
fmt.Println(strings.Map(unpunct, "hello!!! HOW ARE YOU???? :))"))
|
||||
fmt.Println(strings.Map(unpunct, "TIME IS UP!"))
|
||||
|
||||
func unpunct(r rune) rune {
|
||||
if unicode.IsPunct(r) {
|
||||
return -1
|
||||
}
|
||||
return unicode.ToLower(r)
|
||||
}
|
||||
*/
|
||||
|
||||
func main() {
|
||||
p := newParser()
|
||||
|
||||
in := bufio.NewScanner(os.Stdin)
|
||||
for in.Scan() {
|
||||
r := parse(p, in.Text()) // TODO: parsed -> r
|
||||
update(p, r)
|
||||
}
|
||||
|
||||
summarize(p)
|
||||
dumpErrs([]error{in.Err(), err(p)})
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summarize the parsing results
|
||||
func summarize(p *parser) {
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
|
||||
fmt.Println(strings.Repeat("-", 45))
|
||||
|
||||
for _, domain := range p.domains {
|
||||
fmt.Printf("%-30s %10d\n", domain, p.sum[domain].visits)
|
||||
}
|
||||
fmt.Printf("\n%-30s %10d\n", "TOTAL", p.total)
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
// update updates all the parsing results using the given parsing result
|
||||
func update(p *parser, r result) {
|
||||
if p.lerr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !filter(r) {
|
||||
return
|
||||
}
|
||||
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[r.domain]; !ok {
|
||||
p.domains = append(p.domains, r.domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
p.total += r.visits
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[r.domain] = result{
|
||||
domain: r.domain,
|
||||
visits: r.visits + p.sum[r.domain].visits,
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
## CHANGES
|
||||
|
||||
### PROBLEM
|
||||
+ adding new fields makes the code complex
|
||||
+ needs to update: `result`, `parser`, `summarizer`
|
||||
+ needs to add new fields to `parser`: `totalVisits` + `totalUniques`
|
||||
+ in `parse()`: repeating line errors
|
||||
+ if we parsing out of it we'd need to have *parser — superfluous
|
||||
|
||||
### SOLUTION
|
||||
+ move all the result related logic to result.go
|
||||
|
||||
+ move `parser.go/result` -> `result.go`
|
||||
+ move `parser.go/parsing` logic -> `result.go`
|
||||
|
||||
+ add `addResult` -> `result.go`
|
||||
+ remove `parser struct`'s: `totalVisits`, `totalUniques`
|
||||
+ change `update()`'s last line: `p.sum[r.domain] = addResult`
|
||||
|
||||
+ remove `(line #d)` errors from `result.go`
|
||||
+ add: `return r, err` — named params are error prone
|
||||
+ always check for the error first
|
||||
+ `if r.visits < 0 || err != nil` -> `if err != nil || r.visits < 0`
|
||||
|
||||
+ `parser.go`: check the `parseFields()`:
|
||||
```golang
|
||||
r, err := parseFields(line)
|
||||
if err != nil {
|
||||
p.lerr = fmt.Errorf("line %d: %v", p.lines, err)
|
||||
}```
|
||||
|
||||
+ - `parser.go` and `summarize.go`
|
||||
- remove `total int`
|
||||
- let `summarize()` calculate the totals
|
||||
@ -1,74 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// parser keeps track of the parsing
|
||||
type parser struct {
|
||||
sum map[string]result // metrics per domain
|
||||
domains []string // unique domain names
|
||||
lines int // number of parsed lines (for the error messages)
|
||||
lerr error // the last error occurred
|
||||
|
||||
// totalVisits int // total visits for all domains
|
||||
// totalUniques int // total uniques for all domains
|
||||
}
|
||||
|
||||
// newParser constructs, initializes and returns a new parser
|
||||
func newParser() *parser {
|
||||
return &parser{sum: make(map[string]result)}
|
||||
}
|
||||
|
||||
// parse a log line and return the result
|
||||
func parse(p *parser, line string) (r result) {
|
||||
if p.lerr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
p.lines++
|
||||
|
||||
r, err := parseResult(line)
|
||||
if err != nil {
|
||||
p.lerr = fmt.Errorf("line %d: %v", p.lines, err)
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// update the parsing results
|
||||
func update(p *parser, r result) {
|
||||
if p.lerr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Collect the unique domains
|
||||
cur, ok := p.sum[r.domain]
|
||||
if !ok {
|
||||
p.domains = append(p.domains, r.domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
// p.totalVisits += r.visits
|
||||
// p.totalUniques += r.uniques
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
// p.sum[r.domain] = result{
|
||||
// domain: r.domain,
|
||||
// visits: r.visits + cur.visits,
|
||||
// uniques: r.uniques + cur.uniques,
|
||||
// }
|
||||
p.sum[r.domain] = addResult(r, cur)
|
||||
}
|
||||
|
||||
// err returns the last error encountered
|
||||
func err(p *parser) error {
|
||||
return p.lerr
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain, page string
|
||||
visits, uniques int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parseResult from a log line
|
||||
func parseResult(line string) (r result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != fieldsLength {
|
||||
return r, fmt.Errorf("wrong input: %v", fields)
|
||||
}
|
||||
|
||||
r.domain = fields[0]
|
||||
r.page = fields[1]
|
||||
|
||||
r.visits, err = strconv.Atoi(fields[2])
|
||||
if err != nil || r.visits < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[2])
|
||||
}
|
||||
|
||||
r.uniques, err = strconv.Atoi(fields[3])
|
||||
if err != nil || r.uniques < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[3])
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// addResult to another one
|
||||
func addResult(r, other result) result {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// DOMAINS PAGES VISITS UNIQUES
|
||||
// ^ ^ ^ ^
|
||||
// | | | |
|
||||
header = "%-25s %-10s %10s %10s\n"
|
||||
line = "%-25s %-10s %10d %10d\n"
|
||||
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
|
||||
dash = "-"
|
||||
dashLength = 58
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(p *parser) {
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
|
||||
fmt.Println(strings.Repeat("-", dashLength))
|
||||
|
||||
var total result
|
||||
|
||||
for _, domain := range p.domains {
|
||||
r := p.sum[domain]
|
||||
total = addResult(total, r)
|
||||
|
||||
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
|
||||
}
|
||||
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
### PROBLEM
|
||||
+ `main.go` (api client) does a lot of things:
|
||||
+ read the log input
|
||||
+ parse line by line
|
||||
+ updates the results
|
||||
+ display the results
|
||||
|
||||
+ inflexible:
|
||||
+ filter by extension (can change)
|
||||
+ group by domain (can change) — group by page?
|
||||
|
||||
## SOLUTION
|
||||
+ hide the parsing api from the client
|
||||
|
||||
+ move `main.go/scanner` -> `parser.go/parse()`
|
||||
+ add `main.go`: err handling from `parse()`
|
||||
|
||||
+ `parser.go/parse()` -> return err directly
|
||||
+ remove: `if p.lerr != nil { return }` from parse() and update()
|
||||
+ remove: `dumpErrs`
|
||||
+ remove: `parser.go/err()`
|
||||
+ remove `parser.go/lerr`
|
||||
+ return `in.Err()` from `parse()`
|
||||
|
||||
+ remove: `p.lines++`
|
||||
+ `return r, fmt.Errorf("line %d: %v", p.lines, err)`
|
||||
+ remove: `lines int`
|
||||
+ `parse()` and `parse()` becomes:
|
||||
```golang
|
||||
func parse(p *parser, line string) (result, error) {
|
||||
return parseFields(line)
|
||||
}
|
||||
|
||||
func parse(p *parser) {
|
||||
// ...
|
||||
r, err := parse(p, in.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("line %d: %v", p.lines, err)
|
||||
}
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
+ remove `parse()`
|
||||
+ call `parseFields` directly in `parse()`:
|
||||
```go
|
||||
var (
|
||||
l = 1
|
||||
in = bufio.NewScanner(os.Stdin)
|
||||
)
|
||||
|
||||
for in.Scan() {
|
||||
r, err := parseFields(in.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("line %d: %v", l, err)
|
||||
}
|
||||
|
||||
update(p, r)
|
||||
l++
|
||||
}
|
||||
```
|
||||
@ -1,23 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
p := newParser()
|
||||
|
||||
if err := parse(p); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
return
|
||||
}
|
||||
|
||||
summarize(p)
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// parser keeps track of the parsing
|
||||
type parser struct {
|
||||
sum map[string]result // metrics per domain
|
||||
domains []string // unique domain names
|
||||
}
|
||||
|
||||
// newParser constructs, initializes and returns a new parser
|
||||
func newParser() *parser {
|
||||
return &parser{sum: make(map[string]result)}
|
||||
}
|
||||
|
||||
// parse the log lines and return results
|
||||
func parse(p *parser) error {
|
||||
var (
|
||||
l = 1
|
||||
in = bufio.NewScanner(os.Stdin)
|
||||
)
|
||||
|
||||
for in.Scan() {
|
||||
r, err := parseResult(in.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("line %d: %v", l, err)
|
||||
}
|
||||
|
||||
l++
|
||||
|
||||
update(p, r)
|
||||
}
|
||||
|
||||
return in.Err()
|
||||
}
|
||||
|
||||
// update the parsing results
|
||||
func update(p *parser, r result) {
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[r.domain]; !ok {
|
||||
p.domains = append(p.domains, r.domain)
|
||||
}
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[r.domain] = addResult(r, p.sum[r.domain])
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain, page string
|
||||
visits, uniques int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parseResult from a log line
|
||||
func parseResult(line string) (r result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != fieldsLength {
|
||||
return r, fmt.Errorf("wrong input: %v", fields)
|
||||
}
|
||||
|
||||
r.domain = fields[0]
|
||||
r.page = fields[1]
|
||||
|
||||
r.visits, err = strconv.Atoi(fields[2])
|
||||
if err != nil || r.visits < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[2])
|
||||
}
|
||||
|
||||
r.uniques, err = strconv.Atoi(fields[3])
|
||||
if err != nil || r.uniques < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[3])
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// addResult to another one
|
||||
func addResult(r, other result) result {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
|
||||
// + give it the []result only.
|
||||
// + let it calculate the total.
|
||||
const (
|
||||
|
||||
// DOMAINS PAGES VISITS UNIQUES
|
||||
// ^ ^ ^ ^
|
||||
// | | | |
|
||||
header = "%-25s %-10s %10s %10s\n"
|
||||
line = "%-25s %-10s %10d %10d\n"
|
||||
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
|
||||
dash = "-"
|
||||
dashLength = 58
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(p *parser) {
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
|
||||
fmt.Println(strings.Repeat("-", dashLength))
|
||||
|
||||
var total result
|
||||
|
||||
for _, domain := range p.domains {
|
||||
r := p.sum[domain]
|
||||
total = addResult(total, r)
|
||||
|
||||
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
|
||||
}
|
||||
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
### PROBLEM
|
||||
+ `summarize()` knows a lot about the internals of the `parser`.
|
||||
+ coupled to the `parser`.
|
||||
|
||||
## SOLUTION
|
||||
+ remove: `parser.go` `sum` and `domains` fields
|
||||
+ remove: `parser.go/newParser()`
|
||||
+ change: `parser.go/parse(p *parser) error` -> `parse() ([]result, error)`
|
||||
+ initialize: `sum` inside `parse()`
|
||||
+ remove: `update()`
|
||||
+ call: `sum` update in the `parse()`
|
||||
+ collect the grouped results and return them from `parser()`
|
||||
|
||||
+ `summarize(p *parser)` -> `summarize([]result)`
|
||||
+ in `summarize()`
|
||||
+ `sort.Slice`
|
||||
+ range over `[]result`
|
||||
|
||||
+ `main.go`
|
||||
+ just: `res, err := parse()`
|
||||
@ -1,48 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// parser keeps track of the parsing
|
||||
type parser struct {
|
||||
}
|
||||
|
||||
// parse the log lines and return results
|
||||
func parse() ([]result, error) {
|
||||
var (
|
||||
l = 1
|
||||
in = bufio.NewScanner(os.Stdin)
|
||||
sum = make(map[string]result)
|
||||
)
|
||||
|
||||
// parse the log lines
|
||||
for in.Scan() {
|
||||
r, err := parseResult(in.Text())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: %v", l, err)
|
||||
}
|
||||
|
||||
l++
|
||||
|
||||
// group the log lines by domain
|
||||
sum[r.domain] = addResult(r, sum[r.domain])
|
||||
}
|
||||
|
||||
// collect the grouped results
|
||||
res := make([]result, 0, len(sum))
|
||||
for _, r := range sum {
|
||||
res = append(res, r)
|
||||
}
|
||||
|
||||
return res, in.Err()
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain, page string
|
||||
visits, uniques int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parseResult from a log line
|
||||
func parseResult(line string) (r result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != fieldsLength {
|
||||
return r, fmt.Errorf("wrong input: %v", fields)
|
||||
}
|
||||
|
||||
r.domain = fields[0]
|
||||
r.page = fields[1]
|
||||
|
||||
r.visits, err = strconv.Atoi(fields[2])
|
||||
if err != nil || r.visits < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[2])
|
||||
}
|
||||
|
||||
r.uniques, err = strconv.Atoi(fields[3])
|
||||
if err != nil || r.uniques < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[3])
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// addResult to another one
|
||||
func addResult(r, other result) result {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
|
||||
// + give it the []result only.
|
||||
// + let it calculate the total.
|
||||
const (
|
||||
|
||||
// DOMAINS PAGES VISITS UNIQUES
|
||||
// ^ ^ ^ ^
|
||||
// | | | |
|
||||
header = "%-25s %-10s %10s %10s\n"
|
||||
line = "%-25s %-10s %10d %10d\n"
|
||||
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
|
||||
dash = "-"
|
||||
dashLength = 58
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(res []result) {
|
||||
// sort.Strings(p.domains)
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].domain <= res[j].domain
|
||||
})
|
||||
|
||||
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
|
||||
fmt.Println(strings.Repeat("-", dashLength))
|
||||
|
||||
var total result
|
||||
|
||||
for _, r := range res {
|
||||
total = addResult(total, r)
|
||||
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
|
||||
}
|
||||
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
### PROBLEM
|
||||
+ `parser.go/parse()` also does updating. back to square one.
|
||||
+ we need to extract the reusable behavior: scanning.
|
||||
|
||||
+ inflexible:
|
||||
+ adding a filter is hard. needs to change the `scan()` code.
|
||||
+ adding a grouper is also hard. domain grouping is hardcoded.
|
||||
|
||||
## SOLUTION
|
||||
+
|
||||
|
||||
## IDEAS:
|
||||
|
||||
+ make domain filter accept variadic args
|
||||
@ -1,36 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
/*
|
||||
p := pipeline{
|
||||
read: textReader(os.Stdin),
|
||||
write: textWriter(os.Stdout),
|
||||
filterBy: notUsing(domainExtFilter("io")),
|
||||
groupBy: domainGrouper,
|
||||
}
|
||||
|
||||
if err := start(p); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
}
|
||||
*/
|
||||
|
||||
func main() {
|
||||
p := newParser()
|
||||
|
||||
if err := parse(p); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
return
|
||||
}
|
||||
|
||||
summarize(p)
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
// parser keeps track of the parsing
|
||||
type parser struct {
|
||||
sum map[string]result // metrics per domain
|
||||
domains []string // unique domain names
|
||||
}
|
||||
|
||||
// newParser constructs, initializes and returns a new parser
|
||||
func newParser() *parser {
|
||||
return &parser{sum: make(map[string]result)}
|
||||
}
|
||||
|
||||
// parse all the log lines and update the results
|
||||
func parse(p *parser) error {
|
||||
process := func(r result) {
|
||||
update(p, r)
|
||||
}
|
||||
|
||||
err := scan(process)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func update(p *parser, r result) {
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[r.domain]; !ok {
|
||||
p.domains = append(p.domains, r.domain)
|
||||
}
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[r.domain] = addResult(r, p.sum[r.domain])
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain, page string
|
||||
visits, uniques int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parseResult from a log line
|
||||
func parseResult(line string) (r result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != fieldsLength {
|
||||
return r, fmt.Errorf("wrong input: %v", fields)
|
||||
}
|
||||
|
||||
r.domain = fields[0]
|
||||
r.page = fields[1]
|
||||
|
||||
r.visits, err = strconv.Atoi(fields[2])
|
||||
if err != nil || r.visits < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[2])
|
||||
}
|
||||
|
||||
r.uniques, err = strconv.Atoi(fields[3])
|
||||
if err != nil || r.uniques < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[3])
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// addResult to another one
|
||||
func addResult(r, other result) result {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type processFn func(r result)
|
||||
|
||||
func scan(process processFn) error {
|
||||
var (
|
||||
l = 1
|
||||
in = bufio.NewScanner(os.Stdin)
|
||||
)
|
||||
|
||||
for in.Scan() {
|
||||
r, err := parseResult(in.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("line %d: %v", l, err)
|
||||
}
|
||||
|
||||
l++
|
||||
|
||||
process(r)
|
||||
}
|
||||
return in.Err()
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
|
||||
// + give it the []result only.
|
||||
// + let it calculate the total.
|
||||
const (
|
||||
|
||||
// DOMAINS PAGES VISITS UNIQUES
|
||||
// ^ ^ ^ ^
|
||||
// | | | |
|
||||
header = "%-25s %-10s %10s %10s\n"
|
||||
line = "%-25s %-10s %10d %10d\n"
|
||||
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
|
||||
dash = "-"
|
||||
dashLength = 58
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(p *parser) {
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
|
||||
fmt.Println(strings.Repeat("-", dashLength))
|
||||
|
||||
var total result
|
||||
|
||||
for _, domain := range p.domains {
|
||||
r := p.sum[domain]
|
||||
total = addResult(total, r)
|
||||
|
||||
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
|
||||
}
|
||||
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
### PROBLEM
|
||||
+ ...
|
||||
|
||||
## SOLUTION
|
||||
+ `parser struct` -> `pipeline struct`
|
||||
+ `parse()` -> `pipe(pipeline)`
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
pipes := pipeline{
|
||||
// read: textParser(),
|
||||
// write: textSummary(),
|
||||
// filterBy: notUsing(domainExtFilter("io", "com")),
|
||||
// groupBy: domainGrouper,
|
||||
}
|
||||
|
||||
res, err := pipe(pipes)
|
||||
if err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
return
|
||||
}
|
||||
|
||||
summarize(res)
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// pipeline determines the behavior of log processing
|
||||
type pipeline struct {
|
||||
}
|
||||
|
||||
// pipe the log lines through funcs and produce a result
|
||||
func pipe(opts pipeline) ([]result, error) {
|
||||
var (
|
||||
l = 1
|
||||
in = bufio.NewScanner(os.Stdin)
|
||||
sum = make(map[string]result)
|
||||
)
|
||||
|
||||
// parse the log lines
|
||||
for in.Scan() {
|
||||
r, err := parseResult(in.Text())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: %v", l, err)
|
||||
}
|
||||
|
||||
l++
|
||||
|
||||
// group the log lines by domain
|
||||
sum[r.domain] = addResult(r, sum[r.domain])
|
||||
}
|
||||
|
||||
// collect the grouped results
|
||||
res := make([]result, 0, len(sum))
|
||||
for _, r := range sum {
|
||||
res = append(res, r)
|
||||
}
|
||||
|
||||
return res, in.Err()
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain, page string
|
||||
visits, uniques int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parseResult from a log line
|
||||
func parseResult(line string) (r result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != fieldsLength {
|
||||
return r, fmt.Errorf("wrong input: %v", fields)
|
||||
}
|
||||
|
||||
r.domain = fields[0]
|
||||
r.page = fields[1]
|
||||
|
||||
r.visits, err = strconv.Atoi(fields[2])
|
||||
if err != nil || r.visits < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[2])
|
||||
}
|
||||
|
||||
r.uniques, err = strconv.Atoi(fields[3])
|
||||
if err != nil || r.uniques < 0 {
|
||||
return r, fmt.Errorf("wrong input: %q", fields[3])
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// addResult to another one
|
||||
func addResult(r, other result) result {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
// + violation: accesses the parsing internals: p.domains + p.sum + p.total
|
||||
// + give it the []result only.
|
||||
// + let it calculate the total.
|
||||
const (
|
||||
|
||||
// DOMAINS PAGES VISITS UNIQUES
|
||||
// ^ ^ ^ ^
|
||||
// | | | |
|
||||
header = "%-25s %-10s %10s %10s\n"
|
||||
line = "%-25s %-10s %10d %10d\n"
|
||||
footer = "\n%-36s %10d %10d\n" // -> "" VISITS UNIQUES
|
||||
dash = "-"
|
||||
dashLength = 58
|
||||
)
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(res []result) {
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].domain <= res[j].domain
|
||||
})
|
||||
|
||||
fmt.Printf(header, "DOMAIN", "PAGES", "VISITS", "UNIQUES")
|
||||
fmt.Println(strings.Repeat("-", dashLength))
|
||||
|
||||
var total result
|
||||
|
||||
for _, r := range res {
|
||||
total = addResult(total, r)
|
||||
fmt.Printf(line, r.domain, r.page, r.visits, r.uniques)
|
||||
}
|
||||
fmt.Printf(footer, "TOTAL", total.visits, total.uniques)
|
||||
}
|
||||
38
logparser/oop/chartreport.go
Normal file
38
logparser/oop/chartreport.go
Normal file
@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
// You need to run:
|
||||
// go get -u github.com/wcharczuk/go-chart
|
||||
|
||||
// type chartReport struct {
|
||||
// title string
|
||||
// width, height int
|
||||
// }
|
||||
|
||||
// func (s *chartReport) digest(records iterator) error {
|
||||
// w := os.Stdout
|
||||
|
||||
// donut := chart.DonutChart{
|
||||
// Title: s.title,
|
||||
// TitleStyle: chart.Style{
|
||||
// FontSize: 35,
|
||||
// Show: true,
|
||||
// FontColor: chart.ColorAlternateGreen,
|
||||
// },
|
||||
// Width: s.width,
|
||||
// Height: s.height,
|
||||
// }
|
||||
|
||||
// records.each(func(r record) {
|
||||
// v := chart.Value{
|
||||
// Label: r.domain + r.page + ": " + strconv.Itoa(r.visits),
|
||||
// Value: float64(r.visits),
|
||||
// Style: chart.Style{
|
||||
// FontSize: 14,
|
||||
// },
|
||||
// }
|
||||
|
||||
// donut.Values = append(donut.Values, v)
|
||||
// })
|
||||
|
||||
// return donut.Render(chart.SVG, w)
|
||||
// }
|
||||
43
logparser/oop/filter.go
Normal file
43
logparser/oop/filter.go
Normal file
@ -0,0 +1,43 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
type filter struct {
|
||||
src iterator
|
||||
filters []filterFunc
|
||||
}
|
||||
|
||||
func filterBy(fn ...filterFunc) *filter {
|
||||
return &filter{filters: fn}
|
||||
}
|
||||
|
||||
// transform the record
|
||||
func (f *filter) digest(records iterator) error {
|
||||
f.src = records
|
||||
return nil
|
||||
}
|
||||
|
||||
// each yields only the filtered records
|
||||
func (f *filter) each(yield recordFn) error {
|
||||
return f.src.each(func(r record) {
|
||||
if !f.check(r) {
|
||||
return
|
||||
}
|
||||
yield(r)
|
||||
})
|
||||
}
|
||||
|
||||
// check all the filters against the record
|
||||
func (f *filter) check(r record) bool {
|
||||
for _, fi := range f.filters {
|
||||
if !fi(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
36
logparser/oop/filters.go
Normal file
36
logparser/oop/filters.go
Normal file
@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import "strings"
|
||||
|
||||
type filterFunc func(record) bool
|
||||
|
||||
func noopFilter(r record) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func notUsing(filter filterFunc) filterFunc {
|
||||
return func(r record) bool {
|
||||
return !filter(r)
|
||||
}
|
||||
}
|
||||
|
||||
func domainExtFilter(domains ...string) filterFunc {
|
||||
return func(r record) bool {
|
||||
for _, domain := range domains {
|
||||
if strings.HasSuffix(r.domain, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func domainFilter(domain string) filterFunc {
|
||||
return func(r record) bool {
|
||||
return strings.Contains(r.domain, domain)
|
||||
}
|
||||
}
|
||||
|
||||
func orgDomainsFilter(r record) bool {
|
||||
return strings.HasSuffix(r.domain, ".org")
|
||||
}
|
||||
49
logparser/oop/group.go
Normal file
49
logparser/oop/group.go
Normal file
@ -0,0 +1,49 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
type group struct {
|
||||
sum map[string]record // metrics per group key
|
||||
keys []string // unique group keys
|
||||
key groupFunc
|
||||
}
|
||||
|
||||
func groupBy(key groupFunc) *group {
|
||||
return &group{
|
||||
sum: make(map[string]record),
|
||||
key: key,
|
||||
}
|
||||
}
|
||||
|
||||
// digest the records
|
||||
func (g *group) digest(records iterator) error {
|
||||
return records.each(func(r record) {
|
||||
k := g.key(r)
|
||||
|
||||
if _, ok := g.sum[k]; !ok {
|
||||
g.keys = append(g.keys, k)
|
||||
}
|
||||
|
||||
g.sum[k] = r.sum(g.sum[k])
|
||||
})
|
||||
}
|
||||
|
||||
// each yields the grouped records
|
||||
func (g *group) each(yield recordFn) error {
|
||||
sort.Strings(g.keys)
|
||||
|
||||
for _, k := range g.keys {
|
||||
yield(g.sum[k])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
15
logparser/oop/groupers.go
Normal file
15
logparser/oop/groupers.go
Normal file
@ -0,0 +1,15 @@
|
||||
package main
|
||||
|
||||
type groupFunc func(record) string
|
||||
|
||||
// domainGrouper groups by domain.
|
||||
// but it keeps the other fields.
|
||||
// for example: it returns pages as well, but you shouldn't use them.
|
||||
// exercise: write a function that erases the unnecessary data.
|
||||
func domainGrouper(r record) string {
|
||||
return r.domain
|
||||
}
|
||||
|
||||
func pageGrouper(r record) string {
|
||||
return r.domain + r.page
|
||||
}
|
||||
43
logparser/oop/jsonlog.go
Normal file
43
logparser/oop/jsonlog.go
Normal file
@ -0,0 +1,43 @@
|
||||
// For more tutorials: https://bj.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
)
|
||||
|
||||
type jsonLog struct {
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func newJSONLog(r io.Reader) *jsonLog {
|
||||
return &jsonLog{reader: r}
|
||||
}
|
||||
|
||||
func (j *jsonLog) each(yield recordFn) error {
|
||||
defer readClose(j.reader)
|
||||
|
||||
dec := json.NewDecoder(bufio.NewReader(j.reader))
|
||||
|
||||
for {
|
||||
var r record
|
||||
|
||||
err := dec.Decode(&r)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
yield(r)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
33
logparser/oop/logcount.go
Normal file
33
logparser/oop/logcount.go
Normal file
@ -0,0 +1,33 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
// logCount counts the yielded records
|
||||
type logCount struct {
|
||||
iterator
|
||||
n int
|
||||
}
|
||||
|
||||
func (lc *logCount) each(yield recordFn) error {
|
||||
err := lc.iterator.each(func(r record) {
|
||||
lc.n++
|
||||
yield(r)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// lc.n+1: iterator.each won't call yield on err
|
||||
return fmt.Errorf("record %d: %v", lc.n+1, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lc *logCount) count() int {
|
||||
return lc.n
|
||||
}
|
||||
44
logparser/oop/main.go
Normal file
44
logparser/oop/main.go
Normal file
@ -0,0 +1,44 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// newGrouper(domainGrouper)
|
||||
|
||||
// s := &chartReport{
|
||||
// title: "visits per domain",
|
||||
// width: 1920,
|
||||
// height: 800,
|
||||
// }
|
||||
|
||||
// pipe, err := fromFile("../logs/log.jsonl")
|
||||
// if err != nil {
|
||||
// log.Fatalln(err)
|
||||
// }
|
||||
|
||||
pipe := newPipeline(
|
||||
newTextLog(os.Stdin),
|
||||
// newJSONLog(os.Stdin),
|
||||
newTextReport(),
|
||||
filterBy(notUsing(domainExtFilter("com", "io"))),
|
||||
groupBy(domainGrouper),
|
||||
)
|
||||
|
||||
if err := pipe.run(); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// if err := reportFromFile(os.Args[1]); err != nil {
|
||||
// log.Fatalln(err)
|
||||
// }
|
||||
}
|
||||
78
logparser/oop/pipeline.go
Normal file
78
logparser/oop/pipeline.go
Normal file
@ -0,0 +1,78 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type recordFn func(record)
|
||||
|
||||
type iterator interface{ each(recordFn) error }
|
||||
type digester interface{ digest(iterator) error }
|
||||
|
||||
type transform interface {
|
||||
digester
|
||||
iterator
|
||||
}
|
||||
|
||||
type pipeline struct {
|
||||
src iterator
|
||||
trans []transform
|
||||
dst digester
|
||||
}
|
||||
|
||||
func (p *pipeline) run() error {
|
||||
defer func() {
|
||||
n := p.src.(*logCount).count()
|
||||
fmt.Printf("%d records processed.\n", n)
|
||||
}()
|
||||
|
||||
last := p.src
|
||||
|
||||
for _, t := range p.trans {
|
||||
if err := t.digest(last); err != nil {
|
||||
return err
|
||||
}
|
||||
last = t
|
||||
}
|
||||
|
||||
return p.dst.digest(last)
|
||||
}
|
||||
|
||||
func newPipeline(src iterator, dst digester, t ...transform) *pipeline {
|
||||
return &pipeline{
|
||||
src: &logCount{iterator: src},
|
||||
dst: dst,
|
||||
trans: t,
|
||||
}
|
||||
}
|
||||
|
||||
// fromFile generates a default report
|
||||
func fromFile(path string) (*pipeline, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var src iterator
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".txt"):
|
||||
src = newTextLog(f)
|
||||
case strings.HasSuffix(path, ".jsonl"):
|
||||
src = newJSONLog(f)
|
||||
}
|
||||
|
||||
return newPipeline(
|
||||
src,
|
||||
newTextReport(),
|
||||
groupBy(domainGrouper),
|
||||
), nil
|
||||
}
|
||||
@ -8,15 +8,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
func main() {
|
||||
res, err := parse()
|
||||
if err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
return
|
||||
func readClose(r io.Reader) {
|
||||
if rc, ok := r.(io.Closer); ok {
|
||||
rc.Close()
|
||||
}
|
||||
|
||||
summarize(res)
|
||||
}
|
||||
82
logparser/oop/record.go
Normal file
82
logparser/oop/record.go
Normal file
@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const fieldsLength = 4
|
||||
|
||||
type record struct {
|
||||
domain string
|
||||
page string
|
||||
visits int
|
||||
uniques int
|
||||
}
|
||||
|
||||
func (r record) sum(other record) record {
|
||||
r.visits += other.visits
|
||||
r.uniques += other.uniques
|
||||
return r
|
||||
}
|
||||
|
||||
// UnmarshalText to a *record
|
||||
func (r *record) UnmarshalText(p []byte) (err error) {
|
||||
fields := strings.Fields(string(p))
|
||||
if len(fields) != fieldsLength {
|
||||
return fmt.Errorf("wrong number of fields %q", fields)
|
||||
}
|
||||
|
||||
r.domain, r.page = fields[0], fields[1]
|
||||
|
||||
if r.visits, err = parseStr("visits", fields[2]); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.uniques, err = parseStr("uniques", fields[3]); err != nil {
|
||||
return err
|
||||
}
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// UnmarshalJSON to a *record
|
||||
func (r *record) UnmarshalJSON(data []byte) error {
|
||||
var re struct {
|
||||
Domain string
|
||||
Page string
|
||||
Visits int
|
||||
Uniques int
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &re); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*r = record{re.Domain, re.Page, re.Visits, re.Uniques}
|
||||
return validate(*r)
|
||||
}
|
||||
|
||||
// parseStr helps UnmarshalText for string to positive int parsing
|
||||
func parseStr(name, v string) (int, error) {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("record.UnmarshalText %q: %v", name, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func validate(r record) (err error) {
|
||||
switch {
|
||||
case r.domain == "":
|
||||
err = errors.New("record.domain cannot be empty")
|
||||
case r.page == "":
|
||||
err = errors.New("record.page cannot be empty")
|
||||
case r.visits < 0:
|
||||
err = errors.New("record.visits cannot be negative")
|
||||
case r.uniques < 0:
|
||||
err = errors.New("record.uniques cannot be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
39
logparser/oop/textlog.go
Normal file
39
logparser/oop/textlog.go
Normal file
@ -0,0 +1,39 @@
|
||||
// For more tutorials: https://bp.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
type textLog struct {
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func newTextLog(r io.Reader) *textLog {
|
||||
return &textLog{reader: r}
|
||||
}
|
||||
|
||||
func (p *textLog) each(yield recordFn) error {
|
||||
defer readClose(p.reader)
|
||||
|
||||
in := bufio.NewScanner(p.reader)
|
||||
|
||||
for in.Scan() {
|
||||
r := new(record)
|
||||
|
||||
if err := r.UnmarshalText(in.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
yield(*r)
|
||||
}
|
||||
|
||||
return in.Err()
|
||||
}
|
||||
49
logparser/oop/textreport.go
Normal file
49
logparser/oop/textreport.go
Normal file
@ -0,0 +1,49 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
)
|
||||
|
||||
// TODO: make this configurable? or exercise?
|
||||
const (
|
||||
minWidth = 0
|
||||
tabWidth = 4
|
||||
padding = 4
|
||||
flags = 0
|
||||
)
|
||||
|
||||
type textReport struct{}
|
||||
|
||||
func newTextReport() *textReport {
|
||||
return new(textReport)
|
||||
}
|
||||
|
||||
func (s *textReport) digest(records iterator) error {
|
||||
w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, ' ', flags)
|
||||
|
||||
write := fmt.Fprintf
|
||||
|
||||
write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
|
||||
write(w, "-------\t-----\t------\t-------\n")
|
||||
|
||||
var total record
|
||||
records.each(func(r record) {
|
||||
total = total.sum(r)
|
||||
|
||||
write(w, "%s\t%s\t%d\t%d\n", r.domain, r.page, r.visits, r.uniques)
|
||||
})
|
||||
|
||||
write(w, "\t\t\t\n")
|
||||
write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", total.visits, total.uniques)
|
||||
|
||||
return w.Flush()
|
||||
}
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"bufio"
|
||||
"os"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
|
||||
"github.com/inancgumus/learngo/logparser/testing/report"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -4,7 +4,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
|
||||
"github.com/inancgumus/learngo/logparser/testing/report"
|
||||
)
|
||||
|
||||
func newParser(lines string) *report.Parser {
|
||||
@ -3,7 +3,7 @@ package report_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
|
||||
"github.com/inancgumus/learngo/logparser/testing/report"
|
||||
)
|
||||
|
||||
func TestSummaryTotal(t *testing.T) {
|
||||
@ -13,7 +13,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/testing/report"
|
||||
"github.com/inancgumus/learngo/logparser/testing/report"
|
||||
)
|
||||
|
||||
// summarize prints the parsing results.
|
||||
6
logparser/v1/log.txt
Normal file
6
logparser/v1/log.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org 4
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v1/log_err_missing.txt
Normal file
6
logparser/v1/log_err_missing.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v1/log_err_negative.txt
Normal file
6
logparser/v1/log_err_negative.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org -100
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v1/log_err_str.txt
Normal file
6
logparser/v1/log_err_str.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org FOUR
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
78
logparser/v1/main.go
Normal file
78
logparser/v1/main.go
Normal file
@ -0,0 +1,78 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
sum map[string]int // total visits per domain
|
||||
domains []string // unique domain names
|
||||
total int // total visits to all domains
|
||||
lines int // number of parsed lines (for the error messages)
|
||||
)
|
||||
|
||||
sum = make(map[string]int)
|
||||
|
||||
// Scan the standard-in line by line
|
||||
in := bufio.NewScanner(os.Stdin)
|
||||
for in.Scan() {
|
||||
lines++
|
||||
|
||||
// Parse the fields
|
||||
fields := strings.Fields(in.Text())
|
||||
if len(fields) != 2 {
|
||||
fmt.Printf("wrong input: %v (line #%d)\n", fields, lines)
|
||||
return
|
||||
}
|
||||
|
||||
domain := fields[0]
|
||||
|
||||
// Sum the total visits per domain
|
||||
visits, err := strconv.Atoi(fields[1])
|
||||
if visits < 0 || err != nil {
|
||||
fmt.Printf("wrong input: %q (line #%d)\n", fields[1], lines)
|
||||
return
|
||||
}
|
||||
|
||||
// Collect the unique domains
|
||||
if _, ok := sum[domain]; !ok {
|
||||
domains = append(domains, domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
total += visits
|
||||
sum[domain] += visits
|
||||
}
|
||||
|
||||
// Print the visits per domain
|
||||
sort.Strings(domains)
|
||||
|
||||
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
|
||||
fmt.Println(strings.Repeat("-", 45))
|
||||
|
||||
for _, domain := range domains {
|
||||
visits := sum[domain]
|
||||
fmt.Printf("%-30s %10d\n", domain, visits)
|
||||
}
|
||||
|
||||
// Print the total visits for all domains
|
||||
fmt.Printf("\n%-30s %10d\n", "TOTAL", total)
|
||||
|
||||
// Let's handle the error
|
||||
if err := in.Err(); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
}
|
||||
}
|
||||
6
logparser/v2/log.txt
Normal file
6
logparser/v2/log.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org 4
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v2/log_err_missing.txt
Normal file
6
logparser/v2/log_err_missing.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v2/log_err_negative.txt
Normal file
6
logparser/v2/log_err_negative.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org -100
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v2/log_err_str.txt
Normal file
6
logparser/v2/log_err_str.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org FOUR
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
94
logparser/v2/main.go
Normal file
94
logparser/v2/main.go
Normal file
@ -0,0 +1,94 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain string
|
||||
visits int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parser keep tracks of the parsing
|
||||
type parser struct {
|
||||
sum map[string]result // metrics per domain
|
||||
domains []string // unique domain names
|
||||
total int // total visits for all domains
|
||||
lines int // number of parsed lines (for the error messages)
|
||||
}
|
||||
|
||||
func main() {
|
||||
p := parser{sum: make(map[string]result)}
|
||||
|
||||
// Scan the standard-in line by line
|
||||
in := bufio.NewScanner(os.Stdin)
|
||||
for in.Scan() {
|
||||
p.lines++
|
||||
|
||||
// Parse the fields
|
||||
fields := strings.Fields(in.Text())
|
||||
if len(fields) != 2 {
|
||||
fmt.Printf("wrong input: %v (line #%d)\n", fields, p.lines)
|
||||
return
|
||||
}
|
||||
|
||||
domain := fields[0]
|
||||
|
||||
// Sum the total visits per domain
|
||||
visits, err := strconv.Atoi(fields[1])
|
||||
if visits < 0 || err != nil {
|
||||
fmt.Printf("wrong input: %q (line #%d)\n", fields[1], p.lines)
|
||||
return
|
||||
}
|
||||
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[domain]; !ok {
|
||||
p.domains = append(p.domains, domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
p.total += visits
|
||||
|
||||
// You cannot assign to composite values
|
||||
// p.sum[domain].visits += visits
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[domain] = result{
|
||||
domain: domain,
|
||||
visits: visits + p.sum[domain].visits,
|
||||
}
|
||||
}
|
||||
|
||||
// Print the visits per domain
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
|
||||
fmt.Println(strings.Repeat("-", 45))
|
||||
|
||||
for _, domain := range p.domains {
|
||||
parsed := p.sum[domain]
|
||||
fmt.Printf("%-30s %10d\n", domain, parsed.visits)
|
||||
}
|
||||
|
||||
// Print the total visits for all domains
|
||||
fmt.Printf("\n%-30s %10d\n", "TOTAL", p.total)
|
||||
|
||||
// Let's handle the error
|
||||
if err := in.Err(); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
}
|
||||
}
|
||||
6
logparser/v3/log.txt
Normal file
6
logparser/v3/log.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org 4
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v3/log_err_missing.txt
Normal file
6
logparser/v3/log_err_missing.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v3/log_err_negative.txt
Normal file
6
logparser/v3/log_err_negative.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org -100
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v3/log_err_str.txt
Normal file
6
logparser/v3/log_err_str.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org FOUR
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
53
logparser/v3/main.go
Normal file
53
logparser/v3/main.go
Normal file
@ -0,0 +1,53 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
p := newParser()
|
||||
|
||||
// Scan the standard-in line by line
|
||||
in := bufio.NewScanner(os.Stdin)
|
||||
for in.Scan() {
|
||||
p.lines++
|
||||
|
||||
parsed, err := parse(p, in.Text())
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
p = update(p, parsed)
|
||||
}
|
||||
|
||||
// Print the visits per domain
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
|
||||
fmt.Println(strings.Repeat("-", 45))
|
||||
|
||||
for _, domain := range p.domains {
|
||||
parsed := p.sum[domain]
|
||||
fmt.Printf("%-30s %10d\n", domain, parsed.visits)
|
||||
}
|
||||
|
||||
// Print the total visits for all domains
|
||||
fmt.Printf("\n%-30s %10d\n", "TOTAL", p.total)
|
||||
|
||||
// Let's handle the error
|
||||
if err := in.Err(); err != nil {
|
||||
fmt.Println("> Err:", err)
|
||||
}
|
||||
}
|
||||
74
logparser/v3/parser.go
Normal file
74
logparser/v3/parser.go
Normal file
@ -0,0 +1,74 @@
|
||||
// For more tutorials: https://blog.learngoprogramming.com
|
||||
//
|
||||
// Copyright © 2018 Inanc Gumus
|
||||
// Learn Go Programming Course
|
||||
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain string
|
||||
visits int
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parser keep tracks of the parsing
|
||||
type parser struct {
|
||||
sum map[string]result // metrics per domain
|
||||
domains []string // unique domain names
|
||||
total int // total visits for all domains
|
||||
lines int // number of parsed lines (for the error messages)
|
||||
}
|
||||
|
||||
// newParser constructs, initializes and returns a new parser
|
||||
func newParser() parser {
|
||||
return parser{sum: make(map[string]result)}
|
||||
}
|
||||
|
||||
// parse parses a log line and returns the parsed result with an error
|
||||
func parse(p parser, line string) (parsed result, err error) {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
err = fmt.Errorf("wrong input: %v (line #%d)", fields, p.lines)
|
||||
return
|
||||
}
|
||||
|
||||
parsed.domain = fields[0]
|
||||
|
||||
parsed.visits, err = strconv.Atoi(fields[1])
|
||||
if parsed.visits < 0 || err != nil {
|
||||
err = fmt.Errorf("wrong input: %q (line #%d)", fields[1], p.lines)
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// update updates the parser for the given parsing result
|
||||
func update(p parser, parsed result) parser {
|
||||
domain, visits := parsed.domain, parsed.visits
|
||||
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[domain]; !ok {
|
||||
p.domains = append(p.domains, domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
p.total += visits
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[domain] = result{
|
||||
domain: domain,
|
||||
visits: visits + p.sum[domain].visits,
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
6
logparser/v4/log.txt
Normal file
6
logparser/v4/log.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org 4
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v4/log_err_missing.txt
Normal file
6
logparser/v4/log_err_missing.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v4/log_err_negative.txt
Normal file
6
logparser/v4/log_err_negative.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org -100
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
6
logparser/v4/log_err_str.txt
Normal file
6
logparser/v4/log_err_str.txt
Normal file
@ -0,0 +1,6 @@
|
||||
learngoprogramming.com 10
|
||||
learngoprogramming.com 10
|
||||
golang.org FOUR
|
||||
golang.org 6
|
||||
blog.golang.org 20
|
||||
blog.golang.org 10
|
||||
@ -11,6 +11,8 @@ import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -26,6 +28,19 @@ func main() {
|
||||
dumpErrs([]error{in.Err(), err(p)})
|
||||
}
|
||||
|
||||
// summarize summarizes and prints the parsing result
|
||||
func summarize(p *parser) {
|
||||
sort.Strings(p.domains)
|
||||
|
||||
fmt.Printf("%-30s %10s\n", "DOMAIN", "VISITS")
|
||||
fmt.Println(strings.Repeat("-", 45))
|
||||
|
||||
for _, domain := range p.domains {
|
||||
fmt.Printf("%-30s %10d\n", domain, p.sum[domain].visits)
|
||||
}
|
||||
fmt.Printf("\n%-30s %10d\n", "TOTAL", p.total)
|
||||
}
|
||||
|
||||
// dumpErrs simplifies handling multiple errors
|
||||
func dumpErrs(errs []error) {
|
||||
for _, err := range errs {
|
||||
@ -13,11 +13,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// result stores details about a log line
|
||||
// result stores the parsed result for a domain
|
||||
type result struct {
|
||||
domain string
|
||||
visits int
|
||||
// add more metrics when needed
|
||||
// add more metrics if needed
|
||||
}
|
||||
|
||||
// parser keep tracks of the parsing
|
||||
@ -29,12 +29,12 @@ type parser struct {
|
||||
lerr error // the last error occurred
|
||||
}
|
||||
|
||||
// newParser creates and returns a new parser
|
||||
// newParser constructs, initializes and returns a new parser
|
||||
func newParser() *parser {
|
||||
return &parser{sum: make(map[string]result)}
|
||||
}
|
||||
|
||||
// update the parsing results
|
||||
// parse parses a log line and returns the parsed result with an error
|
||||
func parse(p *parser, line string) (r result) {
|
||||
if p.lerr != nil {
|
||||
return
|
||||
@ -59,6 +59,27 @@ func parse(p *parser, line string) (r result) {
|
||||
return
|
||||
}
|
||||
|
||||
// update updates all the parsing results using the given parsing result
|
||||
func update(p *parser, r result) {
|
||||
if p.lerr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Collect the unique domains
|
||||
if _, ok := p.sum[r.domain]; !ok {
|
||||
p.domains = append(p.domains, r.domain)
|
||||
}
|
||||
|
||||
// Keep track of total and per domain visits
|
||||
p.total += r.visits
|
||||
|
||||
// create and assign a new copy of `visit`
|
||||
p.sum[r.domain] = result{
|
||||
domain: r.domain,
|
||||
visits: r.visits + p.sum[r.domain].visits,
|
||||
}
|
||||
}
|
||||
|
||||
// err returns the last error encountered
|
||||
func err(p *parser) error {
|
||||
return p.lerr
|
||||
5
logparser/v5/Makefile
Normal file
5
logparser/v5/Makefile
Normal file
@ -0,0 +1,5 @@
|
||||
r:
|
||||
go run . < ../../logs/log.txt
|
||||
|
||||
t:
|
||||
time go run . < ../../logs/log.txt
|
||||
@ -11,10 +11,10 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/group"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/parse"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/report"
|
||||
)
|
||||
|
||||
// fromFile generates a default pipeline.
|
||||
@ -11,11 +11,11 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/filter"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse"
|
||||
"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/filter"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/group"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/parse"
|
||||
"github.com/inancgumus/learngo/logparser/v5/pipe/report"
|
||||
)
|
||||
|
||||
func main() {
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user