refactor: oop log parser to pkgs
This commit is contained in:
		| @@ -1,38 +0,0 @@ | ||||
| package main | ||||
|  | ||||
| // You need to run: | ||||
| // go get -u github.com/wcharczuk/go-chart | ||||
|  | ||||
| // type chartReport struct { | ||||
| // 	title         string | ||||
| // 	width, height int | ||||
| // } | ||||
|  | ||||
| // func (s *chartReport) digest(records iterator) error { | ||||
| // 	w := os.Stdout | ||||
|  | ||||
| // 	donut := chart.DonutChart{ | ||||
| // 		Title: s.title, | ||||
| // 		TitleStyle: chart.Style{ | ||||
| // 			FontSize:  35, | ||||
| // 			Show:      true, | ||||
| // 			FontColor: chart.ColorAlternateGreen, | ||||
| // 		}, | ||||
| // 		Width:  s.width, | ||||
| // 		Height: s.height, | ||||
| // 	} | ||||
|  | ||||
| // 	records.each(func(r record) { | ||||
| // 		v := chart.Value{ | ||||
| // 			Label: r.domain + r.page + ": " + strconv.Itoa(r.visits), | ||||
| // 			Value: float64(r.visits), | ||||
| // 			Style: chart.Style{ | ||||
| // 				FontSize: 14, | ||||
| // 			}, | ||||
| // 		} | ||||
|  | ||||
| // 		donut.Values = append(donut.Values, v) | ||||
| // 	}) | ||||
|  | ||||
| // 	return donut.Render(chart.SVG, w) | ||||
| // } | ||||
							
								
								
									
										42
									
								
								interfaces/log-parser/oop/filepipe.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										42
									
								
								interfaces/log-parser/oop/filepipe.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,42 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package main | ||||
|  | ||||
| import ( | ||||
| 	"os" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report" | ||||
| ) | ||||
|  | ||||
| // fromFile generates a default pipeline. | ||||
| // Detects the correct parser by the file extension. | ||||
| // Uses a TextReport and groups by domain. | ||||
| func fromFile(path string) (*pipe.Pipeline, error) { | ||||
| 	f, err := os.Open(path) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	var src pipe.Iterator | ||||
| 	switch { | ||||
| 	case strings.HasSuffix(path, ".txt"): | ||||
| 		src = parse.FromText(f) | ||||
| 	case strings.HasSuffix(path, ".jsonl"): | ||||
| 		src = parse.FromJSON(f) | ||||
| 	} | ||||
|  | ||||
| 	return pipe.New( | ||||
| 		src, | ||||
| 		report.AsText(os.Stdout), | ||||
| 		group.By(group.Domain), | ||||
| 	), nil | ||||
| } | ||||
| @@ -1,43 +0,0 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package main | ||||
|  | ||||
| type filter struct { | ||||
| 	src     iterator | ||||
| 	filters []filterFunc | ||||
| } | ||||
|  | ||||
| func filterBy(fn ...filterFunc) *filter { | ||||
| 	return &filter{filters: fn} | ||||
| } | ||||
|  | ||||
| // transform the record | ||||
| func (f *filter) digest(records iterator) error { | ||||
| 	f.src = records | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // each yields only the filtered records | ||||
| func (f *filter) each(yield recordFn) error { | ||||
| 	return f.src.each(func(r record) { | ||||
| 		if !f.check(r) { | ||||
| 			return | ||||
| 		} | ||||
| 		yield(r) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| // check all the filters against the record | ||||
| func (f *filter) check(r record) bool { | ||||
| 	for _, fi := range f.filters { | ||||
| 		if !fi(r) { | ||||
| 			return false | ||||
| 		} | ||||
| 	} | ||||
| 	return true | ||||
| } | ||||
| @@ -1,36 +0,0 @@ | ||||
| package main | ||||
|  | ||||
| import "strings" | ||||
|  | ||||
| type filterFunc func(record) bool | ||||
|  | ||||
| func noopFilter(r record) bool { | ||||
| 	return true | ||||
| } | ||||
|  | ||||
| func notUsing(filter filterFunc) filterFunc { | ||||
| 	return func(r record) bool { | ||||
| 		return !filter(r) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func domainExtFilter(domains ...string) filterFunc { | ||||
| 	return func(r record) bool { | ||||
| 		for _, domain := range domains { | ||||
| 			if strings.HasSuffix(r.domain, "."+domain) { | ||||
| 				return true | ||||
| 			} | ||||
| 		} | ||||
| 		return false | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func domainFilter(domain string) filterFunc { | ||||
| 	return func(r record) bool { | ||||
| 		return strings.Contains(r.domain, domain) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func orgDomainsFilter(r record) bool { | ||||
| 	return strings.HasSuffix(r.domain, ".org") | ||||
| } | ||||
| @@ -1,49 +0,0 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package main | ||||
|  | ||||
| import ( | ||||
| 	"sort" | ||||
| ) | ||||
|  | ||||
| type group struct { | ||||
| 	sum  map[string]record // metrics per group key | ||||
| 	keys []string          // unique group keys | ||||
| 	key  groupFunc | ||||
| } | ||||
|  | ||||
| func groupBy(key groupFunc) *group { | ||||
| 	return &group{ | ||||
| 		sum: make(map[string]record), | ||||
| 		key: key, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // digest the records | ||||
| func (g *group) digest(records iterator) error { | ||||
| 	return records.each(func(r record) { | ||||
| 		k := g.key(r) | ||||
|  | ||||
| 		if _, ok := g.sum[k]; !ok { | ||||
| 			g.keys = append(g.keys, k) | ||||
| 		} | ||||
|  | ||||
| 		g.sum[k] = r.sum(g.sum[k]) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| // each yields the grouped records | ||||
| func (g *group) each(yield recordFn) error { | ||||
| 	sort.Strings(g.keys) | ||||
|  | ||||
| 	for _, k := range g.keys { | ||||
| 		yield(g.sum[k]) | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
| @@ -1,15 +0,0 @@ | ||||
| package main | ||||
|  | ||||
| type groupFunc func(record) string | ||||
|  | ||||
| // domainGrouper groups by domain. | ||||
| // but it keeps the other fields. | ||||
| // for example: it returns pages as well, but you shouldn't use them. | ||||
| // exercise: write a function that erases the unnecessary data. | ||||
| func domainGrouper(r record) string { | ||||
| 	return r.domain | ||||
| } | ||||
|  | ||||
| func pageGrouper(r record) string { | ||||
| 	return r.domain + r.page | ||||
| } | ||||
| @@ -10,35 +10,40 @@ package main | ||||
| import ( | ||||
| 	"log" | ||||
| 	"os" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/filter" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/group" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/parse" | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe/report" | ||||
| ) | ||||
|  | ||||
| func main() { | ||||
| 	// newGrouper(domainGrouper) | ||||
|  | ||||
| 	// s := &chartReport{ | ||||
| 	//  title:  "visits per domain", | ||||
| 	//  width:  1920, | ||||
| 	//  height: 800, | ||||
| 	// } | ||||
|  | ||||
| 	// pipe, err := fromFile("../logs/log.jsonl") | ||||
| 	// if err != nil { | ||||
| 	// 	log.Fatalln(err) | ||||
| 	// } | ||||
|  | ||||
| 	pipe := newPipeline( | ||||
| 		newTextLog(os.Stdin), | ||||
| 		// newJSONLog(os.Stdin), | ||||
| 		newTextReport(), | ||||
| 		filterBy(notUsing(domainExtFilter("com", "io"))), | ||||
| 		groupBy(domainGrouper), | ||||
| 	pipe := pipe.New( | ||||
| 		parse.FromText(os.Stdin), | ||||
| 		// parse.FromJSON(os.Stdin), | ||||
| 		report.AsText(os.Stdout), | ||||
| 		filter.By(filter.Not(filter.DomainExt("com", "io"))), | ||||
| 		group.By(group.Domain), | ||||
| 		new(logger), | ||||
| 	) | ||||
|  | ||||
| 	if err := pipe.run(); err != nil { | ||||
| 	if err := pipe.Run(); err != nil { | ||||
| 		log.Fatalln(err) | ||||
| 	} | ||||
|  | ||||
| 	// if err := reportFromFile(os.Args[1]); err != nil { | ||||
| 	// 	log.Fatalln(err) | ||||
| 	// } | ||||
| } | ||||
|  | ||||
| type logger struct { | ||||
| 	src pipe.Iterator | ||||
| } | ||||
|  | ||||
| func (l *logger) Digest(records pipe.Iterator) error { | ||||
| 	l.src = records | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func (l *logger) Each(yield func(pipe.Record)) error { | ||||
| 	return l.src.Each(func(r pipe.Record) { | ||||
| 		yield(r) | ||||
| 	}) | ||||
| } | ||||
|   | ||||
							
								
								
									
										38
									
								
								interfaces/log-parser/oop/pipe/filter/domain.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										38
									
								
								interfaces/log-parser/oop/pipe/filter/domain.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,38 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package filter | ||||
|  | ||||
| import ( | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
|  | ||||
| // DomainExt filters a set of domain extensions. | ||||
| func DomainExt(domains ...string) Func { | ||||
| 	return func(r pipe.Record) bool { | ||||
| 		for _, domain := range domains { | ||||
| 			if strings.HasSuffix(r.Str("domain"), "."+domain) { | ||||
| 				return true | ||||
| 			} | ||||
| 		} | ||||
| 		return false | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // Domain filters a domain if it contains the given text. | ||||
| func Domain(text string) Func { | ||||
| 	return func(r pipe.Record) bool { | ||||
| 		return strings.Contains(r.Str("domain"), text) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // DomainOrg filters only the ".org" domains. | ||||
| func DomainOrg(r pipe.Record) bool { | ||||
| 	return strings.HasSuffix(r.Str("domain"), ".org") | ||||
| } | ||||
							
								
								
									
										50
									
								
								interfaces/log-parser/oop/pipe/filter/filter.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										50
									
								
								interfaces/log-parser/oop/pipe/filter/filter.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,50 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package filter | ||||
|  | ||||
| import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
|  | ||||
| // Func represents a filtering pipeline func. | ||||
| type Func func(pipe.Record) (pass bool) | ||||
|  | ||||
| // Filter the records. | ||||
| type Filter struct { | ||||
| 	src     pipe.Iterator | ||||
| 	filters []Func | ||||
| } | ||||
|  | ||||
| // By returns a new filter pipeline. | ||||
| func By(fn ...Func) *Filter { | ||||
| 	return &Filter{filters: fn} | ||||
| } | ||||
|  | ||||
| // Digest saves the iterator for later processing. | ||||
| func (f *Filter) Digest(records pipe.Iterator) error { | ||||
| 	f.src = records | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // Each yields only the filtered records. | ||||
| func (f *Filter) Each(yield func(pipe.Record)) error { | ||||
| 	return f.src.Each(func(r pipe.Record) { | ||||
| 		if !f.check(r) { | ||||
| 			return | ||||
| 		} | ||||
| 		yield(r) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| // check all the filters against the record. | ||||
| func (f *Filter) check(r pipe.Record) bool { | ||||
| 	for _, fi := range f.filters { | ||||
| 		if !fi(r) { | ||||
| 			return false | ||||
| 		} | ||||
| 	} | ||||
| 	return true | ||||
| } | ||||
							
								
								
									
										15
									
								
								interfaces/log-parser/oop/pipe/filter/noop.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										15
									
								
								interfaces/log-parser/oop/pipe/filter/noop.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,15 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package filter | ||||
|  | ||||
| import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
|  | ||||
| // Noop filter that does nothing. | ||||
| func Noop(r pipe.Record) bool { | ||||
| 	return true | ||||
| } | ||||
							
								
								
									
										17
									
								
								interfaces/log-parser/oop/pipe/filter/not.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										17
									
								
								interfaces/log-parser/oop/pipe/filter/not.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,17 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package filter | ||||
|  | ||||
| import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
|  | ||||
| // Not reverses a filter. True becomes false, and vice versa. | ||||
| func Not(filter Func) Func { | ||||
| 	return func(r pipe.Record) bool { | ||||
| 		return !filter(r) | ||||
| 	} | ||||
| } | ||||
							
								
								
									
										18
									
								
								interfaces/log-parser/oop/pipe/group/domain.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								interfaces/log-parser/oop/pipe/group/domain.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,18 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package group | ||||
|  | ||||
| import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
|  | ||||
| // Domain groups the records by domain. | ||||
| // It keeps the other fields intact. | ||||
| // For example: It returns the page field as well. | ||||
| // Exercise: Write a solution that removes the unnecessary data. | ||||
| func Domain(r pipe.Record) string { | ||||
| 	return r.Str("domain") | ||||
| } | ||||
							
								
								
									
										60
									
								
								interfaces/log-parser/oop/pipe/group/group.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										60
									
								
								interfaces/log-parser/oop/pipe/group/group.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,60 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package group | ||||
|  | ||||
| import ( | ||||
| 	"sort" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
|  | ||||
| // Func represents a grouping func that returns a grouping key. | ||||
| type Func func(pipe.Record) (key string) | ||||
|  | ||||
| // Group records by a key. | ||||
| type Group struct { | ||||
| 	sum  map[string]pipe.Record // metrics per group key | ||||
| 	keys []string               // unique group keys | ||||
| 	key  Func | ||||
| } | ||||
|  | ||||
| // By returns a new Group. | ||||
| // It takes a group func that returns a group key. | ||||
| // The returned group will group the record using the key. | ||||
| func By(key Func) *Group { | ||||
| 	return &Group{ | ||||
| 		sum: make(map[string]pipe.Record), | ||||
| 		key: key, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // Digest records for grouping. | ||||
| func (g *Group) Digest(records pipe.Iterator) error { | ||||
| 	return records.Each(func(r pipe.Record) { | ||||
| 		k := g.key(r) | ||||
|  | ||||
| 		if _, ok := g.sum[k]; !ok { | ||||
| 			g.keys = append(g.keys, k) | ||||
| 		} | ||||
|  | ||||
| 		if r, ok := r.(pipe.Summer); ok { | ||||
| 			g.sum[k] = r.Sum(g.sum[k]) | ||||
| 		} | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| // Each sorts and yields the grouped records. | ||||
| func (g *Group) Each(yield func(pipe.Record)) error { | ||||
| 	sort.Strings(g.keys) | ||||
|  | ||||
| 	for _, k := range g.keys { | ||||
| 		yield(g.sum[k]) | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
							
								
								
									
										15
									
								
								interfaces/log-parser/oop/pipe/group/page.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										15
									
								
								interfaces/log-parser/oop/pipe/group/page.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,15 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package group | ||||
|  | ||||
| import "github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
|  | ||||
| // Page groups records by page. | ||||
| func Page(r pipe.Record) string { | ||||
| 	return r.Str("domain") + r.Str("page") | ||||
| } | ||||
| @@ -5,18 +5,20 @@ | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
| 
 | ||||
| package main | ||||
| package pipe | ||||
| 
 | ||||
| import "fmt" | ||||
| 
 | ||||
| // logCount counts the yielded records | ||||
| // logCount counts the yielded records. | ||||
| type logCount struct { | ||||
| 	iterator | ||||
| 	Iterator | ||||
| 	n int | ||||
| } | ||||
| 
 | ||||
| func (lc *logCount) each(yield recordFn) error { | ||||
| 	err := lc.iterator.each(func(r record) { | ||||
| // Each yields to the inner iterator while counting the records. | ||||
| // Reports the record number on an error. | ||||
| func (lc *logCount) Each(yield func(Record)) error { | ||||
| 	err := lc.Iterator.Each(func(r Record) { | ||||
| 		lc.n++ | ||||
| 		yield(r) | ||||
| 	}) | ||||
| @@ -28,6 +30,7 @@ func (lc *logCount) each(yield recordFn) error { | ||||
| 	return nil | ||||
| } | ||||
| 
 | ||||
| // count returns the last read record number. | ||||
| func (lc *logCount) count() int { | ||||
| 	return lc.n | ||||
| } | ||||
| @@ -5,12 +5,13 @@ | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
| 
 | ||||
| package main | ||||
| package parse | ||||
| 
 | ||||
| import ( | ||||
| 	"io" | ||||
| ) | ||||
| 
 | ||||
| // readClose the reader if it's a io.Closer. | ||||
| func readClose(r io.Reader) { | ||||
| 	if rc, ok := r.(io.Closer); ok { | ||||
| 		rc.Close() | ||||
| @@ -5,26 +5,30 @@ | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
| 
 | ||||
| package main | ||||
| package parse | ||||
| 
 | ||||
| import ( | ||||
| 	"bufio" | ||||
| 	"encoding/json" | ||||
| 	"io" | ||||
| 
 | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
| 
 | ||||
| type jsonLog struct { | ||||
| // JSON parses json records. | ||||
| type JSON struct { | ||||
| 	reader io.Reader | ||||
| } | ||||
| 
 | ||||
| func newJSONLog(r io.Reader) *jsonLog { | ||||
| 	return &jsonLog{reader: r} | ||||
| // FromJSON creates a json parser. | ||||
| func FromJSON(r io.Reader) *JSON { | ||||
| 	return &JSON{reader: r} | ||||
| } | ||||
| 
 | ||||
| func (j *jsonLog) each(yield recordFn) error { | ||||
| // Each yields records from a json reader. | ||||
| func (j *JSON) Each(yield func(pipe.Record)) error { | ||||
| 	defer readClose(j.reader) | ||||
| 
 | ||||
| 	dec := json.NewDecoder(bufio.NewReader(j.reader)) | ||||
| 	dec := json.NewDecoder(j.reader) | ||||
| 
 | ||||
| 	for { | ||||
| 		var r record | ||||
							
								
								
									
										116
									
								
								interfaces/log-parser/oop/pipe/parse/record.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										116
									
								
								interfaces/log-parser/oop/pipe/parse/record.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,116 @@ | ||||
| package parse | ||||
|  | ||||
| import ( | ||||
| 	"encoding/json" | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"strconv" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
|  | ||||
| const fieldsLength = 4 | ||||
|  | ||||
| // record stores fields of a log line. | ||||
| type record struct { | ||||
| 	Domain  string | ||||
| 	Page    string | ||||
| 	Visits  int | ||||
| 	Uniques int | ||||
| } | ||||
|  | ||||
| // Str gets a string field by name. | ||||
| func (r record) Str(field string) string { | ||||
| 	switch field { | ||||
| 	case "domain": | ||||
| 		return r.Domain | ||||
| 	case "page": | ||||
| 		return r.Page | ||||
| 	} | ||||
| 	panic(fieldErr(field)) | ||||
| } | ||||
|  | ||||
| // Int gets an integer field by name. | ||||
| func (r record) Int(field string) int { | ||||
| 	switch field { | ||||
| 	case "visits": | ||||
| 		return r.Visits | ||||
| 	case "uniques": | ||||
| 		return r.Uniques | ||||
| 	} | ||||
| 	panic(fieldErr(field)) | ||||
| } | ||||
|  | ||||
| // Sum the numeric fields with another record. | ||||
| func (r record) Sum(other pipe.Record) pipe.Record { | ||||
| 	if other == nil { | ||||
| 		return r | ||||
| 	} | ||||
| 	r.Visits += other.(record).Visits | ||||
| 	r.Uniques += other.(record).Uniques | ||||
| 	return r | ||||
| } | ||||
|  | ||||
| // UnmarshalText to a *record. | ||||
| func (r *record) UnmarshalText(p []byte) (err error) { | ||||
| 	fields := strings.Fields(string(p)) | ||||
| 	if len(fields) != fieldsLength { | ||||
| 		return fmt.Errorf("wrong number of fields %q", fields) | ||||
| 	} | ||||
|  | ||||
| 	r.Domain, r.Page = fields[0], fields[1] | ||||
|  | ||||
| 	if r.Visits, err = parseStr("visits", fields[2]); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	if r.Uniques, err = parseStr("uniques", fields[3]); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return validate(*r) | ||||
| } | ||||
|  | ||||
| // UnmarshalJSON to a *record. | ||||
| func (r *record) UnmarshalJSON(data []byte) error { | ||||
| 	// `methodless` doesn't have any methods including UnmarshalJSON. | ||||
| 	// This trick prevents the stack-overflow (infinite loop). | ||||
| 	type methodless record | ||||
|  | ||||
| 	var m methodless | ||||
| 	if err := json.Unmarshal(data, &m); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	// Cast back to the record and save. | ||||
| 	*r = record(m) | ||||
|  | ||||
| 	return validate(*r) | ||||
| } | ||||
|  | ||||
| // parseStr helps UnmarshalText for string to positive int parsing. | ||||
| func parseStr(name, v string) (int, error) { | ||||
| 	n, err := strconv.Atoi(v) | ||||
| 	if err != nil { | ||||
| 		return 0, fmt.Errorf("Record.UnmarshalText %q: %v", name, err) | ||||
| 	} | ||||
| 	return n, nil | ||||
| } | ||||
|  | ||||
| // validate whether a parsed record is valid or not. | ||||
| func validate(r record) (err error) { | ||||
| 	switch { | ||||
| 	case r.Domain == "": | ||||
| 		err = errors.New("record.domain cannot be empty") | ||||
| 	case r.Page == "": | ||||
| 		err = errors.New("record.page cannot be empty") | ||||
| 	case r.Visits < 0: | ||||
| 		err = errors.New("record.visits cannot be negative") | ||||
| 	case r.Uniques < 0: | ||||
| 		err = errors.New("record.uniques cannot be negative") | ||||
| 	} | ||||
| 	return | ||||
| } | ||||
|  | ||||
| func fieldErr(field string) error { | ||||
| 	return fmt.Errorf("record field: %q does not exist", field) | ||||
| } | ||||
| @@ -5,22 +5,27 @@ | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
| 
 | ||||
| package main | ||||
| package parse | ||||
| 
 | ||||
| import ( | ||||
| 	"bufio" | ||||
| 	"io" | ||||
| 
 | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
| 
 | ||||
| type textLog struct { | ||||
| // Text parses text based log lines. | ||||
| type Text struct { | ||||
| 	reader io.Reader | ||||
| } | ||||
| 
 | ||||
| func newTextLog(r io.Reader) *textLog { | ||||
| 	return &textLog{reader: r} | ||||
| // FromText creates a text parser. | ||||
| func FromText(r io.Reader) *Text { | ||||
| 	return &Text{reader: r} | ||||
| } | ||||
| 
 | ||||
| func (p *textLog) each(yield recordFn) error { | ||||
| // Each yields records from a text log. | ||||
| func (p *Text) Each(yield func(pipe.Record)) error { | ||||
| 	defer readClose(p.reader) | ||||
| 
 | ||||
| 	in := bufio.NewScanner(p.reader) | ||||
| @@ -32,7 +37,7 @@ func (p *textLog) each(yield recordFn) error { | ||||
| 			return err | ||||
| 		} | ||||
| 
 | ||||
| 		yield(*r) | ||||
| 		yield(r) | ||||
| 	} | ||||
| 
 | ||||
| 	return in.Err() | ||||
							
								
								
									
										26
									
								
								interfaces/log-parser/oop/pipe/pipe.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										26
									
								
								interfaces/log-parser/oop/pipe/pipe.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,26 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package pipe | ||||
|  | ||||
| // Iterator yields a record. | ||||
| type Iterator interface { | ||||
| 	Each(func(Record)) error | ||||
| } | ||||
|  | ||||
| // Digester represents a record consumer. | ||||
| type Digester interface { | ||||
| 	Digest(Iterator) error | ||||
| } | ||||
|  | ||||
| // Transform represents both a record consumer and producer. | ||||
| // It has an input and output. | ||||
| // It takes a single record and provides an iterator for all the records. | ||||
| type Transform interface { | ||||
| 	Digester // consumer | ||||
| 	Iterator // producer | ||||
| } | ||||
							
								
								
									
										48
									
								
								interfaces/log-parser/oop/pipe/pipeline.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										48
									
								
								interfaces/log-parser/oop/pipe/pipeline.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,48 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package pipe | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| ) | ||||
|  | ||||
| // Pipeline takes records from a source, transforms, and sends them to a destionation. | ||||
| type Pipeline struct { | ||||
| 	src   Iterator | ||||
| 	trans []Transform | ||||
| 	dst   Digester | ||||
| } | ||||
|  | ||||
| // New creates a new pipeline. | ||||
| func New(src Iterator, dst Digester, t ...Transform) *Pipeline { | ||||
| 	return &Pipeline{ | ||||
| 		src:   &logCount{Iterator: src}, | ||||
| 		dst:   dst, | ||||
| 		trans: t, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // Run the pipeline. | ||||
| func (p *Pipeline) Run() error { | ||||
| 	defer func() { | ||||
| 		n := p.src.(*logCount).count() | ||||
| 		fmt.Fprintf(os.Stderr, "%d records processed.\n", n) | ||||
| 	}() | ||||
|  | ||||
| 	last := p.src | ||||
|  | ||||
| 	for _, t := range p.trans { | ||||
| 		if err := t.Digest(last); err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		last = t | ||||
| 	} | ||||
|  | ||||
| 	return p.dst.Digest(last) | ||||
| } | ||||
							
								
								
									
										19
									
								
								interfaces/log-parser/oop/pipe/record.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										19
									
								
								interfaces/log-parser/oop/pipe/record.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,19 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package pipe | ||||
|  | ||||
| // Record provides a generic interface for any sort of records. | ||||
| type Record interface { | ||||
| 	Str(field string) string | ||||
| 	Int(field string) int | ||||
| } | ||||
|  | ||||
| // Summer provides a method for summing the numeric fields. | ||||
| type Summer interface { | ||||
| 	Sum(Record) Record | ||||
| } | ||||
							
								
								
									
										49
									
								
								interfaces/log-parser/oop/pipe/report/chart.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										49
									
								
								interfaces/log-parser/oop/pipe/report/chart.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,49 @@ | ||||
| package report | ||||
|  | ||||
| /* | ||||
| // You need to run: | ||||
| // go get -u github.com/wcharczuk/go-chart | ||||
|  | ||||
| // Chart renders a chart. | ||||
| type Chart struct { | ||||
| 	Title         string | ||||
| 	Width, Height int | ||||
|  | ||||
| 	w io.Writer | ||||
| } | ||||
|  | ||||
| // AsChart returns a Chart report generator. | ||||
| func AsChart(w io.Writer) *Chart { | ||||
| 	return &Chart{w: w} | ||||
| } | ||||
|  | ||||
| // Digest generates a chart report. | ||||
| func (c *Chart) Digest(records pipe.Iterator) error { | ||||
| 	w := os.Stdout | ||||
|  | ||||
| 	donut := chart.DonutChart{ | ||||
| 		Title: c.Title, | ||||
| 		TitleStyle: chart.Style{ | ||||
| 			FontSize:  35, | ||||
| 			Show:      true, | ||||
| 			FontColor: chart.ColorAlternateGreen, | ||||
| 		}, | ||||
| 		Width:  c.Width, | ||||
| 		Height: c.Height, | ||||
| 	} | ||||
|  | ||||
| 	records.Each(func(r pipe.Record) { | ||||
| 		v := chart.Value{ | ||||
| 			Label: r.Str("domain") + r.Str("page") + ": " + strconv.Itoa(r.Int("visits")), | ||||
| 			Value: float64(r.Int("visits")), | ||||
| 			Style: chart.Style{ | ||||
| 				FontSize: 14, | ||||
| 			}, | ||||
| 		} | ||||
|  | ||||
| 		donut.Values = append(donut.Values, v) | ||||
| 	}) | ||||
|  | ||||
| 	return donut.Render(chart.SVG, w) | ||||
| } | ||||
| */ | ||||
							
								
								
									
										64
									
								
								interfaces/log-parser/oop/pipe/report/text.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										64
									
								
								interfaces/log-parser/oop/pipe/report/text.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,64 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package report | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"text/tabwriter" | ||||
|  | ||||
| 	"github.com/inancgumus/learngo/interfaces/log-parser/oop/pipe" | ||||
| ) | ||||
|  | ||||
| const ( | ||||
| 	minWidth = 0 | ||||
| 	tabWidth = 4 | ||||
| 	padding  = 4 | ||||
| 	flags    = 0 | ||||
| ) | ||||
|  | ||||
| // Text report generator. | ||||
| type Text struct { | ||||
| 	w io.Writer | ||||
| } | ||||
|  | ||||
| // AsText returns a Text report generator. | ||||
| func AsText(w io.Writer) *Text { | ||||
| 	return &Text{w: w} | ||||
| } | ||||
|  | ||||
| // Digest generates a text report. | ||||
| func (t *Text) Digest(records pipe.Iterator) error { | ||||
| 	w := tabwriter.NewWriter(t.w, minWidth, tabWidth, padding, ' ', flags) | ||||
|  | ||||
| 	write := fmt.Fprintf | ||||
|  | ||||
| 	write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") | ||||
| 	write(w, "-------\t-----\t------\t-------\n") | ||||
|  | ||||
| 	var total pipe.Record | ||||
|  | ||||
| 	records.Each(func(r pipe.Record) { | ||||
| 		if r, ok := r.(pipe.Summer); ok { | ||||
| 			total = r.Sum(total) | ||||
| 		} | ||||
|  | ||||
| 		write(w, "%s\t%s\t%d\t%d\n", | ||||
| 			r.Str("domain"), r.Str("page"), | ||||
| 			r.Int("visits"), r.Int("uniques"), | ||||
| 		) | ||||
| 	}) | ||||
|  | ||||
| 	write(w, "\t\t\t\n") | ||||
| 	write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", | ||||
| 		total.Int("visits"), | ||||
| 		total.Int("uniques"), | ||||
| 	) | ||||
|  | ||||
| 	return w.Flush() | ||||
| } | ||||
| @@ -1,78 +0,0 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package main | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"strings" | ||||
| ) | ||||
|  | ||||
| type recordFn func(record) | ||||
|  | ||||
| type iterator interface{ each(recordFn) error } | ||||
| type digester interface{ digest(iterator) error } | ||||
|  | ||||
| type transform interface { | ||||
| 	digester | ||||
| 	iterator | ||||
| } | ||||
|  | ||||
| type pipeline struct { | ||||
| 	src   iterator | ||||
| 	trans []transform | ||||
| 	dst   digester | ||||
| } | ||||
|  | ||||
| func (p *pipeline) run() error { | ||||
| 	defer func() { | ||||
| 		n := p.src.(*logCount).count() | ||||
| 		fmt.Printf("%d records processed.\n", n) | ||||
| 	}() | ||||
|  | ||||
| 	last := p.src | ||||
|  | ||||
| 	for _, t := range p.trans { | ||||
| 		if err := t.digest(last); err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		last = t | ||||
| 	} | ||||
|  | ||||
| 	return p.dst.digest(last) | ||||
| } | ||||
|  | ||||
| func newPipeline(src iterator, dst digester, t ...transform) *pipeline { | ||||
| 	return &pipeline{ | ||||
| 		src:   &logCount{iterator: src}, | ||||
| 		dst:   dst, | ||||
| 		trans: t, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // fromFile generates a default report | ||||
| func fromFile(path string) (*pipeline, error) { | ||||
| 	f, err := os.Open(path) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	var src iterator | ||||
| 	switch { | ||||
| 	case strings.HasSuffix(path, ".txt"): | ||||
| 		src = newTextLog(f) | ||||
| 	case strings.HasSuffix(path, ".jsonl"): | ||||
| 		src = newJSONLog(f) | ||||
| 	} | ||||
|  | ||||
| 	return newPipeline( | ||||
| 		src, | ||||
| 		newTextReport(), | ||||
| 		groupBy(domainGrouper), | ||||
| 	), nil | ||||
| } | ||||
| @@ -1,82 +0,0 @@ | ||||
| package main | ||||
|  | ||||
| import ( | ||||
| 	"encoding/json" | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"strconv" | ||||
| 	"strings" | ||||
| ) | ||||
|  | ||||
| const fieldsLength = 4 | ||||
|  | ||||
| type record struct { | ||||
| 	domain  string | ||||
| 	page    string | ||||
| 	visits  int | ||||
| 	uniques int | ||||
| } | ||||
|  | ||||
| func (r record) sum(other record) record { | ||||
| 	r.visits += other.visits | ||||
| 	r.uniques += other.uniques | ||||
| 	return r | ||||
| } | ||||
|  | ||||
| // UnmarshalText to a *record | ||||
| func (r *record) UnmarshalText(p []byte) (err error) { | ||||
| 	fields := strings.Fields(string(p)) | ||||
| 	if len(fields) != fieldsLength { | ||||
| 		return fmt.Errorf("wrong number of fields %q", fields) | ||||
| 	} | ||||
|  | ||||
| 	r.domain, r.page = fields[0], fields[1] | ||||
|  | ||||
| 	if r.visits, err = parseStr("visits", fields[2]); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	if r.uniques, err = parseStr("uniques", fields[3]); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return validate(*r) | ||||
| } | ||||
|  | ||||
| // UnmarshalJSON to a *record | ||||
| func (r *record) UnmarshalJSON(data []byte) error { | ||||
| 	var re struct { | ||||
| 		Domain  string | ||||
| 		Page    string | ||||
| 		Visits  int | ||||
| 		Uniques int | ||||
| 	} | ||||
|  | ||||
| 	if err := json.Unmarshal(data, &re); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	*r = record{re.Domain, re.Page, re.Visits, re.Uniques} | ||||
| 	return validate(*r) | ||||
| } | ||||
|  | ||||
| // parseStr helps UnmarshalText for string to positive int parsing | ||||
| func parseStr(name, v string) (int, error) { | ||||
| 	n, err := strconv.Atoi(v) | ||||
| 	if err != nil { | ||||
| 		return 0, fmt.Errorf("record.UnmarshalText %q: %v", name, err) | ||||
| 	} | ||||
| 	return n, nil | ||||
| } | ||||
|  | ||||
| func validate(r record) (err error) { | ||||
| 	switch { | ||||
| 	case r.domain == "": | ||||
| 		err = errors.New("record.domain cannot be empty") | ||||
| 	case r.page == "": | ||||
| 		err = errors.New("record.page cannot be empty") | ||||
| 	case r.visits < 0: | ||||
| 		err = errors.New("record.visits cannot be negative") | ||||
| 	case r.uniques < 0: | ||||
| 		err = errors.New("record.uniques cannot be negative") | ||||
| 	} | ||||
| 	return | ||||
| } | ||||
| @@ -1,49 +0,0 @@ | ||||
| // For more tutorials: https://blog.learngoprogramming.com | ||||
| // | ||||
| // Copyright © 2018 Inanc Gumus | ||||
| // Learn Go Programming Course | ||||
| // License: https://creativecommons.org/licenses/by-nc-sa/4.0/ | ||||
| // | ||||
|  | ||||
| package main | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"text/tabwriter" | ||||
| ) | ||||
|  | ||||
| // TODO: make this configurable? or exercise? | ||||
| const ( | ||||
| 	minWidth = 0 | ||||
| 	tabWidth = 4 | ||||
| 	padding  = 4 | ||||
| 	flags    = 0 | ||||
| ) | ||||
|  | ||||
| type textReport struct{} | ||||
|  | ||||
| func newTextReport() *textReport { | ||||
| 	return new(textReport) | ||||
| } | ||||
|  | ||||
| func (s *textReport) digest(records iterator) error { | ||||
| 	w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, ' ', flags) | ||||
|  | ||||
| 	write := fmt.Fprintf | ||||
|  | ||||
| 	write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") | ||||
| 	write(w, "-------\t-----\t------\t-------\n") | ||||
|  | ||||
| 	var total record | ||||
| 	records.each(func(r record) { | ||||
| 		total = total.sum(r) | ||||
|  | ||||
| 		write(w, "%s\t%s\t%d\t%d\n", r.domain, r.page, r.visits, r.uniques) | ||||
| 	}) | ||||
|  | ||||
| 	write(w, "\t\t\t\n") | ||||
| 	write(w, "%s\t%s\t%d\t%d\n", "TOTAL", "", total.visits, total.uniques) | ||||
|  | ||||
| 	return w.Flush() | ||||
| } | ||||
		Reference in New Issue
	
	Block a user