fix: logparser v5 json err

This commit is contained in:
Inanc Gumus
2019-08-29 01:50:04 +03:00
parent 4910370808
commit d28c6d54d2
5 changed files with 36 additions and 24 deletions

View File

@ -23,10 +23,14 @@ func main() {
p := pipe.New( p := pipe.New(
pipe.NewTextLog(os.Stdin), pipe.NewTextLog(os.Stdin),
// pipe.NewJSONLog(os.Stdin),
pipe.NewTextReport(os.Stdout), pipe.NewTextReport(os.Stdout),
// pipe.NewJSONReport(os.Stdout), // pipe.NewJSONReport(os.Stdout),
pipe.FilterBy(pipe.DomainExtFilter("com", "io")),
pipe.GroupBy(pipe.DomainGrouper), // pipe.FilterBy(pipe.DomainExtFilter("com", "io")),
// pipe.GroupBy(pipe.DomainGrouper),
// new(passThrough), // new(passThrough),
) )

View File

@ -17,7 +17,7 @@ type GroupFunc = func(Record) (key string)
// Group records by a key. // Group records by a key.
type Group struct { type Group struct {
sum map[string]Record // metrics per group key sum map[string]record // metrics per group key
keys []string // unique group keys keys []string // unique group keys
key GroupFunc key GroupFunc
} }
@ -27,7 +27,7 @@ type Group struct {
// The returned group will group the record using the key. // The returned group will group the record using the key.
func GroupBy(key GroupFunc) *Group { func GroupBy(key GroupFunc) *Group {
return &Group{ return &Group{
sum: make(map[string]Record), sum: make(map[string]record),
key: key, key: key,
} }
} }
@ -41,7 +41,7 @@ func (g *Group) Consume(records Iterator) error {
g.keys = append(g.keys, k) g.keys = append(g.keys, k)
} }
g.sum[k] = r.Sum(g.sum[k]) g.sum[k] = r.sum(g.sum[k])
return nil return nil
}) })
@ -52,7 +52,7 @@ func (g *Group) Each(yield func(Record) error) error {
sort.Strings(g.keys) sort.Strings(g.keys)
for _, k := range g.keys { for _, k := range g.keys {
if err := yield(g.sum[k]); err != nil { if err := yield(Record{g.sum[k]}); err != nil {
return err return err
} }
} }

View File

@ -27,6 +27,6 @@ func (t *JSONReport) Consume(records Iterator) error {
enc := json.NewEncoder(t.w) enc := json.NewEncoder(t.w)
return records.Each(func(r Record) error { return records.Each(func(r Record) error {
return enc.Encode(r) return enc.Encode(&r)
}) })
} }

View File

@ -18,15 +18,23 @@ type record struct {
uniques int uniques int
} }
// Sum the numeric fields with another record. // recordJSON is used for marshaling and unmarshaling JSON.
func (r Record) Sum(other Record) Record { type recordJSON struct {
Domain string
Page string
Visits int
Uniques int
}
// sum the numeric fields with another record.
func (r record) sum(other record) record {
r.visits += other.visits r.visits += other.visits
r.uniques += other.uniques r.uniques += other.uniques
return r return r
} }
// UnmarshalText to a *Record. // UnmarshalText to a *record.
func (r *Record) UnmarshalText(p []byte) (err error) { func (r *record) UnmarshalText(p []byte) (err error) {
fields := strings.Fields(string(p)) fields := strings.Fields(string(p))
if len(fields) != fieldsLength { if len(fields) != fieldsLength {
return fmt.Errorf("wrong number of fields %q", fields) return fmt.Errorf("wrong number of fields %q", fields)
@ -43,23 +51,23 @@ func (r *Record) UnmarshalText(p []byte) (err error) {
return validate(*r) return validate(*r)
} }
// UnmarshalJSON to a *Record. // UnmarshalJSON to a *record.
func (r *Record) UnmarshalJSON(data []byte) error { func (r *record) UnmarshalJSON(data []byte) error {
// `methodless` doesn't have any methods including UnmarshalJSON. var rj recordJSON
// This trick prevents the stack-overflow (infinite loop).
type methodless Record
var m methodless if err := json.Unmarshal(data, &rj); err != nil {
if err := json.Unmarshal(data, &m); err != nil {
return err return err
} }
// Cast back to the Record and save. *r = record{rj.Domain, rj.Page, rj.Visits, rj.Uniques}
*r = Record(m)
return validate(*r) return validate(*r)
} }
// MarshalJSON of a *record.
func (r *record) MarshalJSON() ([]byte, error) {
return json.Marshal(recordJSON{r.domain, r.page, r.visits, r.uniques})
}
// parseStr helps UnmarshalText for string to positive int parsing. // parseStr helps UnmarshalText for string to positive int parsing.
func parseStr(name, v string) (int, error) { func parseStr(name, v string) (int, error) {
n, err := strconv.Atoi(v) n, err := strconv.Atoi(v)
@ -70,7 +78,7 @@ func parseStr(name, v string) (int, error) {
} }
// validate whether a parsed record is valid or not. // validate whether a parsed record is valid or not.
func validate(r Record) (err error) { func validate(r record) (err error) {
switch { switch {
case r.domain == "": case r.domain == "":
err = errors.New("record.domain cannot be empty") err = errors.New("record.domain cannot be empty")

View File

@ -39,10 +39,10 @@ func (t *TextReport) Consume(records Iterator) error {
write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n") write(w, "DOMAINS\tPAGES\tVISITS\tUNIQUES\n")
write(w, "-------\t-----\t------\t-------\n") write(w, "-------\t-----\t------\t-------\n")
var total Record var total record
err := records.Each(func(r Record) error { err := records.Each(func(r Record) error {
total = r.Sum(total) total = r.sum(total)
write(w, "%s\t%s\t%d\t%d\n", write(w, "%s\t%s\t%d\t%d\n",
r.domain, r.page, r.domain, r.page,