2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Go port of Coda Hale's Metrics library
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-07 02:54:22 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// <https://github.com/rcrowley/go-metrics>
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-07 02:54:22 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Coda Hale's original work: <https://github.com/codahale/metrics>
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								package metrics
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import (
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"os"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									"runtime"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"strings"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									"time"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-22 14:10:07 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"github.com/ethereum/go-ethereum/log"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Enabled is checked by the constructor functions for all of the
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// standard metrics. If it is true, the metric returned is a stub.
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// This global kill-switch helps quantify the observer effect and makes
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// for less cluttered pprof profiles.
							 | 
						
					
						
							
								
									
										
										
										
											2019-02-18 03:37:31 -08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								var Enabled = false
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// EnabledExpensive is a soft-flag meant for external packages to check if costly
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// metrics gathering is allowed or not. The goal is to separate standard metrics
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// for health monitoring and debug metrics that might impact runtime performance.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var EnabledExpensive = false
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// enablerFlags is the CLI flag names to use to enable metrics collections.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var enablerFlags = []string{"metrics", "dashboard"}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var expensiveEnablerFlags = []string{"metrics.expensive"}
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// Init enables or disables the metrics system. Since we need this to run before
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// any other code gets to create meters and timers, we'll actually do an ugly hack
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// and peek into the command line args for the metrics flag.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								func init() {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for _, arg := range os.Args {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										flag := strings.TrimLeft(arg, "-")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										for _, enabler := range enablerFlags {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											if !Enabled && flag == enabler {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												log.Info("Enabling metrics collection")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												Enabled = true
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										for _, enabler := range expensiveEnablerFlags {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:40:46 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
											if !EnabledExpensive && flag == enabler {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
												log.Info("Enabling expensive metrics collection")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												EnabledExpensive = true
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											}
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// CollectProcessMetrics periodically collects various metrics about the running
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// process.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								func CollectProcessMetrics(refresh time.Duration) {
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									// Short circuit if the metrics system is disabled
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-02 14:13:46 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									if !Enabled {
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										return
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Create the various data collectors
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									memstats := make([]*runtime.MemStats, 2)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									diskstats := make([]*DiskStats, 2)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for i := 0; i < len(memstats); i++ {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memstats[i] = new(runtime.MemStats)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskstats[i] = new(DiskStats)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Define the various metrics to collect
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									var diskReadBytesCounter, diskWriteBytesCounter Counter
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									if err := ReadDiskStats(diskstats[0]); err == nil {
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									} else {
							 | 
						
					
						
							
								
									
										
										
										
											2017-03-03 11:41:52 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										log.Debug("Failed to read disk metrics", "err", err)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Iterate loading the different stats and updating the meters
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for i := 1; ; i++ {
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-12 14:02:36 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										location1 := i % 2
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										location2 := (i - 1) % 2
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-11 19:45:25 +08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										runtime.ReadMemStats(memstats[location1])
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										if ReadDiskStats(diskstats[location1]) == nil {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReadBytesCounter.Inc(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWriteBytesCounter.Inc(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										time.Sleep(refresh)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 |