2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Go port of Coda Hale's Metrics library
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-07 02:54:22 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// <https://github.com/rcrowley/go-metrics>
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-07 02:54:22 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Coda Hale's original work: <https://github.com/codahale/metrics>
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								package metrics
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import (
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"os"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									"runtime"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"strings"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									"time"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-22 14:10:07 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									"github.com/ethereum/go-ethereum/log"
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// Enabled is checked by the constructor functions for all of the
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// standard metrics. If it is true, the metric returned is a stub.
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								//
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// This global kill-switch helps quantify the observer effect and makes
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// for less cluttered pprof profiles.
							 | 
						
					
						
							
								
									
										
										
										
											2019-02-18 03:37:31 -08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								var Enabled = false
							 | 
						
					
						
							
								
									
										
										
										
											2018-02-23 10:56:08 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								// EnabledExpensive is a soft-flag meant for external packages to check if costly
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// metrics gathering is allowed or not. The goal is to separate standard metrics
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// for health monitoring and debug metrics that might impact runtime performance.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var EnabledExpensive = false
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// enablerFlags is the CLI flag names to use to enable metrics collections.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var enablerFlags = []string{"metrics", "dashboard"}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								var expensiveEnablerFlags = []string{"metrics.expensive"}
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// Init enables or disables the metrics system. Since we need this to run before
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// any other code gets to create meters and timers, we'll actually do an ugly hack
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// and peek into the command line args for the metrics flag.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								func init() {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for _, arg := range os.Args {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										flag := strings.TrimLeft(arg, "-")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										for _, enabler := range enablerFlags {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											if !Enabled && flag == enabler {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												log.Info("Enabling metrics collection")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												Enabled = true
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										for _, enabler := range expensiveEnablerFlags {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:40:46 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
											if !EnabledExpensive && flag == enabler {
							 | 
						
					
						
							
								
									
										
										
										
											2019-03-25 10:01:18 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
												log.Info("Enabling expensive metrics collection")
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
												EnabledExpensive = true
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											}
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// CollectProcessMetrics periodically collects various metrics about the running
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// process.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								func CollectProcessMetrics(refresh time.Duration) {
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									// Short circuit if the metrics system is disabled
							 | 
						
					
						
							
								
									
										
										
										
											2015-07-02 14:13:46 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									if !Enabled {
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-29 16:11:01 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										return
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									refreshFreq := int64(refresh / time.Second)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Create the various data collectors
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									cpuStats := make([]*CPUStats, 2)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									memstats := make([]*runtime.MemStats, 2)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									diskstats := make([]*DiskStats, 2)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for i := 0; i < len(memstats); i++ {
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										cpuStats[i] = new(CPUStats)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memstats[i] = new(runtime.MemStats)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskstats[i] = new(DiskStats)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Define the various metrics to collect
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-17 10:53:17 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									var (
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuSysLoad    = GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuSysWait    = GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuProcLoad   = GetOrRegisterGauge("system/cpu/procload", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuThreads    = GetOrRegisterGauge("system/cpu/threads", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuGoroutines = GetOrRegisterGauge("system/cpu/goroutines", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-17 10:53:17 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										memPauses = GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memAllocs = GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memFrees  = GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memHeld   = GetOrRegisterGauge("system/memory/held", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memUsed   = GetOrRegisterGauge("system/memory/used", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-17 10:53:17 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskReads             = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskReadBytes         = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskReadBytesCounter  = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskWrites            = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										diskWriteBytes        = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry)
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-17 10:53:17 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
									)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
									// Iterate loading the different stats and updating the meters
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									for i := 1; ; i++ {
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-12 14:02:36 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										location1 := i % 2
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										location2 := (i - 1) % 2
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										ReadCPUStats(cpuStats[location1])
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuSysLoad.Update((cpuStats[location1].GlobalTime - cpuStats[location2].GlobalTime) / refreshFreq)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuSysWait.Update((cpuStats[location1].GlobalWait - cpuStats[location2].GlobalWait) / refreshFreq)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuProcLoad.Update((cpuStats[location1].LocalTime - cpuStats[location2].LocalTime) / refreshFreq)
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-17 10:53:17 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										cpuThreads.Update(int64(threadCreateProfile.Count()))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										cpuGoroutines.Update(int64(runtime.NumGoroutine()))
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-11 19:45:25 +08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										runtime.ReadMemStats(memstats[location1])
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-11 19:45:25 +08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees))
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-10 14:21:02 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
										memHeld.Update(int64(memstats[location1].HeapSys - memstats[location1].HeapReleased))
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										memUsed.Update(int64(memstats[location1].Alloc))
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-11 19:45:25 +08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										if ReadDiskStats(diskstats[location1]) == nil {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
							 | 
						
					
						
							
								
									
										
										
										
											2018-07-09 14:11:49 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskReadBytesCounter.Inc(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
											diskWriteBytesCounter.Inc(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
							 | 
						
					
						
							
								
									
										
										
										
											2015-06-27 18:12:58 +03:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
										}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
										time.Sleep(refresh)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
									}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 |