From f416a3756ecd7b606be1a6da279a7c0f12578075 Mon Sep 17 00:00:00 2001 From: Dan Ballard Date: Thu, 28 Jun 2018 12:36:50 -0700 Subject: [PATCH] ability for metrics to total or average; monitorHistory test will have to be integ test; uptime instead of starttime --- server/metrics/metrics.go | 48 ++++++++++++++++++++++------------ server/metrics/metrics_test.go | 39 --------------------------- server/metrics/monitors.go | 10 +++---- 3 files changed, 36 insertions(+), 61 deletions(-) diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index 779dcb8..abbcac4 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -61,8 +61,19 @@ const ( MegaBytes ) +// MonitorAccumulation controls how monitor data is accumulated over time into larger summary buckets +type MonitorAccumulation int + +const ( + // Cumulative values with sum over time + Cumulative MonitorAccumulation = iota + // Average values will average over time + Average +) + type monitorHistory struct { - monitorType MonitorType + monitorType MonitorType + monitorAccumulation MonitorAccumulation starttime time.Time perMinutePerHour [60]float64 @@ -96,8 +107,10 @@ type MonitorHistory interface { } // NewMonitorHistory returns a new MonitorHistory with starttime of time.Now and Started running with supplied monitor -func NewMonitorHistory(t MonitorType, monitor func() float64) MonitorHistory { - mh := &monitorHistory{monitorType: t, starttime: time.Now(), monitor: monitor, breakChannel: make(chan bool)} +func NewMonitorHistory(t MonitorType, a MonitorAccumulation, monitor func() float64) MonitorHistory { + mh := &monitorHistory{monitorType: t, monitorAccumulation: a, starttime: time.Now(), monitor: monitor, breakChannel: make(chan bool), + timeLastHourRotate: time.Now(), timeLastDayRotate: time.Now(), timeLastWeekRotate: time.Now(), + timeLastMonthRotate: time.Now()} mh.Start() return mh } @@ -148,7 +161,7 @@ func (mh *monitorHistory) Report(w *bufio.Writer) { func reportLine(t MonitorType, array []float64) string { switch t { case Count: - return strings.Trim(strings.Join(strings.Fields(fmt.Sprint(array)), " "), "[]") + return strings.Trim(strings.Join(strings.Fields(fmt.Sprintf("%.0f", array)), " "), "[]") case Percent: return strings.Trim(strings.Join(strings.Fields(fmt.Sprintf("%.2f", array)), " "), "[]") case MegaBytes: @@ -171,7 +184,7 @@ func (mh *monitorHistory) returnCopy(slice []float64) []float64 { return retSlice } -func rotateAndAvg(array []float64, newVal float64) float64 { +func rotateAndAccumulate(array []float64, newVal float64, acc MonitorAccumulation) float64 { total := float64(0.0) for i := len(array) - 1; i > 0; i-- { array[i] = array[i-1] @@ -179,52 +192,53 @@ func rotateAndAvg(array []float64, newVal float64) float64 { } array[0] = newVal total += newVal + if acc == Cumulative { + return total + } return total / float64(len(array)) } -func average(array []float64) float64 { +func accumulate(array []float64, acc MonitorAccumulation) float64 { total := float64(0) for _, x := range array { total += x } + if acc == Cumulative { + return total + } return total / float64(len(array)) } // monitorThread is the goroutine in a monitorHistory that does per minute monitoring and rotation func (mh *monitorHistory) monitorThread() { - timeout := time.Duration(0) // first pass right away - for { select { - case <-time.After(timeout): + case <-time.After(time.Minute): mh.lock.Lock() - minuteAvg := rotateAndAvg(mh.perMinutePerHour[:], mh.monitor()) + minuteAvg := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation) if time.Now().Sub(mh.timeLastHourRotate) > time.Hour { - rotateAndAvg(mh.perHourForDay[:], minuteAvg) + rotateAndAccumulate(mh.perHourForDay[:], minuteAvg, mh.monitorAccumulation) mh.timeLastHourRotate = time.Now() } if time.Now().Sub(mh.timeLastDayRotate) > time.Hour*24 { - rotateAndAvg(mh.perDayForWeek[:], average(mh.perHourForDay[:])) + rotateAndAccumulate(mh.perDayForWeek[:], accumulate(mh.perHourForDay[:], mh.monitorAccumulation), mh.monitorAccumulation) mh.timeLastDayRotate = time.Now() } if time.Now().Sub(mh.timeLastWeekRotate) > time.Hour*24*7 { - rotateAndAvg(mh.perWeekForMonth[:], average(mh.perDayForWeek[:])) + rotateAndAccumulate(mh.perWeekForMonth[:], accumulate(mh.perDayForWeek[:], mh.monitorAccumulation), mh.monitorAccumulation) mh.timeLastWeekRotate = time.Now() } if time.Now().Sub(mh.timeLastMonthRotate) > time.Hour*24*7*4 { - rotateAndAvg(mh.perMonthForYear[:], average(mh.perWeekForMonth[:])) + rotateAndAccumulate(mh.perMonthForYear[:], accumulate(mh.perWeekForMonth[:], mh.monitorAccumulation), mh.monitorAccumulation) mh.timeLastMonthRotate = time.Now() } mh.lock.Unlock() - // Repeat every minute - timeout = time.Duration(time.Minute) - case <-mh.breakChannel: return } diff --git a/server/metrics/metrics_test.go b/server/metrics/metrics_test.go index b838f33..5897d58 100644 --- a/server/metrics/metrics_test.go +++ b/server/metrics/metrics_test.go @@ -1,7 +1,6 @@ package metrics import ( - "runtime" "testing" "time" ) @@ -29,41 +28,3 @@ func TestCounter(t *testing.T) { t.Error("counter's starttime was innaccurate") } } - -func TestMonitorHistory(t *testing.T) { - value := float64(60 * 24 * 7 * 4 * 12) - numGoRoutinesStart := runtime.NumGoroutine() - mh := NewMonitorHistory(Count, func() float64 { return value }) - time.Sleep(1 * time.Second) - mh.Stop() - time.Sleep(1 * time.Second) - - minutes := mh.Minutes() - if minutes[0] != value { - t.Errorf("Monitor minutes's first minutes slot had the wrong value. Expected: %f Actual: %f", value, minutes[0]) - } - - hours := mh.Hours() - if hours[0] != value/60 { - t.Errorf("Monitor hour's first hour slot had the wrong value. Expected: %f Actual: %f", value, hours[0]) - } - - days := mh.Days() - if days[0] != value/60/24 { - t.Errorf("Monitor day's first day slot had the wrong value. Expected: %f Actual: %f", value/24, days[0]) - } - - weeks := mh.Weeks() - if weeks[0] != value/60/24/7 { - t.Errorf("Monitor week's first day slot had the wrong value. Expected: %f Actual: %f", value/24/7, weeks[0]) - } - - months := mh.Months() - if months[0] != 12 { - t.Errorf("Monitor months's first day slot had the wrong value. Expected: %d Actual: %f", 12, months[0]) - } - - if numGoRoutinesStart != runtime.NumGoroutine() { - t.Errorf("Monitor hour did not stop") - } -} diff --git a/server/metrics/monitors.go b/server/metrics/monitors.go index a22c4e9..0fd775c 100644 --- a/server/metrics/monitors.go +++ b/server/metrics/monitors.go @@ -32,10 +32,10 @@ func (mp *Monitors) Start(ra *application.RicochetApplication, log bool) { mp.starttime = time.Now() mp.breakChannel = make(chan bool) mp.MessageCounter = NewCounter() - mp.Messages = NewMonitorHistory(Count, func() (c float64) { c = float64(mp.MessageCounter.Count()); mp.MessageCounter.Reset(); return }) - mp.CPU = NewMonitorHistory(Percent, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.CPU) }) - mp.Memory = NewMonitorHistory(MegaBytes, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.Memory) }) - mp.ClientConns = NewMonitorHistory(Count, func() float64 { return float64(ra.ConnectionCount()) }) + mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) { c = float64(mp.MessageCounter.Count()); mp.MessageCounter.Reset(); return }) + mp.CPU = NewMonitorHistory(Percent, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.CPU) }) + mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.Memory) }) + mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ra.ConnectionCount()) }) if mp.log { go mp.run() @@ -63,7 +63,7 @@ func (mp *Monitors) report() { w := bufio.NewWriter(f) - fmt.Fprintf(w, "Started: %v\n\n", mp.starttime) + fmt.Fprintf(w, "Uptime: %v\n\n", time.Now().Sub(mp.starttime)) fmt.Fprintln(w, "Messages:") mp.Messages.Report(w)