forked from cwtch.im/cwtch
Merge branch 'metrics-total' of dan/cwtch into master
This commit is contained in:
commit
8d2a1aaf32
|
@ -61,8 +61,19 @@ const (
|
||||||
MegaBytes
|
MegaBytes
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MonitorAccumulation controls how monitor data is accumulated over time into larger summary buckets
|
||||||
|
type MonitorAccumulation int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Cumulative values with sum over time
|
||||||
|
Cumulative MonitorAccumulation = iota
|
||||||
|
// Average values will average over time
|
||||||
|
Average
|
||||||
|
)
|
||||||
|
|
||||||
type monitorHistory struct {
|
type monitorHistory struct {
|
||||||
monitorType MonitorType
|
monitorType MonitorType
|
||||||
|
monitorAccumulation MonitorAccumulation
|
||||||
|
|
||||||
starttime time.Time
|
starttime time.Time
|
||||||
perMinutePerHour [60]float64
|
perMinutePerHour [60]float64
|
||||||
|
@ -96,8 +107,10 @@ type MonitorHistory interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMonitorHistory returns a new MonitorHistory with starttime of time.Now and Started running with supplied monitor
|
// NewMonitorHistory returns a new MonitorHistory with starttime of time.Now and Started running with supplied monitor
|
||||||
func NewMonitorHistory(t MonitorType, monitor func() float64) MonitorHistory {
|
func NewMonitorHistory(t MonitorType, a MonitorAccumulation, monitor func() float64) MonitorHistory {
|
||||||
mh := &monitorHistory{monitorType: t, starttime: time.Now(), monitor: monitor, breakChannel: make(chan bool)}
|
mh := &monitorHistory{monitorType: t, monitorAccumulation: a, starttime: time.Now(), monitor: monitor, breakChannel: make(chan bool),
|
||||||
|
timeLastHourRotate: time.Now(), timeLastDayRotate: time.Now(), timeLastWeekRotate: time.Now(),
|
||||||
|
timeLastMonthRotate: time.Now()}
|
||||||
mh.Start()
|
mh.Start()
|
||||||
return mh
|
return mh
|
||||||
}
|
}
|
||||||
|
@ -148,7 +161,7 @@ func (mh *monitorHistory) Report(w *bufio.Writer) {
|
||||||
func reportLine(t MonitorType, array []float64) string {
|
func reportLine(t MonitorType, array []float64) string {
|
||||||
switch t {
|
switch t {
|
||||||
case Count:
|
case Count:
|
||||||
return strings.Trim(strings.Join(strings.Fields(fmt.Sprint(array)), " "), "[]")
|
return strings.Trim(strings.Join(strings.Fields(fmt.Sprintf("%.0f", array)), " "), "[]")
|
||||||
case Percent:
|
case Percent:
|
||||||
return strings.Trim(strings.Join(strings.Fields(fmt.Sprintf("%.2f", array)), " "), "[]")
|
return strings.Trim(strings.Join(strings.Fields(fmt.Sprintf("%.2f", array)), " "), "[]")
|
||||||
case MegaBytes:
|
case MegaBytes:
|
||||||
|
@ -171,7 +184,7 @@ func (mh *monitorHistory) returnCopy(slice []float64) []float64 {
|
||||||
return retSlice
|
return retSlice
|
||||||
}
|
}
|
||||||
|
|
||||||
func rotateAndAvg(array []float64, newVal float64) float64 {
|
func rotateAndAccumulate(array []float64, newVal float64, acc MonitorAccumulation) float64 {
|
||||||
total := float64(0.0)
|
total := float64(0.0)
|
||||||
for i := len(array) - 1; i > 0; i-- {
|
for i := len(array) - 1; i > 0; i-- {
|
||||||
array[i] = array[i-1]
|
array[i] = array[i-1]
|
||||||
|
@ -179,52 +192,53 @@ func rotateAndAvg(array []float64, newVal float64) float64 {
|
||||||
}
|
}
|
||||||
array[0] = newVal
|
array[0] = newVal
|
||||||
total += newVal
|
total += newVal
|
||||||
|
if acc == Cumulative {
|
||||||
|
return total
|
||||||
|
}
|
||||||
return total / float64(len(array))
|
return total / float64(len(array))
|
||||||
}
|
}
|
||||||
func average(array []float64) float64 {
|
func accumulate(array []float64, acc MonitorAccumulation) float64 {
|
||||||
total := float64(0)
|
total := float64(0)
|
||||||
for _, x := range array {
|
for _, x := range array {
|
||||||
total += x
|
total += x
|
||||||
}
|
}
|
||||||
|
if acc == Cumulative {
|
||||||
|
return total
|
||||||
|
}
|
||||||
return total / float64(len(array))
|
return total / float64(len(array))
|
||||||
}
|
}
|
||||||
|
|
||||||
// monitorThread is the goroutine in a monitorHistory that does per minute monitoring and rotation
|
// monitorThread is the goroutine in a monitorHistory that does per minute monitoring and rotation
|
||||||
func (mh *monitorHistory) monitorThread() {
|
func (mh *monitorHistory) monitorThread() {
|
||||||
timeout := time.Duration(0) // first pass right away
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-time.After(timeout):
|
case <-time.After(time.Minute):
|
||||||
mh.lock.Lock()
|
mh.lock.Lock()
|
||||||
|
|
||||||
minuteAvg := rotateAndAvg(mh.perMinutePerHour[:], mh.monitor())
|
minuteAvg := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
|
||||||
|
|
||||||
if time.Now().Sub(mh.timeLastHourRotate) > time.Hour {
|
if time.Now().Sub(mh.timeLastHourRotate) > time.Hour {
|
||||||
rotateAndAvg(mh.perHourForDay[:], minuteAvg)
|
rotateAndAccumulate(mh.perHourForDay[:], minuteAvg, mh.monitorAccumulation)
|
||||||
mh.timeLastHourRotate = time.Now()
|
mh.timeLastHourRotate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
if time.Now().Sub(mh.timeLastDayRotate) > time.Hour*24 {
|
if time.Now().Sub(mh.timeLastDayRotate) > time.Hour*24 {
|
||||||
rotateAndAvg(mh.perDayForWeek[:], average(mh.perHourForDay[:]))
|
rotateAndAccumulate(mh.perDayForWeek[:], accumulate(mh.perHourForDay[:], mh.monitorAccumulation), mh.monitorAccumulation)
|
||||||
mh.timeLastDayRotate = time.Now()
|
mh.timeLastDayRotate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
if time.Now().Sub(mh.timeLastWeekRotate) > time.Hour*24*7 {
|
if time.Now().Sub(mh.timeLastWeekRotate) > time.Hour*24*7 {
|
||||||
rotateAndAvg(mh.perWeekForMonth[:], average(mh.perDayForWeek[:]))
|
rotateAndAccumulate(mh.perWeekForMonth[:], accumulate(mh.perDayForWeek[:], mh.monitorAccumulation), mh.monitorAccumulation)
|
||||||
mh.timeLastWeekRotate = time.Now()
|
mh.timeLastWeekRotate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
if time.Now().Sub(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
|
if time.Now().Sub(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
|
||||||
rotateAndAvg(mh.perMonthForYear[:], average(mh.perWeekForMonth[:]))
|
rotateAndAccumulate(mh.perMonthForYear[:], accumulate(mh.perWeekForMonth[:], mh.monitorAccumulation), mh.monitorAccumulation)
|
||||||
mh.timeLastMonthRotate = time.Now()
|
mh.timeLastMonthRotate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
mh.lock.Unlock()
|
mh.lock.Unlock()
|
||||||
|
|
||||||
// Repeat every minute
|
|
||||||
timeout = time.Duration(time.Minute)
|
|
||||||
|
|
||||||
case <-mh.breakChannel:
|
case <-mh.breakChannel:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package metrics
|
package metrics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
@ -29,41 +28,3 @@ func TestCounter(t *testing.T) {
|
||||||
t.Error("counter's starttime was innaccurate")
|
t.Error("counter's starttime was innaccurate")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMonitorHistory(t *testing.T) {
|
|
||||||
value := float64(60 * 24 * 7 * 4 * 12)
|
|
||||||
numGoRoutinesStart := runtime.NumGoroutine()
|
|
||||||
mh := NewMonitorHistory(Count, func() float64 { return value })
|
|
||||||
time.Sleep(1 * time.Second)
|
|
||||||
mh.Stop()
|
|
||||||
time.Sleep(1 * time.Second)
|
|
||||||
|
|
||||||
minutes := mh.Minutes()
|
|
||||||
if minutes[0] != value {
|
|
||||||
t.Errorf("Monitor minutes's first minutes slot had the wrong value. Expected: %f Actual: %f", value, minutes[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
hours := mh.Hours()
|
|
||||||
if hours[0] != value/60 {
|
|
||||||
t.Errorf("Monitor hour's first hour slot had the wrong value. Expected: %f Actual: %f", value, hours[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
days := mh.Days()
|
|
||||||
if days[0] != value/60/24 {
|
|
||||||
t.Errorf("Monitor day's first day slot had the wrong value. Expected: %f Actual: %f", value/24, days[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
weeks := mh.Weeks()
|
|
||||||
if weeks[0] != value/60/24/7 {
|
|
||||||
t.Errorf("Monitor week's first day slot had the wrong value. Expected: %f Actual: %f", value/24/7, weeks[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
months := mh.Months()
|
|
||||||
if months[0] != 12 {
|
|
||||||
t.Errorf("Monitor months's first day slot had the wrong value. Expected: %d Actual: %f", 12, months[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
if numGoRoutinesStart != runtime.NumGoroutine() {
|
|
||||||
t.Errorf("Monitor hour did not stop")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -32,10 +32,10 @@ func (mp *Monitors) Start(ra *application.RicochetApplication, log bool) {
|
||||||
mp.starttime = time.Now()
|
mp.starttime = time.Now()
|
||||||
mp.breakChannel = make(chan bool)
|
mp.breakChannel = make(chan bool)
|
||||||
mp.MessageCounter = NewCounter()
|
mp.MessageCounter = NewCounter()
|
||||||
mp.Messages = NewMonitorHistory(Count, func() (c float64) { c = float64(mp.MessageCounter.Count()); mp.MessageCounter.Reset(); return })
|
mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) { c = float64(mp.MessageCounter.Count()); mp.MessageCounter.Reset(); return })
|
||||||
mp.CPU = NewMonitorHistory(Percent, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.CPU) })
|
mp.CPU = NewMonitorHistory(Percent, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.CPU) })
|
||||||
mp.Memory = NewMonitorHistory(MegaBytes, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.Memory) })
|
mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.Memory) })
|
||||||
mp.ClientConns = NewMonitorHistory(Count, func() float64 { return float64(ra.ConnectionCount()) })
|
mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ra.ConnectionCount()) })
|
||||||
|
|
||||||
if mp.log {
|
if mp.log {
|
||||||
go mp.run()
|
go mp.run()
|
||||||
|
@ -63,7 +63,7 @@ func (mp *Monitors) report() {
|
||||||
|
|
||||||
w := bufio.NewWriter(f)
|
w := bufio.NewWriter(f)
|
||||||
|
|
||||||
fmt.Fprintf(w, "Started: %v\n\n", mp.starttime)
|
fmt.Fprintf(w, "Uptime: %v\n\n", time.Now().Sub(mp.starttime))
|
||||||
|
|
||||||
fmt.Fprintln(w, "Messages:")
|
fmt.Fprintln(w, "Messages:")
|
||||||
mp.Messages.Report(w)
|
mp.Messages.Report(w)
|
||||||
|
|
Loading…
Reference in New Issue