Add talkgroup activity alerting #17
8 changed files with 107 additions and 48 deletions
|
@ -24,3 +24,8 @@ rateLimit:
|
||||||
enable: true
|
enable: true
|
||||||
requests: 200
|
requests: 200
|
||||||
over: 2m
|
over: 2m
|
||||||
|
alerting:
|
||||||
|
enable: true
|
||||||
|
lookbackDays: 7
|
||||||
|
halfLife: 30m
|
||||||
|
recent: 2h
|
||||||
|
|
|
@ -22,7 +22,7 @@ func newItem[K comparable](id K, options *options[K]) *item[K] {
|
||||||
defaultHourlyCount := float64(options.baseCount) * float64(options.storageDuration/time.Hour)
|
defaultHourlyCount := float64(options.baseCount) * float64(options.storageDuration/time.Hour)
|
||||||
defaultExpectation := float64(options.baseCount) / float64(time.Hour/options.recentDuration)
|
defaultExpectation := float64(options.baseCount) / float64(time.Hour/options.recentDuration)
|
||||||
return &item[K]{
|
return &item[K]{
|
||||||
eventSeries: options.creator(id, options.clock),
|
eventSeries: options.creator(id),
|
||||||
maxSeries: options.slidingWindowCreator(id),
|
maxSeries: options.slidingWindowCreator(id),
|
||||||
options: options,
|
options: options,
|
||||||
|
|
||||||
|
|
|
@ -118,9 +118,9 @@ type TimeSeries interface {
|
||||||
Range(start, end time.Time) (float64, error)
|
Range(start, end time.Time) (float64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type TimeSeriesCreator[K comparable] func(K, timeseries.Clock) TimeSeries
|
type TimeSeriesCreator[K comparable] func(K) TimeSeries
|
||||||
|
|
||||||
func NewMemoryTimeSeries[K comparable](id K, clock timeseries.Clock) TimeSeries {
|
func NewMemoryTimeSeries[K comparable](id K) TimeSeries {
|
||||||
ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities(
|
ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities(
|
||||||
[]timeseries.Granularity{
|
[]timeseries.Granularity{
|
||||||
{Granularity: time.Second, Count: 60},
|
{Granularity: time.Second, Count: 60},
|
||||||
|
@ -128,7 +128,7 @@ func NewMemoryTimeSeries[K comparable](id K, clock timeseries.Clock) TimeSeries
|
||||||
{Granularity: time.Hour, Count: 24},
|
{Granularity: time.Hour, Count: 24},
|
||||||
{Granularity: time.Hour * 24, Count: 7},
|
{Granularity: time.Hour * 24, Count: 7},
|
||||||
},
|
},
|
||||||
), timeseries.WithClock(clock))
|
))
|
||||||
return ts
|
return ts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,9 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cl "dynatron.me/x/stillbox/pkg/calls"
|
cl "dynatron.me/x/stillbox/pkg/calls"
|
||||||
|
"dynatron.me/x/stillbox/pkg/gordio/config"
|
||||||
"dynatron.me/x/stillbox/pkg/gordio/database"
|
"dynatron.me/x/stillbox/pkg/gordio/database"
|
||||||
|
"dynatron.me/x/stillbox/pkg/gordio/sinks"
|
||||||
|
|
||||||
"dynatron.me/x/stillbox/internal/timeseries"
|
"dynatron.me/x/stillbox/internal/timeseries"
|
||||||
"dynatron.me/x/stillbox/internal/trending"
|
"dynatron.me/x/stillbox/internal/trending"
|
||||||
|
@ -16,49 +18,85 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
StorageLookbackDays = 7
|
|
||||||
HalfLife = 30 * time.Minute
|
|
||||||
RecentDuration = 2 * time.Hour
|
|
||||||
ScoreThreshold = -1
|
ScoreThreshold = -1
|
||||||
CountThreshold = 1.0
|
CountThreshold = 1.0
|
||||||
AlerterTickInterval = time.Minute
|
alerterTickInterval = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
type Alerter struct {
|
type Alerter interface {
|
||||||
|
sinks.Sink
|
||||||
|
|
||||||
|
Enabled() bool
|
||||||
|
Go(context.Context)
|
||||||
|
|
||||||
|
stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type alerter struct {
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
|
clock timeseries.Clock
|
||||||
|
cfg config.Alerting
|
||||||
scorer trending.Scorer[cl.Talkgroup]
|
scorer trending.Scorer[cl.Talkgroup]
|
||||||
scores trending.Scores[cl.Talkgroup]
|
scores trending.Scores[cl.Talkgroup]
|
||||||
lastScore time.Time
|
lastScore time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
type myClock struct {
|
type noopAlerter struct{}
|
||||||
offset time.Duration
|
|
||||||
|
type offsetClock time.Duration
|
||||||
|
|
||||||
|
func (c *offsetClock) Now() time.Time {
|
||||||
|
return time.Now().Add(c.Duration())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *myClock) Now() time.Time {
|
func (c *offsetClock) Duration() time.Duration {
|
||||||
return time.Now().Add(c.offset)
|
return time.Duration(*c)
|
||||||
}
|
}
|
||||||
|
|
||||||
func New() *Alerter {
|
func OffsetClock(d time.Duration) offsetClock {
|
||||||
as := &Alerter{
|
return offsetClock(d)
|
||||||
scorer: trending.NewScorer[cl.Talkgroup](
|
}
|
||||||
trending.WithTimeSeries(newTimeSeries),
|
|
||||||
trending.WithStorageDuration[cl.Talkgroup](StorageLookbackDays*24*time.Hour),
|
type AlertOption func(*alerter)
|
||||||
trending.WithRecentDuration[cl.Talkgroup](RecentDuration),
|
|
||||||
trending.WithHalfLife[cl.Talkgroup](HalfLife),
|
func WithClock(clock timeseries.Clock) AlertOption {
|
||||||
trending.WithScoreThreshold[cl.Talkgroup](ScoreThreshold),
|
return func(as *alerter) {
|
||||||
trending.WithCountThreshold[cl.Talkgroup](CountThreshold),
|
as.clock = clock
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(cfg config.Alerting, opts ...AlertOption) Alerter {
|
||||||
|
if !cfg.Enable {
|
||||||
|
return &noopAlerter{}
|
||||||
|
}
|
||||||
|
|
||||||
|
as := &alerter{
|
||||||
|
cfg: cfg,
|
||||||
|
clock: timeseries.DefaultClock,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(as)
|
||||||
|
}
|
||||||
|
|
||||||
|
as.scorer = trending.NewScorer[cl.Talkgroup](
|
||||||
|
trending.WithTimeSeries(as.newTimeSeries),
|
||||||
|
trending.WithStorageDuration[cl.Talkgroup](time.Hour*24*time.Duration(cfg.LookbackDays)),
|
||||||
|
trending.WithRecentDuration[cl.Talkgroup](cfg.Recent),
|
||||||
|
trending.WithHalfLife[cl.Talkgroup](cfg.HalfLife),
|
||||||
|
trending.WithScoreThreshold[cl.Talkgroup](ScoreThreshold),
|
||||||
|
trending.WithCountThreshold[cl.Talkgroup](CountThreshold),
|
||||||
|
trending.WithClock[cl.Talkgroup](as.clock),
|
||||||
|
)
|
||||||
|
|
||||||
return as
|
return as
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) Go(ctx context.Context) {
|
func (as *alerter) Go(ctx context.Context) {
|
||||||
as.startBackfill(ctx)
|
as.startBackfill(ctx)
|
||||||
|
|
||||||
as.score(ctx, time.Now())
|
as.score(ctx, time.Now())
|
||||||
ticker := time.NewTicker(AlerterTickInterval)
|
ticker := time.NewTicker(alerterTickInterval)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
@ -72,21 +110,21 @@ func (as *Alerter) Go(ctx context.Context) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTimeSeries(id cl.Talkgroup, clock timeseries.Clock) trending.TimeSeries {
|
func (as *alerter) newTimeSeries(id cl.Talkgroup) trending.TimeSeries {
|
||||||
ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities(
|
ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities(
|
||||||
[]timeseries.Granularity{
|
[]timeseries.Granularity{
|
||||||
{Granularity: time.Second, Count: 60},
|
{Granularity: time.Second, Count: 60},
|
||||||
{Granularity: time.Minute, Count: 10},
|
{Granularity: time.Minute, Count: 10},
|
||||||
{Granularity: time.Hour, Count: 24},
|
{Granularity: time.Hour, Count: 24},
|
||||||
{Granularity: time.Hour * 24, Count: StorageLookbackDays},
|
{Granularity: time.Hour * 24, Count: int(as.cfg.LookbackDays)},
|
||||||
},
|
},
|
||||||
), timeseries.WithClock(clock))
|
), timeseries.WithClock(as.clock))
|
||||||
return ts
|
return ts
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) startBackfill(ctx context.Context) {
|
func (as *alerter) startBackfill(ctx context.Context) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
since := now.Add(StorageLookbackDays * -24 * time.Hour)
|
since := now.Add(-24 * time.Hour * time.Duration(as.cfg.LookbackDays))
|
||||||
log.Debug().Time("since", since).Msg("starting stats backfill")
|
log.Debug().Time("since", since).Msg("starting stats backfill")
|
||||||
count, err := as.backfill(ctx, since)
|
count, err := as.backfill(ctx, since)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -96,7 +134,7 @@ func (as *Alerter) startBackfill(ctx context.Context) {
|
||||||
log.Debug().Int("count", count).Str("in", time.Now().Sub(now).String()).Int("len", as.scorer.Score().Len()).Msg("backfill finished")
|
log.Debug().Int("count", count).Str("in", time.Now().Sub(now).String()).Int("len", as.scorer.Score().Len()).Msg("backfill finished")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) score(ctx context.Context, now time.Time) {
|
func (as *alerter) score(ctx context.Context, now time.Time) {
|
||||||
as.Lock()
|
as.Lock()
|
||||||
defer as.Unlock()
|
defer as.Unlock()
|
||||||
|
|
||||||
|
@ -105,7 +143,7 @@ func (as *Alerter) score(ctx context.Context, now time.Time) {
|
||||||
sort.Sort(as.scores)
|
sort.Sort(as.scores)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) backfill(ctx context.Context, since time.Time) (count int, err error) {
|
func (as *alerter) backfill(ctx context.Context, since time.Time) (count int, err error) {
|
||||||
db := database.FromCtx(ctx)
|
db := database.FromCtx(ctx)
|
||||||
const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2`
|
const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2`
|
||||||
|
|
||||||
|
@ -135,14 +173,21 @@ func (as *Alerter) backfill(ctx context.Context, since time.Time) (count int, er
|
||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) SinkType() string {
|
func (as *alerter) SinkType() string {
|
||||||
return "alerting"
|
return "alerting"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) Call(ctx context.Context, call *cl.Call) error {
|
func (as *alerter) Call(ctx context.Context, call *cl.Call) error {
|
||||||
as.Lock()
|
as.Lock()
|
||||||
defer as.Unlock()
|
defer as.Unlock()
|
||||||
as.scorer.AddEvent(call.TalkgroupTuple(), call.DateTime)
|
as.scorer.AddEvent(call.TalkgroupTuple(), call.DateTime)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*alerter) Enabled() bool { return true }
|
||||||
|
|
||||||
|
func (*noopAlerter) SinkType() string { return "noopAlerter" }
|
||||||
|
func (*noopAlerter) Call(_ context.Context, _ *cl.Call) error { return nil }
|
||||||
|
func (*noopAlerter) Go(_ context.Context) {}
|
||||||
|
func (*noopAlerter) Enabled() bool { return false }
|
||||||
|
|
|
@ -19,6 +19,10 @@ import (
|
||||||
//go:embed stats.html
|
//go:embed stats.html
|
||||||
var statsTemplateFile string
|
var statsTemplateFile string
|
||||||
|
|
||||||
|
type stats interface {
|
||||||
|
PrivateRoutes(chi.Router)
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
funcMap = template.FuncMap{
|
funcMap = template.FuncMap{
|
||||||
"f": func(v float64) string {
|
"f": func(v float64) string {
|
||||||
|
@ -28,11 +32,13 @@ var (
|
||||||
statTmpl = template.Must(template.New("stats").Funcs(funcMap).Parse(statsTemplateFile))
|
statTmpl = template.Must(template.New("stats").Funcs(funcMap).Parse(statsTemplateFile))
|
||||||
)
|
)
|
||||||
|
|
||||||
func (as *Alerter) PrivateRoutes(r chi.Router) {
|
func (as *alerter) PrivateRoutes(r chi.Router) {
|
||||||
r.Get("/tgstats", as.tgStats)
|
r.Get("/tgstats", as.tgStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (as *Alerter) tgStats(w http.ResponseWriter, r *http.Request) {
|
func (as *noopAlerter) PrivateRoutes(r chi.Router) {}
|
||||||
|
|
||||||
|
func (as *alerter) tgStats(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
db := database.FromCtx(ctx)
|
db := database.FromCtx(ctx)
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@ type Config struct {
|
||||||
DB DB `yaml:"db"`
|
DB DB `yaml:"db"`
|
||||||
CORS CORS `yaml:"cors"`
|
CORS CORS `yaml:"cors"`
|
||||||
Auth Auth `yaml:"auth"`
|
Auth Auth `yaml:"auth"`
|
||||||
|
Alerting Alerting `yaml:"alerting"`
|
||||||
Log []Logger `yaml:"log"`
|
Log []Logger `yaml:"log"`
|
||||||
Listen string `yaml:"listen"`
|
Listen string `yaml:"listen"`
|
||||||
Public bool `yaml:"public"`
|
Public bool `yaml:"public"`
|
||||||
|
@ -49,6 +50,13 @@ type RateLimit struct {
|
||||||
verifyError sync.Once
|
verifyError sync.Once
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type Alerting struct {
|
||||||
|
Enable bool `yaml:"enable"`
|
||||||
|
LookbackDays uint `yaml:"lookbackDays"`
|
||||||
|
HalfLife time.Duration `yaml:"halfLife"`
|
||||||
|
Recent time.Duration `yaml:"recent"`
|
||||||
|
}
|
||||||
|
|
||||||
func (rl *RateLimit) Verify() bool {
|
func (rl *RateLimit) Verify() bool {
|
||||||
if rl.Enable {
|
if rl.Enable {
|
||||||
if rl.Requests > 0 && rl.Over > 0 {
|
if rl.Requests > 0 && rl.Over > 0 {
|
||||||
|
|
|
@ -30,7 +30,7 @@ type Server struct {
|
||||||
sinks sinks.Sinks
|
sinks sinks.Sinks
|
||||||
nex *nexus.Nexus
|
nex *nexus.Nexus
|
||||||
logger *Logger
|
logger *Logger
|
||||||
alerter *alerting.Alerter
|
alerter alerting.Alerter
|
||||||
hup chan os.Signal
|
hup chan os.Signal
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -56,12 +56,16 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
|
||||||
r: r,
|
r: r,
|
||||||
nex: nexus.New(),
|
nex: nexus.New(),
|
||||||
logger: logger,
|
logger: logger,
|
||||||
alerter: alerting.New(),
|
alerter: alerting.New(cfg.Alerting),
|
||||||
}
|
}
|
||||||
|
|
||||||
srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db), true)
|
srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db), true)
|
||||||
srv.sinks.Register("nexus", sinks.NewNexusSink(srv.nex), false)
|
srv.sinks.Register("nexus", sinks.NewNexusSink(srv.nex), false)
|
||||||
srv.sinks.Register("alerting", sinks.NewAlerterSink(srv.alerter), false)
|
|
||||||
|
if srv.alerter.Enabled() {
|
||||||
|
srv.sinks.Register("alerting", srv.alerter, false)
|
||||||
|
}
|
||||||
|
|
||||||
srv.sources.Register("rdio-http", sources.NewRdioHTTP(authenticator, srv))
|
srv.sources.Register("rdio-http", sources.NewRdioHTTP(authenticator, srv))
|
||||||
|
|
||||||
r.Use(middleware.RequestID)
|
r.Use(middleware.RequestID)
|
||||||
|
|
|
@ -1,9 +0,0 @@
|
||||||
package sinks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"dynatron.me/x/stillbox/pkg/gordio/alerting"
|
|
||||||
)
|
|
||||||
|
|
||||||
func NewAlerterSink(a *alerting.Alerter) *alerting.Alerter {
|
|
||||||
return a
|
|
||||||
}
|
|
Loading…
Reference in a new issue