diff --git a/.gitignore b/.gitignore index a37f180..0223701 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ config.yaml +config.test.yaml mydb.sql client/calls/ !client/calls/.gitkeep @@ -6,3 +7,4 @@ client/calls/ /calls Session.vim *.log +*.dlv diff --git a/internal/timeseries/timeseries.go b/internal/timeseries/timeseries.go index 889cec5..a0a8747 100644 --- a/internal/timeseries/timeseries.go +++ b/internal/timeseries/timeseries.go @@ -85,9 +85,11 @@ type Clock interface { Now() time.Time } -// defaultClock is used in case no clock is provided to the constructor. +// DefaultClock is used in case no clock is provided to the constructor. type defaultClock struct{} +var DefaultClock Clock = &defaultClock{} + func (c *defaultClock) Now() time.Time { return time.Now() } @@ -137,7 +139,7 @@ func NewTimeSeries(os ...Option) (*TimeSeries, error) { o(&opts) } if opts.clock == nil { - opts.clock = &defaultClock{} + opts.clock = DefaultClock } if opts.granularities == nil { opts.granularities = defaultGranularities diff --git a/internal/timeseries/timeseries_test.go b/internal/timeseries/timeseries_test.go index 2d0f745..8d87c30 100644 --- a/internal/timeseries/timeseries_test.go +++ b/internal/timeseries/timeseries_test.go @@ -3,8 +3,6 @@ package timeseries import ( "testing" "time" - - "github.com/benbjohnson/clock" ) // TODO: do table based testing diff --git a/internal/trending/item.go b/internal/trending/item.go index 534ef41..625a943 100644 --- a/internal/trending/item.go +++ b/internal/trending/item.go @@ -3,25 +3,27 @@ package trending import ( "math" "time" + + timeseries "dynatron.me/x/stillbox/internal/timeseries" ) -type item struct { +type item[K comparable] struct { eventSeries TimeSeries maxSeries SlidingWindow max float64 maxTime time.Time - options *options + options *options[K] // TODO: move outside of item because it's the same for all items defaultExpectation float64 defaultHourlyCount float64 } -func newItem(id string, options *options) *item { +func newItem[K comparable](id K, options *options[K]) *item[K] { defaultHourlyCount := float64(options.baseCount) * float64(options.storageDuration/time.Hour) defaultExpectation := float64(options.baseCount) / float64(time.Hour/options.recentDuration) - return &item{ + return &item[K]{ eventSeries: options.creator(id), maxSeries: options.slidingWindowCreator(id), options: options, @@ -31,10 +33,10 @@ func newItem(id string, options *options) *item { } } -func (i *item) score() score { +func (i *item[K]) score() Score[K] { recentCount, count := i.computeCounts() if recentCount < i.options.countThreshold { - return score{} + return Score[K]{} } if recentCount == count { // we see this for the first time so there is no historical data @@ -58,39 +60,41 @@ func (i *item) score() score { } i.decayMax() - mixedScore := 0.5 * (klScore + i.max) + mixedScore := 5 * (klScore + i.max) - return score{ + return Score[K]{ Score: mixedScore, Probability: probability, Expectation: expectation, Maximum: i.max, KLScore: klScore, + Count: count, + RecentCount: recentCount, } } -func (i *item) computeCounts() (float64, float64) { - now := time.Now() +func (i *item[K]) computeCounts() (float64, float64) { + now := timeseries.DefaultClock.Now() totalCount, _ := i.eventSeries.Range(now.Add(-i.options.storageDuration), now) count, _ := i.eventSeries.Range(now.Add(-i.options.recentDuration), now) return count, totalCount } -func (i *item) computeRecentMax() float64 { +func (i *item[K]) computeRecentMax() float64 { return i.maxSeries.Max() } -func (i *item) decayMax() { +func (i *item[K]) decayMax() { i.updateMax(i.max * i.computeExponentialDecayMultiplier()) } -func (i *item) updateMax(score float64) { +func (i *item[K]) updateMax(score float64) { i.max = score - i.maxTime = time.Now() + i.maxTime = timeseries.DefaultClock.Now() } -func (i *item) computeExponentialDecayMultiplier() float64 { - return math.Pow(0.5, float64(time.Now().Unix()-i.maxTime.Unix())/i.options.halfLife.Seconds()) +func (i *item[K]) computeExponentialDecayMultiplier() float64 { + return math.Pow(0.5, float64(timeseries.DefaultClock.Now().Unix()-i.maxTime.Unix())/i.options.halfLife.Seconds()) } func computeKullbackLeibler(probability float64, expectation float64) float64 { diff --git a/internal/trending/score.go b/internal/trending/score.go index 22e5f3a..f59368b 100644 --- a/internal/trending/score.go +++ b/internal/trending/score.go @@ -1,36 +1,38 @@ package trending -type score struct { - ID string +type Score[K comparable] struct { + ID K Score float64 Probability float64 Expectation float64 Maximum float64 KLScore float64 + Count float64 + RecentCount float64 } -type Scores []score +type Scores[K comparable] []Score[K] -func (s Scores) Len() int { +func (s Scores[K]) Len() int { return len(s) } -func (s Scores) Swap(i, j int) { +func (s Scores[K]) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s Scores) Less(i, j int) bool { +func (s Scores[K]) Less(i, j int) bool { return s[i].Score > s[j].Score } -func (s Scores) take(count int) Scores { +func (s Scores[K]) take(count int) Scores[K] { if count >= len(s) { return s } return s[0 : count-1] } -func (s Scores) threshold(t float64) Scores { +func (s Scores[K]) threshold(t float64) Scores[K] { for i := range s { if s[i].Score < t { return s[0:i] diff --git a/internal/trending/trending.go b/internal/trending/trending.go index 6117872..20e1df2 100644 --- a/internal/trending/trending.go +++ b/internal/trending/trending.go @@ -25,13 +25,13 @@ var defaultHalfLife = 2 * time.Hour var defaultRecentDuration = 5 * time.Minute var defaultStorageDuration = 7 * 24 * time.Hour var defaultMaxResults = 100 -var defaultBaseCount = 3 +var defaultBaseCount = 1 var defaultScoreThreshold = 0.01 var defaultCountThreshold = 3.0 -type options struct { - creator TimeSeriesCreator - slidingWindowCreator SlidingWindowCreator +type options[K comparable] struct { + creator TimeSeriesCreator[K] + slidingWindowCreator SlidingWindowCreator[K] halfLife time.Duration @@ -44,59 +44,59 @@ type options struct { countThreshold float64 } -type Option func(*options) +type Option[K comparable] func(*options[K]) -func WithTimeSeries(creator TimeSeriesCreator) Option { - return func(o *options) { +func WithTimeSeries[K comparable](creator TimeSeriesCreator[K]) Option[K] { + return func(o *options[K]) { o.creator = creator } } -func WithSlidingWindow(creator SlidingWindowCreator) Option { - return func(o *options) { +func WithSlidingWindow[K comparable](creator SlidingWindowCreator[K]) Option[K] { + return func(o *options[K]) { o.slidingWindowCreator = creator } } -func WithHalfLife(halfLife time.Duration) Option { - return func(o *options) { +func WithHalfLife[K comparable](halfLife time.Duration) Option[K] { + return func(o *options[K]) { o.halfLife = halfLife } } -func WithRecentDuration(recentDuration time.Duration) Option { - return func(o *options) { +func WithRecentDuration[K comparable](recentDuration time.Duration) Option[K] { + return func(o *options[K]) { o.recentDuration = recentDuration } } -func WithStorageDuration(storageDuration time.Duration) Option { - return func(o *options) { +func WithStorageDuration[K comparable](storageDuration time.Duration) Option[K] { + return func(o *options[K]) { o.storageDuration = storageDuration } } -func WithMaxResults(maxResults int) Option { - return func(o *options) { +func WithMaxResults[K comparable](maxResults int) Option[K] { + return func(o *options[K]) { o.maxResults = maxResults } } -func WithScoreThreshold(threshold float64) Option { - return func(o *options) { +func WithScoreThreshold[K comparable](threshold float64) Option[K] { + return func(o *options[K]) { o.scoreThreshold = threshold } } -func WithCountThreshold(threshold float64) Option { - return func(o *options) { +func WithCountThreshold[K comparable](threshold float64) Option[K] { + return func(o *options[K]) { o.countThreshold = threshold } } -type Scorer struct { - options options - items map[string]*item +type Scorer[K comparable] struct { + options options[K] + items map[K]*item[K] } type SlidingWindow interface { @@ -104,16 +104,16 @@ type SlidingWindow interface { Max() float64 } -type SlidingWindowCreator func(string) SlidingWindow +type SlidingWindowCreator[K comparable] func(K) SlidingWindow type TimeSeries interface { IncreaseAtTime(amount int, time time.Time) Range(start, end time.Time) (float64, error) } -type TimeSeriesCreator func(string) TimeSeries +type TimeSeriesCreator[K comparable] func(K) TimeSeries -func NewMemoryTimeSeries(id string) TimeSeries { +func NewMemoryTimeSeries[K comparable](id K) TimeSeries { ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities( []timeseries.Granularity{ {Granularity: time.Second, Count: 60}, @@ -125,13 +125,13 @@ func NewMemoryTimeSeries(id string) TimeSeries { return ts } -func NewScorer(options ...Option) Scorer { - scorer := Scorer{items: make(map[string]*item)} +func NewScorer[K comparable](options ...Option[K]) Scorer[K] { + scorer := Scorer[K]{items: make(map[K]*item[K])} for _, o := range options { o(&scorer.options) } if scorer.options.creator == nil { - scorer.options.creator = NewMemoryTimeSeries + scorer.options.creator = NewMemoryTimeSeries[K] } if scorer.options.halfLife == 0 { scorer.options.halfLife = defaultHalfLife @@ -155,7 +155,7 @@ func NewScorer(options ...Option) Scorer { scorer.options.baseCount = defaultBaseCount } if scorer.options.slidingWindowCreator == nil { - scorer.options.slidingWindowCreator = func(id string) SlidingWindow { + scorer.options.slidingWindowCreator = func(id K) SlidingWindow { return slidingwindow.NewSlidingWindow( slidingwindow.WithStep(time.Hour*24), slidingwindow.WithDuration(scorer.options.storageDuration), @@ -165,7 +165,7 @@ func NewScorer(options ...Option) Scorer { return scorer } -func (s *Scorer) AddEvent(id string, time time.Time) { +func (s *Scorer[K]) AddEvent(id K, time time.Time) { item := s.items[id] if item == nil { item = newItem(id, &s.options) @@ -174,12 +174,12 @@ func (s *Scorer) AddEvent(id string, time time.Time) { s.addToItem(item, time) } -func (s *Scorer) addToItem(item *item, time time.Time) { - item.eventSeries.IncreaseAtTime(1, time) +func (s *Scorer[K]) addToItem(item *item[K], tm time.Time) { + item.eventSeries.IncreaseAtTime(1, tm) } -func (s *Scorer) Score() Scores { - var scores Scores +func (s *Scorer[K]) Score() Scores[K] { + var scores Scores[K] for id, item := range s.items { score := item.score() score.ID = id diff --git a/pkg/gordio/alerting/alerting.go b/pkg/gordio/alerting/alerting.go new file mode 100644 index 0000000..cb5192a --- /dev/null +++ b/pkg/gordio/alerting/alerting.go @@ -0,0 +1,148 @@ +package alerting + +import ( + "context" + "sort" + "sync" + "time" + + cl "dynatron.me/x/stillbox/pkg/calls" + "dynatron.me/x/stillbox/pkg/gordio/database" + + "dynatron.me/x/stillbox/internal/timeseries" + "dynatron.me/x/stillbox/internal/trending" + + "github.com/rs/zerolog/log" +) + +const ( + StorageLookbackDays = 7 + HalfLife = 30 * time.Minute + RecentDuration = 2 * time.Hour + ScoreThreshold = -1 + CountThreshold = 1.0 + AlerterTickInterval = time.Minute +) + +type Alerter struct { + sync.RWMutex + scorer trending.Scorer[cl.Talkgroup] + scores trending.Scores[cl.Talkgroup] + lastScore time.Time +} + +type myClock struct { + offset time.Duration +} + +func (c *myClock) Now() time.Time { + return time.Now().Add(c.offset) +} + +func New() *Alerter { + as := &Alerter{ + scorer: trending.NewScorer[cl.Talkgroup]( + trending.WithTimeSeries(newTimeSeries), + trending.WithStorageDuration[cl.Talkgroup](StorageLookbackDays*24*time.Hour), + trending.WithRecentDuration[cl.Talkgroup](RecentDuration), + trending.WithHalfLife[cl.Talkgroup](HalfLife), + trending.WithScoreThreshold[cl.Talkgroup](ScoreThreshold), + trending.WithCountThreshold[cl.Talkgroup](CountThreshold), + ), + } + + return as +} + +func (as *Alerter) Go(ctx context.Context) { + as.startBackfill(ctx) + + as.score(ctx, time.Now()) + ticker := time.NewTicker(AlerterTickInterval) + + for { + select { + case now := <-ticker.C: + as.score(ctx, now) + case <-ctx.Done(): + ticker.Stop() + return + } + } + +} + +func newTimeSeries(id cl.Talkgroup) trending.TimeSeries { + ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities( + []timeseries.Granularity{ + {Granularity: time.Second, Count: 60}, + {Granularity: time.Minute, Count: 10}, + {Granularity: time.Hour, Count: 24}, + {Granularity: time.Hour * 24, Count: StorageLookbackDays}, + }, + )) + return ts +} + +func (as *Alerter) startBackfill(ctx context.Context) { + now := time.Now() + since := now.Add(StorageLookbackDays * -24 * time.Hour) + log.Debug().Time("since", since).Msg("starting stats backfill") + count, err := as.backfill(ctx, since) + if err != nil { + log.Error().Err(err).Msg("backfill failed") + return + } + log.Debug().Int("count", count).Str("in", time.Now().Sub(now).String()).Int("len", as.scorer.Score().Len()).Msg("backfill finished") +} + +func (as *Alerter) score(ctx context.Context, now time.Time) { + as.Lock() + defer as.Unlock() + + as.scores = as.scorer.Score() + as.lastScore = now + sort.Sort(as.scores) +} + +func (as *Alerter) backfill(ctx context.Context, since time.Time) (count int, err error) { + db := database.FromCtx(ctx) + const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2` + + rows, err := db.Query(ctx, backfillStatsQuery, since, timeseries.DefaultClock.Now()) + if err != nil { + return count, err + } + defer rows.Close() + + as.Lock() + defer as.Unlock() + + for rows.Next() { + var tg cl.Talkgroup + var callDate time.Time + if err := rows.Scan(&tg.System, &tg.Talkgroup, &callDate); err != nil { + return count, err + } + as.scorer.AddEvent(tg, callDate) + count++ + } + + if err := rows.Err(); err != nil { + return count, err + } + + return count, nil +} + +func (as *Alerter) SinkType() string { + return "alerting" +} + +func (as *Alerter) Call(ctx context.Context, call *cl.Call) error { + as.Lock() + defer as.Unlock() + as.scorer.AddEvent(call.TalkgroupTuple(), call.DateTime) + + return nil +} diff --git a/pkg/gordio/alerting/stats.go b/pkg/gordio/alerting/stats.go new file mode 100644 index 0000000..9e1fafa --- /dev/null +++ b/pkg/gordio/alerting/stats.go @@ -0,0 +1,71 @@ +package alerting + +import ( + _ "embed" + "fmt" + "html/template" + "net/http" + "time" + + "dynatron.me/x/stillbox/pkg/calls" + "dynatron.me/x/stillbox/pkg/gordio/database" + + "dynatron.me/x/stillbox/internal/trending" + + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" +) + +//go:embed stats.html +var statsTemplateFile string + +var ( + funcMap = template.FuncMap{ + "f": func(v float64) string { + return fmt.Sprintf("%.4f", v) + }, + } + statTmpl = template.Must(template.New("stats").Funcs(funcMap).Parse(statsTemplateFile)) +) + +func (as *Alerter) PrivateRoutes(r chi.Router) { + r.Get("/tgstats", as.tgStats) +} + +func (as *Alerter) tgStats(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + db := database.FromCtx(ctx) + + packed := make([]int64, 0, len(as.scores)) + for _, s := range as.scores { + packed = append(packed, s.ID.Pack()) + } + + tgs, err := db.GetTalkgroupsByPackedIDs(ctx, packed) + if err != nil { + log.Error().Err(err).Msg("stats TG get failed") + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + tgMap := make(map[calls.Talkgroup]database.GetTalkgroupsByPackedIDsRow, len(tgs)) + for _, t := range tgs { + tgMap[calls.Talkgroup{System: uint32(t.SystemID), Talkgroup: uint32(t.ID)}] = t + } + + renderData := struct { + TGs map[calls.Talkgroup]database.GetTalkgroupsByPackedIDsRow + Scores trending.Scores[calls.Talkgroup] + LastScore time.Time + }{ + TGs: tgMap, + Scores: as.scores, + LastScore: as.lastScore, + } + + w.WriteHeader(http.StatusOK) + err = statTmpl.Execute(w, renderData) + if err != nil { + log.Error().Err(err).Msg("stat template exec") + } +} diff --git a/pkg/gordio/alerting/stats.html b/pkg/gordio/alerting/stats.html new file mode 100644 index 0000000..61e4e56 --- /dev/null +++ b/pkg/gordio/alerting/stats.html @@ -0,0 +1,86 @@ + + +
+System | +TG | +TG ID | +Count | +Recent | +Score | +Probab | +Expect | +Max | +KL | +
---|---|---|---|---|---|---|---|---|---|
{{ $tg.Name_2}} | +{{ $tg.Name}} | +{{ .ID.Talkgroup }} | +{{ .Count }} | +{{ f .RecentCount }} | +{{ f .Score }} | +{{ f .Probability }} | +{{ f .Expectation }} | +{{ f .Maximum }} | +{{ f .KLScore }} | +
No Data | +|||||||||
Last updated at {{ .LastScore }} | +