Compare commits

..

No commits in common. "6ea61bf820ea7c54fe16d85e26ad28b5f4ec190f" and "d0c398f6f74996d89588846238d996e046f9786c" have entirely different histories.

9 changed files with 157 additions and 269 deletions

View file

@ -33,11 +33,5 @@ alerting:
renotify: 30m renotify: 30m
notify: notify:
- provider: slackwebhook - provider: slackwebhook
# subjectTemplate: "Stillbox Alert ({{ highest . }})"
# bodyTemplate: |
# {{ range . -}}
# {{ .TGName }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
#
# {{ end -}}
config: config:
webhookURL: "http://somewhere" webhookURL: "http://somewhere"

View file

@ -1,6 +1,9 @@
package common package common
import ( import (
"fmt"
"strconv"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -44,3 +47,10 @@ func PtrOrNull[T comparable](val T) *T {
return &val return &val
} }
func FmtFloat(v float64, places ...int) string {
if len(places) > 0 {
return fmt.Sprintf("%."+strconv.Itoa(places[0])+"f", v)
}
return fmt.Sprintf("%.4f", v)
}

View file

@ -1,48 +0,0 @@
package common
import (
"errors"
"fmt"
"strconv"
"text/template"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
)
var (
FuncMap = template.FuncMap{
"f": fmtFloat,
"dict": func(values ...interface{}) (map[string]interface{}, error) {
if len(values)%2 != 0 {
return nil, errors.New("invalid dict call")
}
dict := make(map[string]interface{}, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, ok := values[i].(string)
if !ok {
return nil, errors.New("dict keys must be strings")
}
dict[key] = values[i+1]
}
return dict, nil
},
"formTime": func(t jsontime.Time) string {
return time.Time(t).Format("2006-01-02T15:04")
},
"ago": func(s string) (string, error) {
d, err := time.ParseDuration(s)
if err != nil {
return "", err
}
return time.Now().Add(-d).Format("2006-01-02T15:04"), nil
},
}
)
func fmtFloat(v float64, places ...int) string {
if len(places) > 0 {
return fmt.Sprintf("%."+strconv.Itoa(places[0])+"f", v)
}
return fmt.Sprintf("%.4f", v)
}

View file

@ -1,81 +0,0 @@
package alert
import (
"context"
"fmt"
"strconv"
"time"
"dynatron.me/x/stillbox/internal/trending"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Alert struct {
ID uuid.UUID
Timestamp time.Time
TGName string
Score trending.Score[talkgroups.ID]
OrigScore float64
Weight float32
Suppressed bool
}
func (a *Alert) ToAddAlertParams() database.AddAlertParams {
f32score := float32(a.Score.Score)
f32origscore := float32(a.OrigScore)
var origScore *float32
if a.Score.Score != a.OrigScore {
origScore = &f32origscore
}
return database.AddAlertParams{
ID: a.ID,
Time: pgtype.Timestamptz{Time: a.Timestamp, Valid: true},
PackedTg: a.Score.ID.Pack(),
Weight: &a.Weight,
Score: &f32score,
OrigScore: origScore,
Notified: !a.Suppressed,
}
}
// makeAlert creates a notification for later rendering by the template.
// It takes a talkgroup Score as input.
func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
d := Alert{
ID: uuid.New(),
Score: score,
Timestamp: time.Now(),
Weight: 1.0,
OrigScore: origScore,
}
tgRecord, err := store.TG(ctx, score.ID)
switch err {
case nil:
d.Weight = tgRecord.Talkgroup.Weight
if tgRecord.System.Name == "" {
tgRecord.System.Name = strconv.Itoa(int(score.ID.System))
}
if tgRecord.Talkgroup.Name != nil {
d.TGName = fmt.Sprintf("%s %s (%d)", tgRecord.System.Name, *tgRecord.Talkgroup.Name, score.ID.Talkgroup)
} else {
d.TGName = fmt.Sprintf("%s:%d", tgRecord.System.Name, int(score.ID.Talkgroup))
}
default:
system, has := store.SystemName(ctx, int(score.ID.System))
if has {
d.TGName = fmt.Sprintf("%s:%d", system, int(score.ID.Talkgroup))
} else {
d.TGName = fmt.Sprintf("%d:%d", int(score.ID.System), int(score.ID.Talkgroup))
}
}
return d, nil
}

View file

@ -1,14 +1,16 @@
package alerting package alerting
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"net/http" "net/http"
"sort" "sort"
"strconv"
"sync" "sync"
"text/template"
"time" "time"
"dynatron.me/x/stillbox/pkg/alerting/alert"
"dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/config"
"dynatron.me/x/stillbox/pkg/database" "dynatron.me/x/stillbox/pkg/database"
@ -19,6 +21,8 @@ import (
"dynatron.me/x/stillbox/internal/timeseries" "dynatron.me/x/stillbox/internal/timeseries"
"dynatron.me/x/stillbox/internal/trending" "dynatron.me/x/stillbox/internal/trending"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
@ -47,7 +51,7 @@ type alerter struct {
scores trending.Scores[talkgroups.ID] scores trending.Scores[talkgroups.ID]
lastScore time.Time lastScore time.Time
sim *Simulation sim *Simulation
alertCache map[talkgroups.ID]alert.Alert alertCache map[talkgroups.ID]Alert
renotify time.Duration renotify time.Duration
notifier notify.Notifier notifier notify.Notifier
tgCache talkgroups.Store tgCache talkgroups.Store
@ -92,7 +96,7 @@ func New(cfg config.Alerting, tgCache talkgroups.Store, opts ...AlertOption) Ale
as := &alerter{ as := &alerter{
cfg: cfg, cfg: cfg,
alertCache: make(map[talkgroups.ID]alert.Alert), alertCache: make(map[talkgroups.ID]Alert),
clock: timeseries.DefaultClock, clock: timeseries.DefaultClock,
renotify: DefaultRenotify, renotify: DefaultRenotify,
tgCache: tgCache, tgCache: tgCache,
@ -146,18 +150,22 @@ func (as *alerter) Go(ctx context.Context) {
} }
func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]alert.Alert, error) { const notificationTemplStr = `{{ range . -}}
{{ .TGName }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
{{ end -}}`
var notificationTemplate = template.Must(template.New("notification").Funcs(funcMap).Parse(notificationTemplStr))
func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]Alert, error) {
err := as.tgCache.Hint(ctx, as.scoredTGs()) err := as.tgCache.Hint(ctx, as.scoredTGs())
if err != nil { if err != nil {
return nil, fmt.Errorf("prime TG cache: %w", err) return nil, fmt.Errorf("prime TG cache: %w", err)
} }
as.Lock()
defer as.Unlock()
db := database.FromCtx(ctx) db := database.FromCtx(ctx)
var notifications []alert.Alert var notifications []Alert
for _, s := range as.scores { for _, s := range as.scores {
origScore := s.Score origScore := s.Score
tgr, err := as.tgCache.TG(ctx, s.ID) tgr, err := as.tgCache.TG(ctx, s.ID)
@ -168,7 +176,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
if s.Score > as.cfg.AlertThreshold || testMode { if s.Score > as.cfg.AlertThreshold || testMode {
if old, inCache := as.alertCache[s.ID]; !inCache || now.Sub(old.Timestamp) > as.renotify { if old, inCache := as.alertCache[s.ID]; !inCache || now.Sub(old.Timestamp) > as.renotify {
s.Score *= as.tgCache.Weight(ctx, s.ID, now) s.Score *= as.tgCache.Weight(ctx, s.ID, now)
a, err := alert.Make(ctx, as.tgCache, s, origScore) a, err := as.makeAlert(ctx, s, origScore)
if err != nil { if err != nil {
return nil, fmt.Errorf("makeAlert: %w", err) return nil, fmt.Errorf("makeAlert: %w", err)
} }
@ -198,7 +206,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
} }
func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) { func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) {
alerts := make([]alert.Alert, 0, len(as.scores)) alerts := make([]Alert, 0, len(as.scores))
ctx := r.Context() ctx := r.Context()
alerts, err := as.eval(ctx, time.Now(), true) alerts, err := as.eval(ctx, time.Now(), true)
@ -208,7 +216,7 @@ func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
err = as.notifier.Send(ctx, alerts) err = as.sendNotification(ctx, alerts)
if err != nil { if err != nil {
log.Error().Err(err).Msg("test notification send") log.Error().Err(err).Msg("test notification send")
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
@ -250,12 +258,92 @@ func (as *alerter) notify(ctx context.Context) error {
} }
if len(notifications) > 0 { if len(notifications) > 0 {
return as.notifier.Send(ctx, notifications) return as.sendNotification(ctx, notifications)
} }
return nil return nil
} }
type Alert struct {
ID uuid.UUID
Timestamp time.Time
TGName string
Score trending.Score[talkgroups.ID]
OrigScore float64
Weight float32
Suppressed bool
}
func (a *Alert) ToAddAlertParams() database.AddAlertParams {
f32score := float32(a.Score.Score)
f32origscore := float32(a.OrigScore)
var origScore *float32
if a.Score.Score != a.OrigScore {
origScore = &f32origscore
}
return database.AddAlertParams{
ID: a.ID,
Time: pgtype.Timestamptz{Time: a.Timestamp, Valid: true},
PackedTg: a.Score.ID.Pack(),
Weight: &a.Weight,
Score: &f32score,
OrigScore: origScore,
Notified: !a.Suppressed,
}
}
// sendNotification renders and sends the notification.
func (as *alerter) sendNotification(ctx context.Context, n []Alert) error {
msgBuffer := new(bytes.Buffer)
err := notificationTemplate.Execute(msgBuffer, n)
if err != nil {
return fmt.Errorf("notification template render: %w", err)
}
log.Debug().Str("msg", msgBuffer.String()).Msg("notifying")
return as.notifier.Send(ctx, NotificationSubject, msgBuffer.String())
}
// makeAlert creates a notification for later rendering by the template.
// It takes a talkgroup Score as input.
func (as *alerter) makeAlert(ctx context.Context, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
d := Alert{
ID: uuid.New(),
Score: score,
Timestamp: time.Now(),
Weight: 1.0,
OrigScore: origScore,
}
tgRecord, err := as.tgCache.TG(ctx, score.ID)
switch err {
case nil:
d.Weight = tgRecord.Talkgroup.Weight
if tgRecord.System.Name == "" {
tgRecord.System.Name = strconv.Itoa(int(score.ID.System))
}
if tgRecord.Talkgroup.Name != nil {
d.TGName = fmt.Sprintf("%s %s (%d)", tgRecord.System.Name, *tgRecord.Talkgroup.Name, score.ID.Talkgroup)
} else {
d.TGName = fmt.Sprintf("%s:%d", tgRecord.System.Name, int(score.ID.Talkgroup))
}
default:
system, has := as.tgCache.SystemName(ctx, int(score.ID.System))
if has {
d.TGName = fmt.Sprintf("%s:%d", system, int(score.ID.Talkgroup))
} else {
d.TGName = fmt.Sprintf("%d:%d", int(score.ID.System), int(score.ID.Talkgroup))
}
}
return d, nil
}
// cleanCache clears the cache of aged-out entries // cleanCache clears the cache of aged-out entries
func (as *alerter) cleanCache() { func (as *alerter) cleanCache() {
if as.notifier == nil { if as.notifier == nil {

View file

@ -2,6 +2,7 @@ package alerting
import ( import (
_ "embed" _ "embed"
"errors"
"html/template" "html/template"
"net/http" "net/http"
"time" "time"
@ -11,6 +12,7 @@ import (
"dynatron.me/x/stillbox/pkg/talkgroups" "dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/internal/common" "dynatron.me/x/stillbox/internal/common"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/trending" "dynatron.me/x/stillbox/internal/trending"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
@ -20,14 +22,41 @@ import (
//go:embed stats.html //go:embed stats.html
var statsTemplateFile string var statsTemplateFile string
var (
statTmpl = template.Must(template.New("stats").Funcs(common.FuncMap).Parse(statsTemplateFile))
)
type stats interface { type stats interface {
PrivateRoutes(chi.Router) PrivateRoutes(chi.Router)
} }
var (
funcMap = template.FuncMap{
"f": common.FmtFloat,
"dict": func(values ...interface{}) (map[string]interface{}, error) {
if len(values)%2 != 0 {
return nil, errors.New("invalid dict call")
}
dict := make(map[string]interface{}, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, ok := values[i].(string)
if !ok {
return nil, errors.New("dict keys must be strings")
}
dict[key] = values[i+1]
}
return dict, nil
},
"formTime": func(t jsontime.Time) string {
return time.Time(t).Format("2006-01-02T15:04")
},
"ago": func(s string) (string, error) {
d, err := time.ParseDuration(s)
if err != nil {
return "", err
}
return time.Now().Add(-d).Format("2006-01-02T15:04"), nil
},
}
statTmpl = template.Must(template.New("stats").Funcs(funcMap).Parse(statsTemplateFile))
)
func (as *alerter) PrivateRoutes(r chi.Router) { func (as *alerter) PrivateRoutes(r chi.Router) {
r.Get("/tgstats", as.tgStatsHandler) r.Get("/tgstats", as.tgStatsHandler)
r.Post("/tgstats", as.simulateHandler) r.Post("/tgstats", as.simulateHandler)

View file

@ -65,10 +65,8 @@ type Alerting struct {
type Notify []NotifyService type Notify []NotifyService
type NotifyService struct { type NotifyService struct {
Provider string `yaml:"provider" json:"provider"` Provider string `json:"provider"`
SubjectTemplate *string `yaml:"subjectTemplate" json:"subjectTemplate"` Config map[string]interface{} `json:"config"`
BodyTemplate *string `yaml:"bodyTemplate" json:"bodyTemplate"`
Config map[string]interface{} `yaml:"config" json:"config"`
} }
func (n *NotifyService) GetS(k, defaultVal string) string { func (n *NotifyService) GetS(k, defaultVal string) string {

View file

@ -1,15 +1,10 @@
package notify package notify
import ( import (
"bytes"
"context"
"fmt" "fmt"
stdhttp "net/http" stdhttp "net/http"
"text/template"
"time" "time"
"dynatron.me/x/stillbox/internal/common"
"dynatron.me/x/stillbox/pkg/alerting/alert"
"dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/config"
"github.com/go-viper/mapstructure/v2" "github.com/go-viper/mapstructure/v2"
@ -18,70 +13,15 @@ import (
) )
type Notifier interface { type Notifier interface {
Send(ctx context.Context, alerts []alert.Alert) error notify.Notifier
}
type backend struct {
*notify.Notify
subject *template.Template
body *template.Template
} }
type notifier struct { type notifier struct {
backends []backend *notify.Notify
} }
func highest(a []alert.Alert) string { func (n *notifier) buildSlackWebhookPayload(cfg *slackWebhookConfig) func(string, string) any {
if len(a) < 1 {
return "none"
}
top := a[0]
for _, a := range a {
if a.Score.Score > top.Score.Score {
top = a
}
}
return top.TGName
}
var alertFm = template.FuncMap{
"highest": highest,
}
const defaultBodyTemplStr = `{{ range . -}}
{{ .TGName }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
{{ end -}}`
var defaultBodyTemplate = template.Must(template.New("body").Funcs(common.FuncMap).Funcs(alertFm).Parse(defaultBodyTemplStr))
var defaultSubjectTemplStr = `Stillbox Alert ({{ highest . }}`
var defaultSubjectTemplate = template.Must(template.New("subject").Funcs(common.FuncMap).Funcs(alertFm).Parse(defaultSubjectTemplStr))
// Send renders and sends the Alerts.
func (b *backend) Send(ctx context.Context, alerts []alert.Alert) (err error) {
var subject, body bytes.Buffer
err = b.subject.ExecuteTemplate(&subject, "subject", alerts)
if err != nil {
return err
}
err = b.body.ExecuteTemplate(&body, "body", alerts)
if err != nil {
return err
}
err = b.Notify.Send(ctx, subject.String(), body.String())
if err != nil {
return err
}
return nil
}
func buildSlackWebhookPayload(cfg *slackWebhookConfig) func(string, string) any {
type Attachment struct { type Attachment struct {
Title string `json:"title"` Title string `json:"title"`
Text string `json:"text"` Text string `json:"text"`
@ -116,35 +56,9 @@ type slackWebhookConfig struct {
WebhookURL string `mapstructure:"webhookURL"` WebhookURL string `mapstructure:"webhookURL"`
Icon string `mapstructure:"icon"` Icon string `mapstructure:"icon"`
MessageURL string `mapstructure:"messageURL"` MessageURL string `mapstructure:"messageURL"`
SubjectTemplate string `mapstructure:"subjectTemplate"`
BodyTemplate string `mapstructure:"bodyTemplate"`
} }
func (n *notifier) addService(cfg config.NotifyService) (err error) { func (n *notifier) addService(cfg config.NotifyService) error {
be := backend{}
switch cfg.SubjectTemplate {
case nil:
be.subject = defaultSubjectTemplate
default:
be.subject, err = template.New("subject").Funcs(common.FuncMap).Funcs(alertFm).Parse(*cfg.SubjectTemplate)
if err != nil {
return err
}
}
switch cfg.BodyTemplate {
case nil:
be.body = defaultBodyTemplate
default:
be.body, err = template.New("body").Funcs(common.FuncMap).Funcs(alertFm).Parse(*cfg.BodyTemplate)
if err != nil {
return err
}
}
be.Notify = notify.New()
switch cfg.Provider { switch cfg.Provider {
case "slackwebhook": case "slackwebhook":
swc := &slackWebhookConfig{ swc := &slackWebhookConfig{
@ -160,31 +74,20 @@ func (n *notifier) addService(cfg config.NotifyService) (err error) {
Header: make(stdhttp.Header), Header: make(stdhttp.Header),
Method: stdhttp.MethodPost, Method: stdhttp.MethodPost,
URL: swc.WebhookURL, URL: swc.WebhookURL,
BuildPayload: buildSlackWebhookPayload(swc), BuildPayload: n.buildSlackWebhookPayload(swc),
}) })
be.UseServices(hs) n.UseServices(hs)
default: default:
return fmt.Errorf("unknown provider '%s'", cfg.Provider) return fmt.Errorf("unknown provider '%s'", cfg.Provider)
} }
n.backends = append(n.backends, be)
return nil
}
func (n *notifier) Send(ctx context.Context, alerts []alert.Alert) error {
for _, be := range n.backends {
err := be.Send(ctx, alerts)
if err != nil {
return err
}
}
return nil return nil
} }
func New(cfg config.Notify) (Notifier, error) { func New(cfg config.Notify) (Notifier, error) {
n := new(notifier) n := &notifier{
Notify: notify.NewWithServices(),
}
for _, s := range cfg { for _, s := range cfg {
err := n.addService(s) err := n.addService(s)

View file

@ -12,11 +12,6 @@ type Talkgroup struct {
Learned bool `json:"learned"` Learned bool `json:"learned"`
} }
type Names struct {
System string
Talkgroup string
}
type ID struct { type ID struct {
System uint32 `json:"sys"` System uint32 `json:"sys"`
Talkgroup uint32 `json:"tg"` Talkgroup uint32 `json:"tg"`