amigan/stillbox
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

Add talkgroup activity alerting #17

Merged
amigan merged 11 commits from alerting into trunk 2024-10-31 00:20:48 -04:00
Conversation 0 Commits 11 Files changed 38 +57 -49
7 changed files with 57 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit b917531f14 - Show all commits

2
internal/trending/trending.go
View file

@ -1,6 +1,7 @@
package trending package trending
import ( import (
"fmt"
"sort" "sort"
"time" "time"
@ -175,6 +176,7 @@ func (s *Scorer[K]) AddEvent(id K, time time.Time) {
} }
func (s *Scorer[K]) addToItem(item *item[K], time time.Time) { func (s *Scorer[K]) addToItem(item *item[K], time time.Time) {
fmt.Println("add", time.String())
item.eventSeries.IncreaseAtTime(1, time) item.eventSeries.IncreaseAtTime(1, time)
} }

34
pkg/gordio/database/calls.sql.go
View file

@ -7,9 +7,9 @@ package database
import ( import (
"context" "context"
"time"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
) )
const addCall = `-- name: AddCall :one const addCall = `-- name: AddCall :one
@ -36,22 +36,22 @@ RETURNING id
` `
type AddCallParams struct { type AddCallParams struct {
Submitter *int32 `json:"submitter"` Submitter *int32 `json:"submitter"`
System int `json:"system"` System int `json:"system"`
Talkgroup int `json:"talkgroup"` Talkgroup int `json:"talkgroup"`
CallDate time.Time `json:"call_date"` CallDate pgtype.Timestamptz `json:"call_date"`
AudioName *string `json:"audio_name"` AudioName *string `json:"audio_name"`
AudioBlob []byte `json:"audio_blob"` AudioBlob []byte `json:"audio_blob"`
AudioType *string `json:"audio_type"` AudioType *string `json:"audio_type"`
AudioUrl *string `json:"audio_url"` AudioUrl *string `json:"audio_url"`
Duration *int32 `json:"duration"` Duration *int32 `json:"duration"`
Frequency int `json:"frequency"` Frequency int `json:"frequency"`
Frequencies []int `json:"frequencies"` Frequencies []int `json:"frequencies"`
Patches []int `json:"patches"` Patches []int `json:"patches"`
TgLabel *string `json:"tg_label"` TgLabel *string `json:"tg_label"`
TgAlphaTag *string `json:"tg_alpha_tag"` TgAlphaTag *string `json:"tg_alpha_tag"`
TgGroup *string `json:"tg_group"` TgGroup *string `json:"tg_group"`
Source int `json:"source"` Source int `json:"source"`
} }
func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) (uuid.UUID, error) { func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) (uuid.UUID, error) {

7
pkg/gordio/database/database.go
View file

@ -40,7 +40,12 @@ func NewClient(ctx context.Context, conf config.DB) (*DB, error) {
m.Close() m.Close()
pool, err := pgxpool.New(ctx, conf.Connect) pgConf, err := pgxpool.ParseConfig(conf.Connect)
if err != nil {
return nil, err
}
pool, err := pgxpool.NewWithConfig(ctx, pgConf)
if err != nil { if err != nil {
return nil, err return nil, err
} }

36
pkg/gordio/database/models.go
View file

@ -21,24 +21,24 @@ type ApiKey struct {
} }
type Call struct { type Call struct {
ID uuid.UUID `json:"id"` ID uuid.UUID `json:"id"`
Submitter *int32 `json:"submitter"` Submitter *int32 `json:"submitter"`
System int `json:"system"` System int `json:"system"`
Talkgroup int `json:"talkgroup"` Talkgroup int `json:"talkgroup"`
CallDate time.Time `json:"call_date"` CallDate pgtype.Timestamptz `json:"call_date"`
AudioName *string `json:"audio_name"` AudioName *string `json:"audio_name"`
AudioBlob []byte `json:"audio_blob"` AudioBlob []byte `json:"audio_blob"`
Duration *int32 `json:"duration"` Duration *int32 `json:"duration"`
AudioType *string `json:"audio_type"` AudioType *string `json:"audio_type"`
AudioUrl *string `json:"audio_url"` AudioUrl *string `json:"audio_url"`
Frequency int `json:"frequency"` Frequency int `json:"frequency"`
Frequencies []int `json:"frequencies"` Frequencies []int `json:"frequencies"`
Patches []int `json:"patches"` Patches []int `json:"patches"`
TgLabel *string `json:"tg_label"` TgLabel *string `json:"tg_label"`
TgAlphaTag *string `json:"tg_alpha_tag"` TgAlphaTag *string `json:"tg_alpha_tag"`
TgGroup *string `json:"tg_group"` TgGroup *string `json:"tg_group"`
Source int `json:"source"` Source int `json:"source"`
Transcript *string `json:"transcript"` Transcript *string `json:"transcript"`
} }
type Incident struct { type Incident struct {

20
pkg/gordio/sinks/alerting/alerting.go
View file

@ -16,9 +16,9 @@ import (
) )
const ( const (
StorageLookbackDays = 4 StorageLookbackDays = 7
HalfLife = 30 * time.Minute HalfLife = 30 * time.Minute
RecentDuration = 12*time.Hour RecentDuration = 2*time.Hour
ScoreThreshold = -1 ScoreThreshold = -1
CountThreshold = 1 CountThreshold = 1
) )
@ -57,7 +57,7 @@ func newTimeSeries(id cl.Talkgroup) trending.TimeSeries {
ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities( ts, _ := timeseries.NewTimeSeries(timeseries.WithGranularities(
[]timeseries.Granularity{ []timeseries.Granularity{
{Granularity: time.Second, Count: 60}, {Granularity: time.Second, Count: 60},
{Granularity: time.Minute, Count: 60}, {Granularity: time.Minute, Count: 10},
{Granularity: time.Hour, Count: 24}, {Granularity: time.Hour, Count: 24},
{Granularity: time.Hour * 24, Count: StorageLookbackDays}, {Granularity: time.Hour * 24, Count: StorageLookbackDays},
}, },
@ -67,9 +67,8 @@ func newTimeSeries(id cl.Talkgroup) trending.TimeSeries {
func (as *AlertSink) startBackfill(ctx context.Context) { func (as *AlertSink) startBackfill(ctx context.Context) {
now := time.Now() now := time.Now()
cl := &myClock{-18*time.Hour} cl := &myClock{-24*StorageLookbackDays*time.Hour}
timeseries.DefaultClock = cl since := now.Add(StorageLookbackDays * -24 * time.Hour)
since := now.Add(StorageLookbackDays * -24 * 3 * time.Hour)
log.Debug().Time("since", since).Msg("starting stats backfill") log.Debug().Time("since", since).Msg("starting stats backfill")
count, err := as.backfill(ctx, since) count, err := as.backfill(ctx, since)
if err != nil { if err != nil {
@ -77,8 +76,9 @@ func (as *AlertSink) startBackfill(ctx context.Context) {
return return
} }
log.Debug().Int("count", count).Str("in", time.Now().Sub(now).String()).Int("len", as.scorer.Score().Len()).Msg("backfill finished") log.Debug().Int("count", count).Str("in", time.Now().Sub(now).String()).Int("len", as.scorer.Score().Len()).Msg("backfill finished")
timeseries.DefaultClock = cl
for { for {
fmt.Printf("offs: %s\n", cl.offset.String()) fmt.Printf("offs: %s (%s)\n", cl.offset.String(), cl.Now().String())
as.printScores(ctx) as.printScores(ctx)
cl.offset += time.Minute*5 cl.offset += time.Minute*5
if cl.offset == time.Minute*5 { if cl.offset == time.Minute*5 {
@ -92,11 +92,11 @@ func (as *AlertSink) printScores(ctx context.Context) {
as.Lock() as.Lock()
defer as.Unlock() defer as.Unlock()
scores := as.scorer.Score() scores := as.scorer.Score()
fmt.Printf("score len is %d\n", scores.Len()) //fmt.Printf("score len is %d\n", scores.Len())
//const scoreMult = 1000000000 //const scoreMult = 1000000000
const scoreMult = 1 const scoreMult = 1
for _, s := range scores { for _, s := range scores {
if s.ID.Talkgroup != 1616 && s.ID.Talkgroup != 1617 { if s.ID.Talkgroup != 1185 {
continue continue
} }
tg, _ := db.GetTalkgroup(ctx, int(s.ID.System), int(s.ID.Talkgroup)) tg, _ := db.GetTalkgroup(ctx, int(s.ID.System), int(s.ID.Talkgroup))
@ -111,7 +111,7 @@ func (as *AlertSink) printScores(ctx context.Context) {
func (as *AlertSink) backfill(ctx context.Context, since time.Time) (count int, err error) { func (as *AlertSink) backfill(ctx context.Context, since time.Time) (count int, err error) {
db := database.FromCtx(ctx) db := database.FromCtx(ctx)
const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2` const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2 AND talkgroup = 1185 `
rows, err := db.Query(ctx, backfillStatsQuery, since, timeseries.DefaultClock.Now()) rows, err := db.Query(ctx, backfillStatsQuery, since, timeseries.DefaultClock.Now())
if err != nil { if err != nil {

4
pkg/gordio/sinks/database.go
View file

@ -8,6 +8,7 @@ import (
"dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/gordio/database" "dynatron.me/x/stillbox/pkg/gordio/database"
"github.com/jackc/pgx/v5/pgtype"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
@ -24,6 +25,7 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
log.Debug().Str("call", call.String()).Msg("received dontStore call") log.Debug().Str("call", call.String()).Msg("received dontStore call")
return nil return nil
} }
dbCall, err := s.db.AddCall(ctx, s.toAddCallParams(call)) dbCall, err := s.db.AddCall(ctx, s.toAddCallParams(call))
if err != nil { if err != nil {
return fmt.Errorf("add call: %w", err) return fmt.Errorf("add call: %w", err)
@ -43,7 +45,7 @@ func (s *DatabaseSink) toAddCallParams(call *calls.Call) database.AddCallParams
Submitter: call.Submitter.Int32Ptr(), Submitter: call.Submitter.Int32Ptr(),
System: call.System, System: call.System,
Talkgroup: call.Talkgroup, Talkgroup: call.Talkgroup,
CallDate: call.DateTime, CallDate: pgtype.Timestamptz{Time: call.DateTime, Valid: true},
AudioName: common.PtrOrNull(call.AudioName), AudioName: common.PtrOrNull(call.AudioName),
AudioBlob: call.Audio, AudioBlob: call.Audio,
AudioType: common.PtrOrNull(call.AudioType), AudioType: common.PtrOrNull(call.AudioType),

3
sql/postgres/migrations/001_initial.up.sql
View file

@ -91,7 +91,7 @@ CREATE TABLE IF NOT EXISTS calls(
submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL,
system INTEGER NOT NULL, system INTEGER NOT NULL,
talkgroup INTEGER NOT NULL, talkgroup INTEGER NOT NULL,
call_date TIMESTAMP NOT NULL, call_date TIMESTAMPTZ NOT NULL,
audio_name TEXT, audio_name TEXT,
audio_blob BYTEA, audio_blob BYTEA,
duration INTEGER, duration INTEGER,
@ -107,7 +107,6 @@ CREATE TABLE IF NOT EXISTS calls(
transcript TEXT transcript TEXT
); );
CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls
FOR EACH ROW EXECUTE FUNCTION learn_talkgroup(); FOR EACH ROW EXECUTE FUNCTION learn_talkgroup();