amigan/stillbox
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

flattenLearned #44

Merged
amigan merged 7 commits from flattenLearned into trunk 2024-11-21 08:06:12 -05:00
Conversation 0 Commits 7 Files changed 25 +578 -329
25 changed files with 578 additions and 329 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/alerting/alert/alert.go
View file

@ -8,6 +8,7 @@ import (
"dynatron.me/x/stillbox/internal/trending"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/jackc/pgx/v5/pgtype"
)
@ -44,7 +45,8 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
}
// Make creates an alert for later rendering or storage.
func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
func Make(ctx context.Context, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
store := tgstore.FromCtx(ctx)
d := Alert{
Score: score,
Timestamp: time.Now(),

11
pkg/alerting/alerting.go
View file

@ -15,7 +15,8 @@ import (
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/notify"
"dynatron.me/x/stillbox/pkg/sinks"
talkgroups "dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"dynatron.me/x/stillbox/internal/timeseries"
"dynatron.me/x/stillbox/internal/trending"
@ -51,7 +52,7 @@ type alerter struct {
alertCache map[talkgroups.ID]alert.Alert
renotify time.Duration
notifier notify.Notifier
tgCache talkgroups.Store
tgCache tgstore.Store
}
type offsetClock time.Duration
@ -86,7 +87,7 @@ func WithNotifier(n notify.Notifier) AlertOption {
}
// New creates a new Alerter using the provided configuration.
func New(cfg config.Alerting, tgCache talkgroups.Store, opts ...AlertOption) Alerter {
func New(cfg config.Alerting, tgCache tgstore.Store, opts ...AlertOption) Alerter {
if !cfg.Enable {
return &noopAlerter{}
}
@ -169,7 +170,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
if s.Score > as.cfg.AlertThreshold || testMode {
if old, inCache := as.alertCache[s.ID]; !inCache || now.Sub(old.Timestamp) > as.renotify {
s.Score *= as.tgCache.Weight(ctx, s.ID, now)
a, err := alert.Make(ctx, as.tgCache, s, origScore)
a, err := alert.Make(ctx, s, origScore)
if err != nil {
return nil, fmt.Errorf("makeAlert: %w", err)
}
@ -202,7 +203,7 @@ func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ridx := rand.Intn(len(as.scores))
a, err := alert.Make(ctx, talkgroups.StoreFrom(ctx), as.scores[ridx], 1.0)
a, err := alert.Make(ctx, as.scores[ridx], 1.0)
if err != nil {
log.Error().Err(err).Msg("test notify make alert fail")
http.Error(w, err.Error(), http.StatusInternalServerError)

3
pkg/alerting/simulate.go
View file

@ -13,6 +13,7 @@ import (
"dynatron.me/x/stillbox/internal/trending"
"dynatron.me/x/stillbox/pkg/config"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/rs/zerolog/log"
)
@ -59,7 +60,7 @@ func (s *Simulation) stepClock(t time.Time) {
// Simulate begins the simulation using the DB handle from ctx. It returns final scores.
func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.ID], error) {
now := time.Now()
tgc := talkgroups.NewCache()
tgc := tgstore.NewCache()
s.Enable = true
s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter)

17
pkg/auth/jwt.go
View file

@ -40,6 +40,23 @@ type jwtAuth interface {
type claims map[string]interface{}
func UIDFrom(ctx context.Context) *int32 {
tok, _, err := jwtauth.FromContext(ctx)
if err != nil {
return nil
}
uidStr := tok.Subject()
uidInt, err := strconv.Atoi(uidStr)
if err != nil {
return nil
}
uid := int32(uidInt)
return &uid
}
func (a *Auth) Authenticated(r *http.Request) (claims, bool) {
// TODO: check IP against ACL, or conf.Public, and against map of routes
tok, cl, err := jwtauth.FromContext(r.Context())

17
pkg/calls/call.go
View file

@ -1,13 +1,11 @@
package calls
import (
"context"
"fmt"
"time"
"dynatron.me/x/stillbox/internal/audio"
"dynatron.me/x/stillbox/pkg/auth"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/pb"
"dynatron.me/x/stillbox/pkg/talkgroups"
@ -113,21 +111,6 @@ func (c *Call) ToPB() *pb.Call {
}
}
func (c *Call) LearnTG(ctx context.Context, db database.Store) (learnedId int, err error) {
err = db.AddTalkgroupWithLearnedFlag(ctx, int32(c.System), int32(c.Talkgroup))
if err != nil {
return 0, fmt.Errorf("addTalkgroupWithLearnedFlag: %w", err)
}
return db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
SystemID: c.System,
TGID: c.Talkgroup,
Name: c.TalkgroupLabel,
AlphaTag: c.TGAlphaTag,
TGGroup: c.TalkgroupGroup,
})
}
func (c *Call) computeLength() (err error) {
var td time.Duration

80
pkg/database/batch.go
View file

@ -18,6 +18,83 @@ var (
ErrBatchAlreadyClosed = errors.New("batch already closed")
)
const storeTGVersion = `-- name: StoreTGVersion :batchexec
INSERT INTO talkgroup_versions(time, created_by,
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned
) SELECT NOW(), $1,
tg.system_id,
tg.tgid,
tg.name,
tg.alpha_tag,
tg.tg_group,
tg.frequency,
tg.metadata,
tg.tags,
tg.alert,
tg.alert_config,
tg.weight,
tg.learned
FROM talkgroups tg WHERE tg.system_id = $2 AND tg.tgid = $3
`
type StoreTGVersionBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type StoreTGVersionParams struct {
Submitter *int32 `json:"submitter"`
SystemID int32 `json:"system_id"`
TGID int32 `json:"tgid"`
}
func (q *Queries) StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.Submitter,
a.SystemID,
a.TGID,
}
batch.Queue(storeTGVersion, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &StoreTGVersionBatchResults{br, len(arg), false}
}
func (b *StoreTGVersionBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *StoreTGVersionBatchResults) Close() error {
b.closed = true
return b.br.Close()
}
const upsertTalkgroup = `-- name: UpsertTalkgroup :batchone
INSERT INTO talkgroups AS tg (
system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
@ -47,7 +124,7 @@ SET
alert_config = COALESCE($10, tg.alert_config),
weight = COALESCE($11, tg.weight),
learned = COALESCE($12, tg.learned)
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
`
type UpsertTalkgroupBatchResults struct {
@ -119,6 +196,7 @@ func (b *UpsertTalkgroupBatchResults) QueryRow(f func(int, Talkgroup, error)) {
&i.AlertConfig,
&i.Weight,
&i.Learned,
&i.Ignored,
)
if f != nil {
f(t, i, err)

168
pkg/database/mocks/Store.go
View file

@ -123,22 +123,22 @@ func (_c *Store_AddCall_Call) RunAndReturn(run func(context.Context, database.Ad
}
// AddLearnedTalkgroup provides a mock function with given fields: ctx, arg
func (_m *Store) AddLearnedTalkgroup(ctx context.Context, arg database.AddLearnedTalkgroupParams) (int, error) {
func (_m *Store) AddLearnedTalkgroup(ctx context.Context, arg database.AddLearnedTalkgroupParams) (database.Talkgroup, error) {
ret := _m.Called(ctx, arg)
if len(ret) == 0 {
panic("no return value specified for AddLearnedTalkgroup")
}
var r0 int
var r0 database.Talkgroup
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) (int, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) (database.Talkgroup, error)); ok {
return rf(ctx, arg)
}
if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) int); ok {
if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) database.Talkgroup); ok {
r0 = rf(ctx, arg)
} else {
r0 = ret.Get(0).(int)
r0 = ret.Get(0).(database.Talkgroup)
}
if rf, ok := ret.Get(1).(func(context.Context, database.AddLearnedTalkgroupParams) error); ok {
@ -169,60 +169,12 @@ func (_c *Store_AddLearnedTalkgroup_Call) Run(run func(ctx context.Context, arg
return _c
}
func (_c *Store_AddLearnedTalkgroup_Call) Return(_a0 int, _a1 error) *Store_AddLearnedTalkgroup_Call {
func (_c *Store_AddLearnedTalkgroup_Call) Return(_a0 database.Talkgroup, _a1 error) *Store_AddLearnedTalkgroup_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *Store_AddLearnedTalkgroup_Call) RunAndReturn(run func(context.Context, database.AddLearnedTalkgroupParams) (int, error)) *Store_AddLearnedTalkgroup_Call {
_c.Call.Return(run)
return _c
}
// AddTalkgroupWithLearnedFlag provides a mock function with given fields: ctx, systemID, tGID
func (_m *Store) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
ret := _m.Called(ctx, systemID, tGID)
if len(ret) == 0 {
panic("no return value specified for AddTalkgroupWithLearnedFlag")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int32, int32) error); ok {
r0 = rf(ctx, systemID, tGID)
} else {
r0 = ret.Error(0)
}
return r0
}
// Store_AddTalkgroupWithLearnedFlag_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddTalkgroupWithLearnedFlag'
type Store_AddTalkgroupWithLearnedFlag_Call struct {
*mock.Call
}
// AddTalkgroupWithLearnedFlag is a helper method to define mock.On call
// - ctx context.Context
// - systemID int32
// - tGID int32
func (_e *Store_Expecter) AddTalkgroupWithLearnedFlag(ctx interface{}, systemID interface{}, tGID interface{}) *Store_AddTalkgroupWithLearnedFlag_Call {
return &Store_AddTalkgroupWithLearnedFlag_Call{Call: _e.mock.On("AddTalkgroupWithLearnedFlag", ctx, systemID, tGID)}
}
func (_c *Store_AddTalkgroupWithLearnedFlag_Call) Run(run func(ctx context.Context, systemID int32, tGID int32)) *Store_AddTalkgroupWithLearnedFlag_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int32), args[2].(int32))
})
return _c
}
func (_c *Store_AddTalkgroupWithLearnedFlag_Call) Return(_a0 error) *Store_AddTalkgroupWithLearnedFlag_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *Store_AddTalkgroupWithLearnedFlag_Call) RunAndReturn(run func(context.Context, int32, int32) error) *Store_AddTalkgroupWithLearnedFlag_Call {
func (_c *Store_AddLearnedTalkgroup_Call) RunAndReturn(run func(context.Context, database.AddLearnedTalkgroupParams) (database.Talkgroup, error)) *Store_AddLearnedTalkgroup_Call {
_c.Call.Return(run)
return _c
}
@ -1569,6 +1521,63 @@ func (_c *Store_InTx_Call) RunAndReturn(run func(context.Context, func(database.
return _c
}
// RestoreTalkgroupVersion provides a mock function with given fields: ctx, versionIds
func (_m *Store) RestoreTalkgroupVersion(ctx context.Context, versionIds int) (database.Talkgroup, error) {
ret := _m.Called(ctx, versionIds)
if len(ret) == 0 {
panic("no return value specified for RestoreTalkgroupVersion")
}
var r0 database.Talkgroup
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int) (database.Talkgroup, error)); ok {
return rf(ctx, versionIds)
}
if rf, ok := ret.Get(0).(func(context.Context, int) database.Talkgroup); ok {
r0 = rf(ctx, versionIds)
} else {
r0 = ret.Get(0).(database.Talkgroup)
}
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, versionIds)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Store_RestoreTalkgroupVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RestoreTalkgroupVersion'
type Store_RestoreTalkgroupVersion_Call struct {
*mock.Call
}
// RestoreTalkgroupVersion is a helper method to define mock.On call
// - ctx context.Context
// - versionIds int
func (_e *Store_Expecter) RestoreTalkgroupVersion(ctx interface{}, versionIds interface{}) *Store_RestoreTalkgroupVersion_Call {
return &Store_RestoreTalkgroupVersion_Call{Call: _e.mock.On("RestoreTalkgroupVersion", ctx, versionIds)}
}
func (_c *Store_RestoreTalkgroupVersion_Call) Run(run func(ctx context.Context, versionIds int)) *Store_RestoreTalkgroupVersion_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int))
})
return _c
}
func (_c *Store_RestoreTalkgroupVersion_Call) Return(_a0 database.Talkgroup, _a1 error) *Store_RestoreTalkgroupVersion_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *Store_RestoreTalkgroupVersion_Call) RunAndReturn(run func(context.Context, int) (database.Talkgroup, error)) *Store_RestoreTalkgroupVersion_Call {
_c.Call.Return(run)
return _c
}
// SetCallTranscript provides a mock function with given fields: ctx, iD, transcript
func (_m *Store) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error {
ret := _m.Called(ctx, iD, transcript)
@ -1666,6 +1675,55 @@ func (_c *Store_SetTalkgroupTags_Call) RunAndReturn(run func(context.Context, []
return _c
}
// StoreTGVersion provides a mock function with given fields: ctx, arg
func (_m *Store) StoreTGVersion(ctx context.Context, arg []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults {
ret := _m.Called(ctx, arg)
if len(ret) == 0 {
panic("no return value specified for StoreTGVersion")
}
var r0 *database.StoreTGVersionBatchResults
if rf, ok := ret.Get(0).(func(context.Context, []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults); ok {
r0 = rf(ctx, arg)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*database.StoreTGVersionBatchResults)
}
}
return r0
}
// Store_StoreTGVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StoreTGVersion'
type Store_StoreTGVersion_Call struct {
*mock.Call
}
// StoreTGVersion is a helper method to define mock.On call
// - ctx context.Context
// - arg []database.StoreTGVersionParams
func (_e *Store_Expecter) StoreTGVersion(ctx interface{}, arg interface{}) *Store_StoreTGVersion_Call {
return &Store_StoreTGVersion_Call{Call: _e.mock.On("StoreTGVersion", ctx, arg)}
}
func (_c *Store_StoreTGVersion_Call) Run(run func(ctx context.Context, arg []database.StoreTGVersionParams)) *Store_StoreTGVersion_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].([]database.StoreTGVersionParams))
})
return _c
}
func (_c *Store_StoreTGVersion_Call) Return(_a0 *database.StoreTGVersionBatchResults) *Store_StoreTGVersion_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *Store_StoreTGVersion_Call) RunAndReturn(run func(context.Context, []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults) *Store_StoreTGVersion_Call {
_c.Call.Return(run)
return _c
}
// UpdatePassword provides a mock function with given fields: ctx, username, password
func (_m *Store) UpdatePassword(ctx context.Context, username string, password string) error {
ret := _m.Called(ctx, username, password)

18
pkg/database/models.go
View file

@ -96,15 +96,25 @@ type Talkgroup struct {
AlertConfig rules.AlertRules `json:"alert_config"`
Weight float32 `json:"weight"`
Learned bool `json:"learned"`
Ignored bool `json:"ignored"`
}
type TalkgroupsLearned struct {
type TalkgroupVersion struct {
ID int `json:"id"`
SystemID int `json:"system_id"`
TGID int `json:"tgid"`
Name string `json:"name"`
Time pgtype.Timestamptz `json:"time"`
CreatedBy *int32 `json:"created_by"`
SystemID *int32 `json:"system_id"`
TGID *int32 `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TGGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata []byte `json:"metadata"`
Tags []string `json:"tags"`
Alert *bool `json:"alert"`
AlertConfig []byte `json:"alert_config"`
Weight *float32 `json:"weight"`
Learned *bool `json:"learned"`
Ignored *bool `json:"ignored"`
}

5
pkg/database/querier.go
View file

@ -14,8 +14,7 @@ import (
type Querier interface {
AddAlert(ctx context.Context, arg AddAlertParams) error
AddCall(ctx context.Context, arg AddCallParams) error
AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error)
AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error
AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error)
CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
DeleteAPIKey(ctx context.Context, apiKey string) error
@ -35,8 +34,10 @@ type Querier interface {
GetUserByUID(ctx context.Context, id int) (User, error)
GetUserByUsername(ctx context.Context, username string) (User, error)
GetUsers(ctx context.Context) ([]User, error)
RestoreTalkgroupVersion(ctx context.Context, versionIds int) (Talkgroup, error)
SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error
StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults
UpdatePassword(ctx context.Context, username string, password string) error
UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
UpsertTalkgroup(ctx context.Context, arg []UpsertTalkgroupParams) *UpsertTalkgroupBatchResults

18
pkg/database/talkgroups.go
View file

@ -41,20 +41,11 @@ func (t *TGTuples) Append(sys, tg uint32) {
// Below queries are here because sqlc refuses to parse unnest(x, y)
const getTalkgroupsWithLearnedBySysTGID = `SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned, tg.ignored
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);`
WHERE tg.learned IS NOT TRUE;`
type GetTalkgroupsRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
@ -86,6 +77,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
&i.System.ID,
&i.System.Name,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
); err != nil {
return nil, err
}
@ -97,7 +89,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
return items, nil
}
const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
WHERE tg.learned IS NOT TRUE;`
@ -124,6 +116,8 @@ func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]G
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
&i.System.ID,
&i.System.Name,
); err != nil {

167
pkg/database/talkgroups.sql.go
View file

@ -13,30 +13,32 @@ import (
)
const addLearnedTalkgroup = `-- name: AddLearnedTalkgroup :one
INSERT INTO talkgroups_learned(
INSERT INTO talkgroups(
system_id,
tgid,
learned,
name,
alpha_tag,
tg_group
) VALUES (
$1,
$2,
TRUE,
$3,
$4,
$5
) RETURNING id
) RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
`
type AddLearnedTalkgroupParams struct {
SystemID int `json:"system_id"`
TGID int `json:"tgid"`
SystemID int32 `json:"system_id"`
TGID int32 `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TGGroup *string `json:"tg_group"`
}
func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error) {
func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error) {
row := q.db.QueryRow(ctx, addLearnedTalkgroup,
arg.SystemID,
arg.TGID,
@ -44,26 +46,24 @@ func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgro
arg.AlphaTag,
arg.TGGroup,
)
var id int
err := row.Scan(&id)
return id, err
}
const addTalkgroupWithLearnedFlag = `-- name: AddTalkgroupWithLearnedFlag :exec
INSERT INTO talkgroups (
system_id,
tgid,
learned
) VALUES(
$1,
$2,
TRUE
var i Talkgroup
err := row.Scan(
&i.ID,
&i.SystemID,
&i.TGID,
&i.Name,
&i.AlphaTag,
&i.TGGroup,
&i.Frequency,
&i.Metadata,
&i.Tags,
&i.Alert,
&i.AlertConfig,
&i.Weight,
&i.Learned,
&i.Ignored,
)
`
func (q *Queries) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
_, err := q.db.Exec(ctx, addTalkgroupWithLearnedFlag, systemID, tGID)
return err
return i, err
}
const getSystemName = `-- name: GetSystemName :one
@ -78,7 +78,7 @@ func (q *Queries) GetSystemName(ctx context.Context, systemID int) (string, erro
}
const getTalkgroup = `-- name: GetTalkgroup :one
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
WHERE (system_id, tgid) = ($1, $2)
`
@ -103,6 +103,7 @@ func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tGID int32)
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
)
return i, err
}
@ -153,19 +154,10 @@ func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tGID int
const getTalkgroupWithLearned = `-- name: GetTalkgroupWithLearned :one
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
WHERE (tg.system_id, tg.tgid) = ($1, $2)
`
type GetTalkgroupWithLearnedRow struct {
@ -190,6 +182,7 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
&i.System.ID,
&i.System.Name,
)
@ -197,7 +190,7 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
}
const getTalkgroupsWithAllTags = `-- name: GetTalkgroupsWithAllTags :many
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
WHERE tags && ARRAY[$1]
`
@ -228,6 +221,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
); err != nil {
return nil, err
}
@ -240,7 +234,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
}
const getTalkgroupsWithAnyTags = `-- name: GetTalkgroupsWithAnyTags :many
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
WHERE tags @> ARRAY[$1]
`
@ -271,6 +265,7 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
); err != nil {
return nil, err
}
@ -284,18 +279,9 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
const getTalkgroupsWithLearned = `-- name: GetTalkgroupsWithLearned :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE
`
@ -327,6 +313,7 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
&i.System.ID,
&i.System.Name,
); err != nil {
@ -342,19 +329,10 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
const getTalkgroupsWithLearnedBySystem = `-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
WHERE tg.system_id = $1
`
type GetTalkgroupsWithLearnedBySystemRow struct {
@ -385,6 +363,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.Talkgroup.Ignored,
&i.System.ID,
&i.System.Name,
); err != nil {
@ -398,6 +377,73 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
return items, nil
}
const restoreTalkgroupVersion = `-- name: RestoreTalkgroupVersion :one
INSERT INTO talkgroups(
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned,
ignored
)
SELECT
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned,
ignored
FROM talkgroup_versions tgv ON CONFLICT (system_id, tgid) DO UPDATE SET
name = excluded.name,
alpha_tag = excluded.alpha_tag,
tg_group = excluded.tg_group,
metadata = excluded.metadata,
tags = excluded.tags,
alert = excluded.alert,
alert_config = excluded.alert_config,
weight = excluded.weight,
learned = excluded.learner,
ignored = excluded.ignored
WHERE tgv.id = ANY($1)
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
`
func (q *Queries) RestoreTalkgroupVersion(ctx context.Context, versionIds int) (Talkgroup, error) {
row := q.db.QueryRow(ctx, restoreTalkgroupVersion, versionIds)
var i Talkgroup
err := row.Scan(
&i.ID,
&i.SystemID,
&i.TGID,
&i.Name,
&i.AlphaTag,
&i.TGGroup,
&i.Frequency,
&i.Metadata,
&i.Tags,
&i.Alert,
&i.AlertConfig,
&i.Weight,
&i.Learned,
&i.Ignored,
)
return i, err
}
const setTalkgroupTags = `-- name: SetTalkgroupTags :exec
UPDATE talkgroups SET tags = $1
WHERE system_id = $2 AND tgid = $3
@ -422,7 +468,7 @@ SET
weight = COALESCE($9, weight),
learned = COALESCE($10, learned)
WHERE id = $11 OR (system_id = $12 AND tgid = $13)
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
`
type UpdateTalkgroupParams struct {
@ -472,6 +518,7 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
&i.AlertConfig,
&i.Weight,
&i.Learned,
&i.Ignored,
)
return i, err
}

64
pkg/database/talkgroups.sql_test.go
View file

@ -1,64 +0,0 @@
package database
import (
"testing"
"github.com/stretchr/testify/assert"
)
const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
`
const getTalkgroupsWithLearnedBySystemTest = `-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
`
const getTalkgroupsWithLearnedTest = `-- name: GetTalkgroupsWithLearned :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE
`
func TestQueryColumnsMatch(t *testing.T) {
assert.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
assert.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
assert.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
}

5
pkg/nexus/commands.go
View file

@ -6,6 +6,7 @@ import (
"dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/pb"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/rs/zerolog/log"
"google.golang.org/protobuf/types/known/structpb"
@ -59,9 +60,9 @@ func (c *client) SendError(cmd *pb.Command, err error) {
}
func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
tgi, err := talkgroups.StoreFrom(ctx).TG(ctx, talkgroups.TG(tg.System, tg.Talkgroup))
tgi, err := tgstore.FromCtx(ctx).TG(ctx, talkgroups.TG(tg.System, tg.Talkgroup))
if err != nil {
if err != talkgroups.ErrNotFound {
if err != tgstore.ErrNotFound {
log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("get talkgroup fail")
}
return err

15
pkg/rest/api.go
View file

@ -4,7 +4,7 @@ import (
"errors"
"net/http"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
@ -67,6 +67,15 @@ func recordNotFound(err error) render.Renderer {
}
}
func errTextNotFound(err error) render.Renderer {
return &errResponse{
Err: err,
Code: http.StatusNotFound,
Error: "Record not found: " + err.Error(),
}
}
func internalError(err error) render.Renderer {
return &errResponse{
Err: err,
@ -78,8 +87,8 @@ func internalError(err error) render.Renderer {
type errResponder func(error) render.Renderer
var statusMapping = map[error]errResponder{
talkgroups.ErrNoSuchSystem: recordNotFound,
talkgroups.ErrNotFound: recordNotFound,
tgstore.ErrNoSuchSystem: errTextNotFound,
tgstore.ErrNotFound: errTextNotFound,
pgx.ErrNoRows: recordNotFound,
}

9
pkg/rest/talkgroups.go
View file

@ -6,6 +6,7 @@ import (
"dynatron.me/x/stillbox/internal/forms"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"dynatron.me/x/stillbox/pkg/talkgroups/importer"
"github.com/go-chi/chi/v5"
@ -53,7 +54,7 @@ func (t tgParams) ToID() talkgroups.ID {
func (tga *talkgroupAPI) get(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
tgs := talkgroups.StoreFrom(ctx)
tgs := tgstore.FromCtx(ctx)
var p tgParams
@ -91,7 +92,7 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
}
ctx := r.Context()
tgs := talkgroups.StoreFrom(ctx)
tgs := tgstore.FromCtx(ctx)
input := database.UpdateTalkgroupParams{}
@ -137,12 +138,12 @@ func (tga *talkgroupAPI) putTalkgroups(w http.ResponseWriter, r *http.Request) {
}
if id.System == nil { // don't think this would ever happen
wErr(w, r, badRequest(talkgroups.ErrNoSuchSystem))
wErr(w, r, badRequest(tgstore.ErrNoSuchSystem))
return
}
ctx := r.Context()
tgs := talkgroups.StoreFrom(ctx)
tgs := tgstore.FromCtx(ctx)
var input []database.UpsertTalkgroupParams

4
pkg/server/routes.go
View file

@ -9,7 +9,7 @@ import (
"dynatron.me/x/stillbox/internal/version"
"dynatron.me/x/stillbox/pkg/config"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/httprate"
@ -28,7 +28,7 @@ func (s *Server) setupRoutes() {
r := s.r
r.Use(middleware.WithValue(database.DBCtxKey, s.db))
r.Use(middleware.WithValue(talkgroups.StoreCtxKey, s.tgs))
r.Use(middleware.WithValue(tgstore.StoreCtxKey, s.tgs))
s.installPprof()

11
pkg/server/server.go
View file

@ -15,7 +15,8 @@ import (
"dynatron.me/x/stillbox/pkg/rest"
"dynatron.me/x/stillbox/pkg/sinks"
"dynatron.me/x/stillbox/pkg/sources"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
@ -37,7 +38,7 @@ type Server struct {
alerter alerting.Alerter
notifier notify.Notifier
hup chan os.Signal
tgs talkgroups.Store
tgs tgstore.Store
rest rest.API
}
@ -61,7 +62,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
return nil, err
}
tgCache := talkgroups.NewCache()
tgCache := tgstore.NewCache()
api := rest.New()
srv := &Server{
@ -78,7 +79,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
rest: api,
}
srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db), true)
srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db, tgCache), true)
srv.sinks.Register("nexus", sinks.NewNexusSink(srv.nex), false)
if srv.alerter.Enabled() {
@ -117,7 +118,7 @@ func (s *Server) Go(ctx context.Context) error {
s.installHupHandler()
ctx = database.CtxWithDB(ctx, s.db)
ctx = talkgroups.CtxWithStore(ctx, s.tgs)
ctx = tgstore.CtxWithStore(ctx, s.tgs)
httpSrv := &http.Server{
Addr: s.conf.Listen,

12
pkg/sinks/database.go
View file

@ -7,6 +7,7 @@ import (
"dynatron.me/x/stillbox/internal/common"
"dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
@ -15,10 +16,11 @@ import (
type DatabaseSink struct {
db database.Store
tgs tgstore.Store
}
func NewDatabaseSink(store database.Store) *DatabaseSink {
return &DatabaseSink{store}
func NewDatabaseSink(store database.Store, tgs tgstore.Store) *DatabaseSink {
return &DatabaseSink{store, tgs}
}
func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
@ -43,14 +45,14 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
if err != nil && database.IsTGConstraintViolation(err) {
return s.db.InTx(ctx, func(tx database.Store) error {
_, err := call.LearnTG(ctx, tx)
_, err := s.tgs.LearnTG(ctx, call)
if err != nil {
return fmt.Errorf("add call: learn tg: %w", err)
return fmt.Errorf("learn tg: %w", err)
}
err = tx.AddCall(ctx, params)
if err != nil {
return fmt.Errorf("add call: retry: %w", err)
return fmt.Errorf("learn tg retry: %w", err)
}
return nil

2
pkg/sources/http.go
View file

@ -131,7 +131,7 @@ func (h *RdioHTTP) routeCallUpload(w http.ResponseWriter, r *http.Request) {
return
}
log.Info().Int("system", cur.System).Int("tgid", cur.Talkgroup).Msg("ingested")
log.Info().Int("system", cur.System).Int("tgid", cur.Talkgroup).Str("duration", call.Duration.Duration().String()).Msg("ingested")
written, err := w.Write([]byte("Call imported successfully."))
if err != nil {

5
pkg/talkgroups/importer/import.go
View file

@ -13,6 +13,7 @@ import (
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
)
type ImportSource string
@ -66,9 +67,9 @@ var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`
func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error) {
sc := bufio.NewScanner(r)
tgs := make([]talkgroups.Talkgroup, 0, 8)
sysn, has := talkgroups.StoreFrom(ctx).SystemName(ctx, sys)
sysn, has := tgstore.FromCtx(ctx).SystemName(ctx, sys)
if !has {
return nil, talkgroups.ErrNoSuchSystem
return nil, tgstore.ErrNoSuchSystem
}
var groupName string

3
pkg/talkgroups/importer/import_test.go
View file

@ -16,6 +16,7 @@ import (
"dynatron.me/x/stillbox/pkg/database/mocks"
"dynatron.me/x/stillbox/pkg/talkgroups"
"dynatron.me/x/stillbox/pkg/talkgroups/importer"
"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
)
func getFixture(fixture string) []byte {
@ -62,7 +63,7 @@ func TestImport(t *testing.T) {
dbMock.EXPECT().GetSystemName(mock.AnythingOfType("*context.valueCtx"), tc.sysID).Return(tc.sysName, nil)
}
ctx := database.CtxWithDB(context.Background(), dbMock)
ctx = talkgroups.CtxWithStore(ctx, talkgroups.NewCache())
ctx = tgstore.CtxWithStore(ctx, tgstore.NewCache())
ij := &importer.ImportJob{
Type: importer.ImportSource(tc.impType),
SystemID: tc.sysID,

2
pkg/talkgroups/importer/testdata/riscon.json vendored
View file

File diff suppressed because one or more lines are too long

121
pkg/talkgroups/store.go → pkg/talkgroups/tgstore/store.go
View file

@ -1,4 +1,4 @@
package talkgroups
package tgstore
import (
"context"
@ -8,14 +8,17 @@ import (
"time"
"dynatron.me/x/stillbox/internal/common"
"dynatron.me/x/stillbox/pkg/auth"
"dynatron.me/x/stillbox/pkg/config"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/calls"
tgsp "dynatron.me/x/stillbox/pkg/talkgroups"
"github.com/jackc/pgx/v5"
"github.com/rs/zerolog/log"
)
type tgMap map[ID]*Talkgroup
type tgMap map[tgsp.ID]*tgsp.Talkgroup
var (
ErrNotFound = errors.New("talkgroup not found")
@ -24,25 +27,28 @@ var (
type Store interface {
// UpdateTG updates a talkgroup record.
UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*Talkgroup, error)
UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*tgsp.Talkgroup, error)
// UpsertTGs upserts a slice of talkgroups.
UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*Talkgroup, error)
UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*tgsp.Talkgroup, error)
// TG retrieves a Talkgroup from the Store.
TG(ctx context.Context, tg ID) (*Talkgroup, error)
TG(ctx context.Context, tg tgsp.ID) (*tgsp.Talkgroup, error)
// TGs retrieves many talkgroups from the Store.
TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error)
TGs(ctx context.Context, tgs tgsp.IDs) ([]*tgsp.Talkgroup, error)
// LearnTG learns the talkgroup from a Call.
LearnTG(ctx context.Context, call *calls.Call) (*tgsp.Talkgroup, error)
// SystemTGs retrieves all Talkgroups associated with a System.
SystemTGs(ctx context.Context, systemID int32) ([]*Talkgroup, error)
SystemTGs(ctx context.Context, systemID int32) ([]*tgsp.Talkgroup, error)
// SystemName retrieves a system name from the store. It returns the record and whether one was found.
SystemName(ctx context.Context, id int) (string, bool)
// Hint hints the Store that the provided talkgroups will be asked for.
Hint(ctx context.Context, tgs []ID) error
Hint(ctx context.Context, tgs []tgsp.ID) error
// Load loads the provided talkgroup ID tuples into the Store.
Load(ctx context.Context, tgs database.TGTuples) error
@ -51,7 +57,7 @@ type Store interface {
Invalidate()
// Weight returns the final weight of this talkgroup, including its static and rules-derived weight.
Weight(ctx context.Context, id ID, t time.Time) float64
Weight(ctx context.Context, id tgsp.ID, t time.Time) float64
// Hupper
HUP(*config.Config)
@ -65,7 +71,7 @@ func CtxWithStore(ctx context.Context, s Store) context.Context {
return context.WithValue(ctx, StoreCtxKey, s)
}
func StoreFrom(ctx context.Context) Store {
func FromCtx(ctx context.Context) Store {
s, ok := ctx.Value(StoreCtxKey).(Store)
if !ok {
return NewCache()
@ -101,7 +107,7 @@ func NewCache() Store {
return tgc
}
func (t *cache) Hint(ctx context.Context, tgs []ID) error {
func (t *cache) Hint(ctx context.Context, tgs []tgsp.ID) error {
t.RLock()
var toLoad database.TGTuples
if len(t.tgs) > len(tgs)/2 { // TODO: instrument this
@ -129,11 +135,11 @@ func (t *cache) Hint(ctx context.Context, tgs []ID) error {
return nil
}
func (t *cache) add(rec *Talkgroup) {
func (t *cache) add(rec *tgsp.Talkgroup) {
t.Lock()
defer t.Unlock()
tg := TG(rec.System.ID, rec.Talkgroup.TGID)
tg := tgsp.TG(rec.System.ID, rec.Talkgroup.TGID)
t.tgs[tg] = rec
t.systems[int32(rec.System.ID)] = rec.System.Name
}
@ -146,15 +152,15 @@ type row interface {
GetLearned() bool
}
func rowToTalkgroup[T row](r T) *Talkgroup {
return &Talkgroup{
func rowToTalkgroup[T row](r T) *tgsp.Talkgroup {
return &tgsp.Talkgroup{
Talkgroup: r.GetTalkgroup(),
System: r.GetSystem(),
Learned: r.GetLearned(),
}
}
func addToRowList[T row](t *cache, r []*Talkgroup, tgRecords []T) []*Talkgroup {
func addToRowList[T row](t *cache, r []*tgsp.Talkgroup, tgRecords []T) []*tgsp.Talkgroup {
for _, rec := range tgRecords {
tg := rowToTalkgroup(rec)
t.add(tg)
@ -165,11 +171,11 @@ func addToRowList[T row](t *cache, r []*Talkgroup, tgRecords []T) []*Talkgroup {
return r
}
func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
r := make([]*Talkgroup, 0, len(tgs))
func (t *cache) TGs(ctx context.Context, tgs tgsp.IDs) ([]*tgsp.Talkgroup, error) {
r := make([]*tgsp.Talkgroup, 0, len(tgs))
var err error
if tgs != nil {
toGet := make(IDs, 0, len(tgs))
toGet := make(tgsp.IDs, 0, len(tgs))
t.RLock()
for _, id := range tgs {
rec, has := t.tgs[id]
@ -210,7 +216,7 @@ func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error {
return nil
}
func (t *cache) Weight(ctx context.Context, id ID, tm time.Time) float64 {
func (t *cache) Weight(ctx context.Context, id tgsp.ID, tm time.Time) float64 {
tg, err := t.TG(ctx, id)
if err != nil {
return 1.0
@ -223,17 +229,17 @@ func (t *cache) Weight(ctx context.Context, id ID, tm time.Time) float64 {
return float64(m)
}
func (t *cache) SystemTGs(ctx context.Context, systemID int32) ([]*Talkgroup, error) {
func (t *cache) SystemTGs(ctx context.Context, systemID int32) ([]*tgsp.Talkgroup, error) {
recs, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySystem(ctx, systemID)
if err != nil {
return nil, err
}
r := make([]*Talkgroup, 0, len(recs))
r := make([]*tgsp.Talkgroup, 0, len(recs))
return addToRowList(t, r, recs), nil
}
func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
func (t *cache) TG(ctx context.Context, tg tgsp.ID) (*tgsp.Talkgroup, error) {
t.RLock()
rec, has := t.tgs[tg]
t.RUnlock()
@ -278,7 +284,7 @@ func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool)
return n, has
}
func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*Talkgroup, error) {
func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*tgsp.Talkgroup, error) {
sysName, has := t.SystemName(ctx, int(*input.SystemID))
if !has {
return nil, ErrNoSuchSystem
@ -289,7 +295,7 @@ func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupPara
return nil, err
}
record := &Talkgroup{
record := &tgsp.Talkgroup{
Talkgroup: tg,
System: database.System{ID: int(tg.SystemID), Name: sysName},
}
@ -298,7 +304,40 @@ func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupPara
return record, nil
}
func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*Talkgroup, error) {
func (t *cache) LearnTG(ctx context.Context, c *calls.Call) (*tgsp.Talkgroup, error) {
db := database.FromCtx(ctx)
sys, has := t.SystemName(ctx, c.System)
if !has {
return nil, ErrNoSuchSystem
}
tgm, err := db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
SystemID: int32(c.System),
TGID: int32(c.Talkgroup),
Name: c.TalkgroupLabel,
AlphaTag: c.TGAlphaTag,
TGGroup: c.TalkgroupGroup,
})
if err != nil {
return nil, err
}
tg := &tgsp.Talkgroup{
Talkgroup: tgm,
System: database.System{
ID: c.System,
Name: sys,
},
Learned: tgm.Learned,
}
t.add(tg)
return tg, nil
}
func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*tgsp.Talkgroup, error) {
db := database.FromCtx(ctx)
sysName, hasSys := t.SystemName(ctx, system)
if !hasSys {
@ -309,9 +348,10 @@ func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.Upse
Name: sysName,
}
tgs := make([]*Talkgroup, 0, len(input))
tgs := make([]*tgsp.Talkgroup, 0, len(input))
err := db.InTx(ctx, func(db database.Store) error {
versionParams := make([]database.StoreTGVersionParams, 0, len(input))
for i := range input {
// normalize tags
for j, tag := range input[i].Tags {
@ -320,19 +360,26 @@ func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.Upse
input[i].SystemID = int32(system)
input[i].Learned = common.PtrTo(false)
}
var oerr error
batch := db.UpsertTalkgroup(ctx, input)
defer batch.Close()
tgUpsertBatch := db.UpsertTalkgroup(ctx, input)
defer tgUpsertBatch.Close()
batch.QueryRow(func(_ int, r database.Talkgroup, err error) {
tgUpsertBatch.QueryRow(func(_ int, r database.Talkgroup, err error) {
if err != nil {
oerr = err
return
}
tgs = append(tgs, &Talkgroup{
versionParams = append(versionParams, database.StoreTGVersionParams{
SystemID: int32(system),
TGID: r.TGID,
Submitter: auth.UIDFrom(ctx),
})
tgs = append(tgs, &tgsp.Talkgroup{
Talkgroup: r,
System: sys,
Learned: r.Learned,
@ -343,7 +390,17 @@ func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.Upse
return oerr
}
return nil
versionBatch := db.StoreTGVersion(ctx, versionParams)
defer versionBatch.Close()
versionBatch.Exec(func(_ int, err error) {
if err != nil {
oerr = err
return
}
})
return oerr
}, pgx.TxOptions{})
if err != nil {

23
sql/postgres/migrations/001_initial.up.sql
View file

@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS talkgroups(
alert_config JSONB,
weight REAL NOT NULL DEFAULT 1.0,
learned BOOLEAN NOT NULL DEFAULT FALSE,
ignored BOOLEAN NOT NULL DEFAULT FALSE,
UNIQUE (system_id, tgid)
);
@ -44,15 +45,25 @@ CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
CREATE TABLE IF NOT EXISTS talkgroups_learned(
CREATE TABLE IF NOT EXISTS talkgroup_versions(
-- version metadata
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
system_id INTEGER REFERENCES systems(id) NOT NULL,
tgid INTEGER NOT NULL,
name TEXT NOT NULL,
time TIMESTAMPTZ NOT NULL,
created_by INTEGER REFERENCES users(id),
-- talkgroup snapshot
system_id INT4 REFERENCES systems(id),
tgid INT4,
name TEXT,
alpha_tag TEXT,
tg_group TEXT,
ignored BOOLEAN,
UNIQUE (system_id, tgid, name)
frequency INTEGER,
metadata JSONB,
tags TEXT[],
alert BOOLEAN,
alert_config JSONB,
weight REAL,
learned BOOLEAN,
ignored BOOLEAN
);
CREATE TABLE IF NOT EXISTS alerts(

121
sql/postgres/queries/talkgroups.sql
View file

@ -29,47 +29,20 @@ SELECT
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid) AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;
WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid);
-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = @system AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = @system AND ignored IS NOT TRUE;
WHERE tg.system_id = @system;
-- name: GetTalkgroupsWithLearned :many
SELECT
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE;
-- name: GetSystemName :one
@ -122,28 +95,92 @@ SET
learned = COALESCE(sqlc.narg('learned'), tg.learned)
RETURNING *;
-- name: AddTalkgroupWithLearnedFlag :exec
-- name: StoreTGVersion :batchexec
INSERT INTO talkgroup_versions(time, created_by,
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned
) SELECT NOW(), @submitter,
tg.system_id,
tg.tgid,
tg.name,
tg.alpha_tag,
tg.tg_group,
tg.frequency,
tg.metadata,
tg.tags,
tg.alert,
tg.alert_config,
tg.weight,
tg.learned
FROM talkgroups tg WHERE tg.system_id = @system_id AND tg.tgid = @tgid;
-- name: AddLearnedTalkgroup :one
INSERT INTO talkgroups(
system_id,
tgid,
learned
) VALUES(
@system_id,
@tgid,
TRUE
);
-- name: AddLearnedTalkgroup :one
INSERT INTO talkgroups_learned(
system_id,
tgid,
learned,
name,
alpha_tag,
tg_group
) VALUES (
@system_id,
@tgid,
TRUE,
sqlc.narg('name'),
sqlc.narg('alpha_tag'),
sqlc.narg('tg_group')
) RETURNING id;
) RETURNING *;
-- name: RestoreTalkgroupVersion :one
INSERT INTO talkgroups(
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned,
ignored
)
SELECT
system_id,
tgid,
name,
alpha_tag,
tg_group,
frequency,
metadata,
tags,
alert,
alert_config,
weight,
learned,
ignored
FROM talkgroup_versions tgv ON CONFLICT (system_id, tgid) DO UPDATE SET
name = excluded.name,
alpha_tag = excluded.alpha_tag,
tg_group = excluded.tg_group,
metadata = excluded.metadata,
tags = excluded.tags,
alert = excluded.alert,
alert_config = excluded.alert_config,
weight = excluded.weight,
learned = excluded.learner,
ignored = excluded.ignored
WHERE tgv.id = ANY(@version_ids)
RETURNING *;