amigan/stillbox
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'callsnormalize' (#40) from callsnormalize into trunk

Browse source 
Reviewed-on: #40
This commit is contained in:
Daniel 2024-11-19 10:17:16 -05:00
commit 26f09ab094
30 changed files with 2123 additions and 1847 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.mockery.yaml
View file

@ -7,4 +7,4 @@ packages:
dynatron.me/x/stillbox/pkg/database:
config:
interfaces:
DB:
Store:

5
cmd/stillbox/main.go
View file

@ -35,8 +35,5 @@ func main() {
cmds := append([]*cobra.Command{serve.Command(cfg)}, admin.Command(cfg)...)
rootCmd.AddCommand(cmds...)
err := rootCmd.Execute()
if err != nil {
log.Fatal().Err(err).Msg("Dying")
}
rootCmd.Execute()
}

5
pkg/alerting/alert/alert.go
View file

@ -10,12 +10,11 @@ import (
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Alert struct {
ID uuid.UUID
ID int
Timestamp time.Time
TGName string
Score trending.Score[talkgroups.ID]
@ -34,7 +33,6 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
}
return database.AddAlertParams{
ID: a.ID,
Time: pgtype.Timestamptz{Time: a.Timestamp, Valid: true},
SystemID: int(a.Score.ID.System),
TGID: int(a.Score.ID.Talkgroup),
@ -48,7 +46,6 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
// Make creates an alert for later rendering or storage.
func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
d := Alert{
ID: uuid.New(),
Score: score,
Timestamp: time.Now(),
Weight: 1.0,

2
pkg/alerting/alerting.go
View file

@ -161,7 +161,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
for _, s := range as.scores {
origScore := s.Score
tgr, err := as.tgCache.TG(ctx, s.ID)
if err == nil && !tgr.Talkgroup.Alert {
if err != nil || !tgr.Talkgroup.Alert {
continue
}

3
pkg/alerting/stats.go
View file

@ -40,7 +40,8 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
db := database.FromCtx(ctx)
tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple())
tgt := as.scoredTGsTuple()
tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, tgt)
if err != nil {
log.Error().Err(err).Msg("stats TG get failed")
http.Error(w, err.Error(), http.StatusInternalServerError)

4
pkg/auth/jwt.go
View file

@ -98,7 +98,7 @@ func (a *Auth) Login(ctx context.Context, username, password string) (token stri
return a.newToken(found.ID), nil
}
func (a *Auth) newToken(uid int32) string {
func (a *Auth) newToken(uid int) string {
claims := claims{
"sub": strconv.Itoa(int(uid)),
}
@ -134,7 +134,7 @@ func (a *Auth) routeRefresh(w http.ResponseWriter, r *http.Request) {
return
}
tok := a.newToken(int32(uid))
tok := a.newToken(uid)
cookie := &http.Cookie{
Name: "jwt",

17
pkg/calls/call.go
View file

@ -1,11 +1,13 @@
package calls
import (
"context"
"fmt"
"time"
"dynatron.me/x/stillbox/internal/audio"
"dynatron.me/x/stillbox/pkg/auth"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/pb"
"dynatron.me/x/stillbox/pkg/talkgroups"
@ -111,6 +113,21 @@ func (c *Call) ToPB() *pb.Call {
}
}
func (c *Call) LearnTG(ctx context.Context, db database.Store) (learnedId int, err error) {
err = db.AddTalkgroupWithLearnedFlag(ctx, int32(c.System), int32(c.Talkgroup))
if err != nil {
return 0, fmt.Errorf("addTalkgroupWithLearnedFlag: %w", err)
}
return db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
SystemID: c.System,
TGID: c.Talkgroup,
Name: c.TalkgroupLabel,
AlphaTag: c.TGAlphaTag,
TGGroup: c.TalkgroupGroup,
})
}
func (c *Call) computeLength() (err error) {
var td time.Duration

19
pkg/database/calls.sql.go
View file

@ -13,7 +13,7 @@ import (
)
const addAlert = `-- name: AddAlert :exec
INSERT INTO alerts (id, time, tgid, system_id, weight, score, orig_score, notified, metadata)
INSERT INTO alerts (time, tgid, system_id, weight, score, orig_score, notified, metadata)
VALUES
(
$1,
@ -23,13 +23,11 @@ VALUES
$5,
$6,
$7,
$8,
$9
$8
)
`
type AddAlertParams struct {
ID uuid.UUID `json:"id"`
Time pgtype.Timestamptz `json:"time"`
TGID int `json:"tgid"`
SystemID int `json:"system_id"`
@ -42,7 +40,6 @@ type AddAlertParams struct {
func (q *Queries) AddAlert(ctx context.Context, arg AddAlertParams) error {
_, err := q.db.Exec(ctx, addAlert,
arg.ID,
arg.Time,
arg.TGID,
arg.SystemID,
@ -109,9 +106,9 @@ type AddCallParams struct {
Frequency int `json:"frequency"`
Frequencies []int `json:"frequencies"`
Patches []int `json:"patches"`
TgLabel *string `json:"tg_label"`
TgAlphaTag *string `json:"tg_alpha_tag"`
TgGroup *string `json:"tg_group"`
TGLabel *string `json:"tg_label"`
TGAlphaTag *string `json:"tg_alpha_tag"`
TGGroup *string `json:"tg_group"`
Source int `json:"source"`
}
@ -130,9 +127,9 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) error {
arg.Frequency,
arg.Frequencies,
arg.Patches,
arg.TgLabel,
arg.TgAlphaTag,
arg.TgGroup,
arg.TGLabel,
arg.TGAlphaTag,
arg.TGGroup,
arg.Source,
)
return err

39
pkg/database/database.go
View file

@ -3,6 +3,7 @@ package database
import (
"context"
"errors"
"fmt"
"strings"
"dynatron.me/x/stillbox/pkg/config"
@ -19,11 +20,12 @@ import (
// DB is a database handle.
//go:generate mockery
type DB interface {
type Store interface {
Querier
talkgroupQuerier
DB() *Database
InTx(context.Context, func(Store) error, pgx.TxOptions) error
}
type Database struct {
@ -35,14 +37,41 @@ func (db *Database) DB() *Database {
return db
}
func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOptions) error {
tx, err := db.DB().Pool.BeginTx(ctx, opts)
if err != nil {
return fmt.Errorf("Tx begin: %w", err)
}
defer tx.Rollback(ctx)
dbtx := &Database{Pool: db.Pool, Queries: db.Queries.WithTx(tx)}
err = f(dbtx)
if err != nil {
return fmt.Errorf("Tx: %w", err)
}
err = tx.Commit(ctx)
if err != nil {
return fmt.Errorf("Tx commit: %w", err)
}
return nil
}
type dbLogger struct{}
func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
log.Debug().Fields(data).Msg(msg)
}
func Close(c Store) {
c.(*Database).Pool.Close()
}
// NewClient creates a new DB using the provided config.
func NewClient(ctx context.Context, conf config.DB) (DB, error) {
func NewClient(ctx context.Context, conf config.DB) (Store, error) {
dir, err := iofs.New(sqlembed.Migrations, "postgres/migrations")
if err != nil {
return nil, err
@ -90,8 +119,8 @@ type dBCtxKey string
const DBCtxKey dBCtxKey = "dbctx"
// FromCtx returns the database handle from the provided Context.
func FromCtx(ctx context.Context) DB {
c, ok := ctx.Value(DBCtxKey).(DB)
func FromCtx(ctx context.Context) Store {
c, ok := ctx.Value(DBCtxKey).(Store)
if !ok {
panic("no DB in context")
}
@ -100,7 +129,7 @@ func FromCtx(ctx context.Context) DB {
}
// CtxWithDB returns a Context with the provided database handle.
func CtxWithDB(ctx context.Context, conn DB) context.Context {
func CtxWithDB(ctx context.Context, conn Store) context.Context {
return context.WithValue(ctx, DBCtxKey, conn)
}

8
pkg/database/extend.go
View file

@ -2,16 +2,16 @@ package database
func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup { return d.Talkgroup }
func (d GetTalkgroupsRow) GetSystem() System { return d.System }
func (d GetTalkgroupsRow) GetLearned() bool { return d.Learned }
func (d GetTalkgroupsRow) GetLearned() bool { return d.Talkgroup.Learned }
func (g GetTalkgroupWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupWithLearnedRow) GetSystem() System { return g.System }
func (g GetTalkgroupWithLearnedRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupWithLearnedRow) GetLearned() bool { return g.Talkgroup.Learned }
func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Talkgroup.Learned }
func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Talkgroup.Learned }
func (g Talkgroup) GetTalkgroup() Talkgroup { return g }
func (g Talkgroup) GetSystem() System { return System{ID: int(g.SystemID)} }
func (g Talkgroup) GetLearned() bool { return false }

1631
pkg/database/mocks/DB.go
View file

File diff suppressed because it is too large Load diff

1786
pkg/database/mocks/Store.go Normal file
View file

File diff suppressed because it is too large Load diff

30
pkg/database/models.go
View file

@ -14,7 +14,7 @@ import (
)
type Alert struct {
ID uuid.UUID `json:"id"`
ID int `json:"id"`
Time pgtype.Timestamptz `json:"time"`
TGID int `json:"tgid"`
SystemID int `json:"system_id"`
@ -26,7 +26,7 @@ type Alert struct {
}
type ApiKey struct {
ID int32 `json:"id"`
ID int `json:"id"`
Owner int `json:"owner"`
CreatedAt time.Time `json:"created_at"`
Expires pgtype.Timestamp `json:"expires"`
@ -48,9 +48,9 @@ type Call struct {
Frequency int `json:"frequency"`
Frequencies []int `json:"frequencies"`
Patches []int `json:"patches"`
TgLabel *string `json:"tg_label"`
TgAlphaTag *string `json:"tg_alpha_tag"`
TgGroup *string `json:"tg_group"`
TGLabel *string `json:"tg_label"`
TGAlphaTag *string `json:"tg_alpha_tag"`
TGGroup *string `json:"tg_group"`
Source int `json:"source"`
Transcript *string `json:"transcript"`
}
@ -83,31 +83,33 @@ type System struct {
}
type Talkgroup struct {
ID uuid.UUID `json:"id"`
ID int `json:"id"`
SystemID int32 `json:"system_id"`
TGID int32 `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
TGGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata jsontypes.Metadata `json:"metadata"`
Tags []string `json:"tags"`
Alert bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight float32 `json:"weight"`
Learned bool `json:"learned"`
}
type TalkgroupsLearned struct {
ID uuid.UUID `json:"id"`
SystemID int `json:"system_id"`
TGID int `json:"tgid"`
Name string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
Ignored *bool `json:"ignored"`
ID int `json:"id"`
SystemID int `json:"system_id"`
TGID int `json:"tgid"`
Name string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TGGroup *string `json:"tg_group"`
Ignored *bool `json:"ignored"`
}
type User struct {
ID int32 `json:"id"`
ID int `json:"id"`
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`

12
pkg/database/querier.go
View file

@ -14,6 +14,8 @@ import (
type Querier interface {
AddAlert(ctx context.Context, arg AddAlertParams) error
AddCall(ctx context.Context, arg AddCallParams) error
AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error)
AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error
CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
DeleteAPIKey(ctx context.Context, apiKey string) error
@ -21,20 +23,20 @@ type Querier interface {
GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error)
GetDatabaseSize(ctx context.Context) (string, error)
GetSystemName(ctx context.Context, systemID int) (string, error)
GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error)
GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error)
GetTalkgroupIDsByTags(ctx context.Context, anyTags []string, allTags []string, notTags []string) ([]GetTalkgroupIDsByTagsRow, error)
GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error)
GetTalkgroupTags(ctx context.Context, systemID int32, tGID int32) ([]string, error)
GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error)
GetTalkgroupsWithAllTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAllTagsRow, error)
GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAnyTagsRow, error)
GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroupsWithLearnedRow, error)
GetTalkgroupsWithLearnedBySystem(ctx context.Context, system int32) ([]GetTalkgroupsWithLearnedBySystemRow, error)
GetUserByID(ctx context.Context, id int32) (User, error)
GetUserByUID(ctx context.Context, id int32) (User, error)
GetUserByID(ctx context.Context, id int) (User, error)
GetUserByUID(ctx context.Context, id int) (User, error)
GetUserByUsername(ctx context.Context, username string) (User, error)
GetUsers(ctx context.Context) ([]User, error)
SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error
SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error
UpdatePassword(ctx context.Context, username string, password string) error
UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
}

35
pkg/database/talkgroups.go
View file

@ -2,6 +2,9 @@ package database
import (
"context"
"errors"
"github.com/jackc/pgx/v5/pgconn"
)
type talkgroupQuerier interface {
@ -12,6 +15,17 @@ type talkgroupQuerier interface {
type TGTuples [2][]uint32
const TGConstraintName = "calls_system_talkgroup_fkey"
func IsTGConstraintViolation(e error) bool {
var err *pgconn.PgError
if errors.As(e, &err) && err.Code == "23503" && err.ConstraintName == TGConstraintName {
return true
}
return false
}
func MakeTGTuples(cap int) TGTuples {
return [2][]uint32{
make([]uint32, 0, cap),
@ -27,18 +41,17 @@ func (t *TGTuples) Append(sys, tg uint32) {
// Below queries are here because sqlc refuses to parse unnest(x, y)
const getTalkgroupsWithLearnedBySysTGID = `SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);`
@ -46,7 +59,6 @@ JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys
type GetTalkgroupsRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
System System `json:"system"`
Learned bool `json:"learned"`
}
func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
@ -64,7 +76,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
@ -73,7 +85,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
&i.Talkgroup.Weight,
&i.System.ID,
&i.System.Name,
&i.Learned,
&i.Talkgroup.Learned,
); err != nil {
return nil, err
}
@ -87,7 +99,8 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
WHERE tg.learned IS NOT TRUE;`
func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
rows, err := q.db.Query(ctx, getTalkgroupsBySysTGID, ids[0], ids[1])
@ -104,7 +117,7 @@ func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]G
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,

143
pkg/database/talkgroups.sql.go
View file

@ -10,9 +10,62 @@ import (
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/alerting/rules"
"github.com/jackc/pgx/v5/pgtype"
)
const addLearnedTalkgroup = `-- name: AddLearnedTalkgroup :one
INSERT INTO talkgroups_learned(
system_id,
tgid,
name,
alpha_tag,
tg_group
) VALUES (
$1,
$2,
$3,
$4,
$5
) RETURNING id
`
type AddLearnedTalkgroupParams struct {
SystemID int `json:"system_id"`
TGID int `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TGGroup *string `json:"tg_group"`
}
func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error) {
row := q.db.QueryRow(ctx, addLearnedTalkgroup,
arg.SystemID,
arg.TGID,
arg.Name,
arg.AlphaTag,
arg.TGGroup,
)
var id int
err := row.Scan(&id)
return id, err
}
const addTalkgroupWithLearnedFlag = `-- name: AddTalkgroupWithLearnedFlag :exec
INSERT INTO talkgroups (
system_id,
tgid,
learned
) VALUES(
$1,
$2,
't'
)
`
func (q *Queries) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
_, err := q.db.Exec(ctx, addTalkgroupWithLearnedFlag, systemID, tGID)
return err
}
const getSystemName = `-- name: GetSystemName :one
SELECT name FROM systems WHERE id = $1
`
@ -25,7 +78,7 @@ func (q *Queries) GetSystemName(ctx context.Context, systemID int) (string, erro
}
const getTalkgroup = `-- name: GetTalkgroup :one
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
WHERE (system_id, tgid) = ($1, $2)
`
@ -33,8 +86,8 @@ type GetTalkgroupRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
}
func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) {
row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgID)
func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error) {
row := q.db.QueryRow(ctx, getTalkgroup, systemID, tGID)
var i GetTalkgroupRow
err := row.Scan(
&i.Talkgroup.ID,
@ -42,13 +95,14 @@ func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32)
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
)
return i, err
}
@ -90,8 +144,8 @@ SELECT tags FROM talkgroups
WHERE system_id = $1 AND tgid = $2
`
func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) {
row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tgID)
func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tGID int32) ([]string, error) {
row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tGID)
var tags []string
err := row.Scan(&tags)
return tags, err
@ -99,18 +153,16 @@ func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int
const getTalkgroupWithLearned = `-- name: GetTalkgroupWithLearned :one
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = ($1, $2)
WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
@ -119,7 +171,6 @@ WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
type GetTalkgroupWithLearnedRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
System System `json:"system"`
Learned bool `json:"learned"`
}
func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error) {
@ -131,22 +182,22 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.System.ID,
&i.System.Name,
&i.Learned,
)
return i, err
}
const getTalkgroupsWithAllTags = `-- name: GetTalkgroupsWithAllTags :many
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
WHERE tags && ARRAY[$1]
`
@ -169,13 +220,14 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
); err != nil {
return nil, err
}
@ -188,7 +240,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
}
const getTalkgroupsWithAnyTags = `-- name: GetTalkgroupsWithAnyTags :many
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
WHERE tags @> ARRAY[$1]
`
@ -211,13 +263,14 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
); err != nil {
return nil, err
}
@ -231,17 +284,16 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
const getTalkgroupsWithLearned = `-- name: GetTalkgroupsWithLearned :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE
@ -250,7 +302,6 @@ WHERE ignored IS NOT TRUE
type GetTalkgroupsWithLearnedRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
System System `json:"system"`
Learned bool `json:"learned"`
}
func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroupsWithLearnedRow, error) {
@ -268,16 +319,16 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.System.ID,
&i.System.Name,
&i.Learned,
); err != nil {
return nil, err
}
@ -291,18 +342,16 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
const getTalkgroupsWithLearnedBySystem = `-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = $1
WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
@ -311,7 +360,6 @@ WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
type GetTalkgroupsWithLearnedBySystemRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
System System `json:"system"`
Learned bool `json:"learned"`
}
func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system int32) ([]GetTalkgroupsWithLearnedBySystemRow, error) {
@ -329,16 +377,16 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
&i.Talkgroup.TGID,
&i.Talkgroup.Name,
&i.Talkgroup.AlphaTag,
&i.Talkgroup.TgGroup,
&i.Talkgroup.TGGroup,
&i.Talkgroup.Frequency,
&i.Talkgroup.Metadata,
&i.Talkgroup.Tags,
&i.Talkgroup.Alert,
&i.Talkgroup.AlertConfig,
&i.Talkgroup.Weight,
&i.Talkgroup.Learned,
&i.System.ID,
&i.System.Name,
&i.Learned,
); err != nil {
return nil, err
}
@ -355,8 +403,8 @@ UPDATE talkgroups SET tags = $1
WHERE system_id = $2 AND tgid = $3
`
func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error {
_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tgID)
func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error {
_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tGID)
return err
}
@ -373,20 +421,20 @@ SET
alert_config = COALESCE($8, alert_config),
weight = COALESCE($9, weight)
WHERE id = $10 OR (system_id = $11 AND tgid = $12)
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight
RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
`
type UpdateTalkgroupParams struct {
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
TGGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata jsontypes.Metadata `json:"metadata"`
Tags []string `json:"tags"`
Alert *bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight *float32 `json:"weight"`
ID pgtype.UUID `json:"id"`
ID *int32 `json:"id"`
SystemID *int32 `json:"system_id"`
TGID *int32 `json:"tgid"`
}
@ -395,7 +443,7 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
row := q.db.QueryRow(ctx, updateTalkgroup,
arg.Name,
arg.AlphaTag,
arg.TgGroup,
arg.TGGroup,
arg.Frequency,
arg.Metadata,
arg.Tags,
@ -413,13 +461,14 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
&i.TGID,
&i.Name,
&i.AlphaTag,
&i.TgGroup,
&i.TGGroup,
&i.Frequency,
&i.Metadata,
&i.Tags,
&i.Alert,
&i.AlertConfig,
&i.Weight,
&i.Learned,
)
return i, err
}

49
pkg/database/talkgroups.sql_test.go
View file

@ -3,23 +3,21 @@ package database
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/assert"
)
const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = ($1, $2)
WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
UNION
SELECT
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
@ -27,18 +25,16 @@ WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
const getTalkgroupsWithLearnedBySystemTest = `-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = $1
WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
UNION
SELECT
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
@ -46,24 +42,23 @@ WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
const getTalkgroupsWithLearnedTest = `-- name: GetTalkgroupsWithLearned :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE
`
func TestQueryColumnsMatch(t *testing.T) {
require.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
require.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
require.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
assert.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
assert.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
assert.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
}

4
pkg/database/users.sql.go
View file

@ -113,7 +113,7 @@ SELECT id, username, password, email, is_admin, prefs FROM users
WHERE id = $1 LIMIT 1
`
func (q *Queries) GetUserByID(ctx context.Context, id int32) (User, error) {
func (q *Queries) GetUserByID(ctx context.Context, id int) (User, error) {
row := q.db.QueryRow(ctx, getUserByID, id)
var i User
err := row.Scan(
@ -132,7 +132,7 @@ SELECT id, username, password, email, is_admin, prefs FROM users
WHERE id = $1 LIMIT 1
`
func (q *Queries) GetUserByUID(ctx context.Context, id int32) (User, error) {
func (q *Queries) GetUserByUID(ctx context.Context, id int) (User, error) {
row := q.db.QueryRow(ctx, getUserByUID, id)
var i User
err := row.Scan(

2
pkg/nexus/commands.go
View file

@ -78,7 +78,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
resp := &pb.TalkgroupInfo{
Tg: tg,
Name: tgi.Talkgroup.Name,
Group: tgi.Talkgroup.TgGroup,
Group: tgi.Talkgroup.TGGroup,
Frequency: tgi.Talkgroup.Frequency,
Metadata: md,
Tags: tgi.Talkgroup.Tags,

4
pkg/server/server.go
View file

@ -27,7 +27,7 @@ const shutdownTimeout = 5 * time.Second
type Server struct {
auth *auth.Auth
conf *config.Config
db database.DB
db database.Store
r *chi.Mux
sources sources.Sources
sinks sinks.Sinks
@ -112,7 +112,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
}
func (s *Server) Go(ctx context.Context) error {
defer s.db.DB().Close()
defer database.Close(s.db)
s.installHupHandler()

48
pkg/sinks/database.go
View file

@ -8,16 +8,17 @@ import (
"dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/database"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/rs/zerolog/log"
)
type DatabaseSink struct {
db database.DB
db database.Store
}
func NewDatabaseSink(db database.DB) *DatabaseSink {
return &DatabaseSink{db: db}
func NewDatabaseSink(store database.Store) *DatabaseSink {
return &DatabaseSink{store}
}
func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
@ -26,14 +27,37 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
return nil
}
err := s.db.AddCall(ctx, s.toAddCallParams(call))
if err != nil {
return fmt.Errorf("add call: %w", err)
params := s.toAddCallParams(call)
err := s.db.InTx(ctx, func(tx database.Store) error {
err := tx.AddCall(ctx, params)
if err != nil {
return fmt.Errorf("add call: %w", err)
}
log.Debug().Str("id", call.ID.String()).Int("system", call.System).Int("tgid", call.Talkgroup).Msg("stored")
return nil
}, pgx.TxOptions{})
if err != nil && database.IsTGConstraintViolation(err) {
return s.db.InTx(ctx, func(tx database.Store) error {
_, err := call.LearnTG(ctx, tx)
if err != nil {
return fmt.Errorf("add call: learn tg: %w", err)
}
err = tx.AddCall(ctx, params)
if err != nil {
return fmt.Errorf("add call: retry: %w", err)
}
return nil
}, pgx.TxOptions{})
}
log.Debug().Str("id", call.ID.String()).Int("system", call.System).Int("tgid", call.Talkgroup).Msg("stored")
return nil
return err
}
func (s *DatabaseSink) SinkType() string {
@ -54,9 +78,9 @@ func (s *DatabaseSink) toAddCallParams(call *calls.Call) database.AddCallParams
Frequency: call.Frequency,
Frequencies: call.Frequencies,
Patches: call.Patches,
TgLabel: call.TalkgroupLabel,
TgAlphaTag: call.TGAlphaTag,
TgGroup: call.TalkgroupGroup,
TGLabel: call.TalkgroupLabel,
TGAlphaTag: call.TGAlphaTag,
TGGroup: call.TalkgroupGroup,
Source: call.Source,
}
}

8
pkg/sinks/relay.go
View file

@ -44,21 +44,20 @@ func NewRelayManager(s Sinks, cfgs []config.Relay) (*RelayManager, error) {
}
for i, cfg := range cfgs {
rs, err := rm.newRelay(cfg)
rs, err := rm.newRelay(i, cfg)
if err != nil {
return nil, err
}
rm.relays = append(rm.relays, rs)
sinkName := fmt.Sprintf("relay%d:%s", i, rs.url.Host)
s.Register(sinkName, rs, cfg.Required)
s.Register(rs.Name, rs, cfg.Required)
}
return rm, nil
}
func (rs *RelayManager) newRelay(cfg config.Relay) (*Relay, error) {
func (rs *RelayManager) newRelay(idx int, cfg config.Relay) (*Relay, error) {
u, err := url.Parse(cfg.URL)
if err != nil {
return nil, err
@ -71,6 +70,7 @@ func (rs *RelayManager) newRelay(cfg config.Relay) (*Relay, error) {
u = u.JoinPath("/api/call-upload")
return &Relay{
Name: fmt.Sprintf("relay%d:%s", idx, u.Host),
Relay: cfg,
url: u,
mgr: rs,

2
pkg/talkgroups/cache.go
View file

@ -252,7 +252,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
case pgx.ErrNoRows:
return nil, ErrNotFound
default:
log.Error().Err(err).Msg("TG() cache add db get")
log.Error().Err(err).Uint32("sys", tg.System).Uint32("tg", tg.Talkgroup).Msg("TG() cache add db get")
return nil, errors.Join(ErrNotFound, err)
}

5
pkg/talkgroups/importer/import.go
View file

@ -10,8 +10,6 @@ import (
"strconv"
"strings"
"github.com/google/uuid"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/database"
"dynatron.me/x/stillbox/pkg/talkgroups"
@ -112,12 +110,11 @@ func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int,
gn := groupName // must take a copy
tgs = append(tgs, talkgroups.Talkgroup{
Talkgroup: database.Talkgroup{
ID: uuid.New(),
TGID: int32(tgt.Talkgroup),
SystemID: int32(tgt.System),
Name: &fields[4],
AlphaTag: &fields[3],
TgGroup: &gn,
TGGroup: &gn,
Metadata: metadata,
Tags: tags,
Weight: 1.0,

2
pkg/talkgroups/importer/import_test.go
View file

@ -57,7 +57,7 @@ func TestImport(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dbMock := mocks.NewDB(t)
dbMock := mocks.NewStore(t)
if tc.expectErr == nil {
dbMock.EXPECT().GetSystemName(mock.AnythingOfType("*context.valueCtx"), tc.sysID).Return(tc.sysName, nil)
}

2
pkg/talkgroups/importer/testdata/riscon.json vendored
View file

File diff suppressed because one or more lines are too long

34
sql/postgres/migrations/001_initial.up.sql
View file

@ -1,5 +1,5 @@
CREATE TABLE IF NOT EXISTS users(
id SERIAL PRIMARY KEY,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
username VARCHAR (255) UNIQUE NOT NULL,
password TEXT NOT NULL,
email TEXT NOT NULL,
@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS users(
CREATE INDEX IF NOT EXISTS users_username_idx ON users(username);
CREATE TABLE IF NOT EXISTS api_keys(
id SERIAL PRIMARY KEY,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
owner INTEGER REFERENCES users(id) NOT NULL,
created_at TIMESTAMP NOT NULL,
expires TIMESTAMP,
@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS systems(
);
CREATE TABLE IF NOT EXISTS talkgroups(
id UUID PRIMARY KEY,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
system_id INT4 REFERENCES systems(id) NOT NULL,
tgid INT4 NOT NULL,
name TEXT,
@ -36,6 +36,7 @@ CREATE TABLE IF NOT EXISTS talkgroups(
alert BOOLEAN NOT NULL DEFAULT 'true',
alert_config JSONB,
weight REAL NOT NULL DEFAULT 1.0,
learned BOOLEAN NOT NULL DEFAULT FALSE,
UNIQUE (system_id, tgid)
);
@ -44,17 +45,18 @@ CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
CREATE TABLE IF NOT EXISTS talkgroups_learned(
id UUID PRIMARY KEY,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
system_id INTEGER REFERENCES systems(id) NOT NULL,
tgid INTEGER NOT NULL,
name TEXT NOT NULL,
alpha_tag TEXT,
tg_group TEXT,
ignored BOOLEAN,
UNIQUE (system_id, tgid, name)
);
CREATE TABLE IF NOT EXISTS alerts(
id UUID PRIMARY KEY,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
time TIMESTAMPTZ NOT NULL,
tgid INTEGER NOT NULL,
system_id INTEGER REFERENCES systems(id) NOT NULL,
@ -65,22 +67,6 @@ CREATE TABLE IF NOT EXISTS alerts(
metadata JSONB
);
CREATE OR REPLACE FUNCTION learn_talkgroup()
RETURNS TRIGGER AS $$
BEGIN
IF NOT EXISTS (
SELECT tg.system_id, tg.tgid, tg.name, tg.alpha_tag FROM talkgroups tg WHERE tg.system_id = NEW.system AND tg.tgid = NEW.talkgroup
UNION
SELECT tgl.system_id, tgl.tgid, tgl.name, tgl.alpha_tag FROM talkgroups_learned tgl WHERE tgl.system_id = NEW.system AND tgl.tgid = NEW.talkgroup
) THEN
INSERT INTO talkgroups_learned(system_id, tgid, name, alpha_tag) VALUES(
NEW.system, NEW.talkgroup, NEW.tg_label, NEW.tg_alpha_tag
) ON CONFLICT DO NOTHING;
END IF;
RETURN NEW;
END
$$ LANGUAGE plpgsql;
CREATE TABLE IF NOT EXISTS calls(
id UUID PRIMARY KEY,
submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL,
@ -99,12 +85,10 @@ CREATE TABLE IF NOT EXISTS calls(
tg_alpha_tag TEXT,
tg_group TEXT,
source INTEGER NOT NULL,
transcript TEXT
transcript TEXT,
FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid)
);
CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls
FOR EACH ROW EXECUTE FUNCTION learn_talkgroup();
CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript));
CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date);

3
sql/postgres/queries/calls.sql
View file

@ -41,10 +41,9 @@ source
UPDATE calls SET transcript = $2 WHERE id = $1;
-- name: AddAlert :exec
INSERT INTO alerts (id, time, tgid, system_id, weight, score, orig_score, notified, metadata)
INSERT INTO alerts (time, tgid, system_id, weight, score, orig_score, notified, metadata)
VALUES
(
sqlc.arg(id),
sqlc.arg(time),
sqlc.arg(tgid),
sqlc.arg(system_id),

61
sql/postgres/queries/talkgroups.sql
View file

@ -26,53 +26,48 @@ WHERE (system_id, tgid) = (@system_id, @tg_id);
-- name: GetTalkgroupWithLearned :one
SELECT
sqlc.embed(tg), sqlc.embed(sys),
FALSE learned
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid)
WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid) AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;
-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
sqlc.embed(tg), sqlc.embed(sys),
FALSE learned
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = @system
WHERE tg.system_id = @system AND tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = @system AND ignored IS NOT TRUE;
-- name: GetTalkgroupsWithLearned :many
SELECT
sqlc.embed(tg), sqlc.embed(sys),
FALSE learned
sqlc.embed(tg), sqlc.embed(sys)
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.learned IS NOT TRUE
UNION
SELECT
tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE;
@ -94,3 +89,29 @@ SET
weight = COALESCE(sqlc.narg('weight'), weight)
WHERE id = sqlc.narg('id') OR (system_id = sqlc.narg('system_id') AND tgid = sqlc.narg('tgid'))
RETURNING *;
-- name: AddTalkgroupWithLearnedFlag :exec
INSERT INTO talkgroups (
system_id,
tgid,
learned
) VALUES(
@system_id,
@tgid,
't'
);
-- name: AddLearnedTalkgroup :one
INSERT INTO talkgroups_learned(
system_id,
tgid,
name,
alpha_tag,
tg_group
) VALUES (
@system_id,
@tgid,
sqlc.narg('name'),
sqlc.narg('alpha_tag'),
sqlc.narg('tg_group')
) RETURNING id;

5
sql/sqlc.yaml
View file

@ -14,6 +14,7 @@ sql:
initialisms:
- id
- tgid
- tg
emit_pointers_for_null_types: true
overrides:
- db_type: "uuid"
@ -22,10 +23,6 @@ sql:
type: "UUID"
- db_type: "pg_catalog.int4"
go_type: "int"
- db_type: "pg_catalog.serial4"
go_type: "int"
- db_type: "integer"
go_type: "int"
- db_type: "pg_catalog.timestamp"
go_type: "time.Time"
- db_type: "pg_catalog.text"