amigan/stillbox
1
0
Fork 0
Code Issues 17 Pull requests Projects Releases Packages Wiki Activity Actions

New tg ID schema and initial importer #35

Merged
amigan merged 17 commits from reid into trunk 2024-11-15 13:28:05 -05:00
Conversation 0 Commits 17 Files changed 44 +111 -352
16 changed files with 111 additions and 352 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit af80e46068 - Show all commits

8
pkg/alerting/alerting.go
View file

@ -228,11 +228,11 @@ func (as *alerter) scoredTGs() []talkgroups.ID {
return tgs
}
// packedScoredTGs gets a list of packed TGIDs.
func (as *alerter) scoredTGsTuple() []database.TalkgroupT {
tgs := make([]database.TalkgroupT, 0, len(as.scores))
// packedScoredTGs gets a list of TGID tuples.
func (as *alerter) scoredTGsTuple() (tgs database.TGTuples) {
tgs = database.MakeTGTuples(len(as.scores))
for _, s := range as.scores {
tgs = append(tgs, s.ID.Tuple())
tgs.Append(s.ID.System, s.ID.Talkgroup)
}
return tgs

2
pkg/alerting/stats.go
View file

@ -40,7 +40,7 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
db := database.FromCtx(ctx)
tgs, err := db.GetTalkgroupsWithLearnedByPackedIDs(ctx, as.scoredTGsTuple())
tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple())
if err != nil {
log.Error().Err(err).Msg("stats TG get failed")
http.Error(w, err.Error(), http.StatusInternalServerError)

3
pkg/config/config.go
View file

@ -37,7 +37,8 @@ type CORS struct {
}
type DB struct {
Connect string `yaml:"connect"`
Connect string `yaml:"connect"`
LogQueries bool `yaml:"logQueries"`
}
type Logger struct {

15
pkg/database/database.go
View file

@ -21,9 +21,9 @@ type DB struct {
*Queries
}
type myLogger struct{}
type dbLogger struct{}
func (m myLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
log.Debug().Fields(data).Msg(msg)
}
@ -51,12 +51,13 @@ func NewClient(ctx context.Context, conf config.DB) (*DB, error) {
return nil, err
}
logger := myLogger{}
tracer := &tracelog.TraceLog{
Logger: logger,
LogLevel: tracelog.LogLevelTrace,
if conf.LogQueries {
pgConf.ConnConfig.Tracer = &tracelog.TraceLog{
Logger: dbLogger{},
LogLevel: tracelog.LogLevelTrace,
}
}
pgConf.ConnConfig.Tracer = tracer
pool, err := pgxpool.NewWithConfig(ctx, pgConf)
if err != nil {
return nil, err

27
pkg/database/extend.go
View file

@ -1,14 +1,17 @@
package database
func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup { return d.Talkgroup }
func (d GetTalkgroupsRow) GetSystem() System { return d.System }
func (d GetTalkgroupsRow) GetLearned() bool { return d.Learned }
func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Learned }
func (g Talkgroup) GetTalkgroup() Talkgroup { return g }
func (g Talkgroup) GetSystem() System { return System{ID: int(g.SystemID)} }
func (g Talkgroup) GetLearned() bool { return false }
func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup { return d.Talkgroup }
func (d GetTalkgroupsRow) GetSystem() System { return d.System }
func (d GetTalkgroupsRow) GetLearned() bool { return d.Learned }
func (g GetTalkgroupWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupWithLearnedRow) GetSystem() System { return g.System }
func (g GetTalkgroupWithLearnedRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Learned }
func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System { return g.System }
func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Learned }
func (g Talkgroup) GetTalkgroup() Talkgroup { return g }
func (g Talkgroup) GetSystem() System { return System{ID: int(g.SystemID)} }
func (g Talkgroup) GetLearned() bool { return false }

7
pkg/database/querier.go
View file

@ -14,7 +14,6 @@ import (
type Querier interface {
AddAlert(ctx context.Context, arg AddAlertParams) error
AddCall(ctx context.Context, arg AddCallParams) error
BulkSetTalkgroupTags(ctx context.Context, iD uuid.UUID, tags []string) error
CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
DeleteAPIKey(ctx context.Context, apiKey string) error
@ -22,9 +21,9 @@ type Querier interface {
GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error)
GetDatabaseSize(ctx context.Context) (string, error)
GetSystemName(ctx context.Context, systemID int) (string, error)
GetTalkgroup(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupRow, error)
GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error)
GetTalkgroupIDsByTags(ctx context.Context, anytags []string, alltags []string, nottags []string) ([]GetTalkgroupIDsByTagsRow, error)
GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error)
GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error)
GetTalkgroupWithLearned(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupWithLearnedRow, error)
GetTalkgroupsWithAllTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAllTagsRow, error)
GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAnyTagsRow, error)
@ -35,7 +34,7 @@ type Querier interface {
GetUserByUsername(ctx context.Context, username string) (User, error)
GetUsers(ctx context.Context) ([]User, error)
SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error
SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error
UpdatePassword(ctx context.Context, username string, password string) error
UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
}

59
pkg/database/talkgroups.manual.sql.go → pkg/database/talkgroups.go
View file

@ -2,41 +2,25 @@ package database
import (
"context"
"database/sql/driver"
"fmt"
"github.com/jackc/pgx/v5/pgtype"
)
type TalkgroupT struct {
System uint32 `json:"system_id"`
Talkgroup uint32 `json:"tgid"`
}
type TGTuples [2][]uint32
type TalkgroupTs []TalkgroupT
func (t TalkgroupTs) Nest() (sys []uint32, tg []uint32) {
sys = make([]uint32, len(t))
tg = make([]uint32, len(t))
for i := range t {
sys[i] = t[i].System
tg[i] = t[i].Talkgroup
func MakeTGTuples(cap int) TGTuples {
return [2][]uint32{
make([]uint32, 0, cap),
make([]uint32, 0, cap),
}
return
}
func (t TalkgroupT) Value() (driver.Value, error) {
return [2]uint32{t.System, t.Talkgroup}, nil
func (t *TGTuples) Append(sys, tg uint32) {
t[0] = append(t[0], sys)
t[1] = append(t[1], tg)
}
func (t TalkgroupT) TextValue() (pgtype.Text, error) {
return pgtype.Text{String: fmt.Sprintf("%d:%d", t.System, t.Talkgroup)}, nil
}
// Below queries are here because sqlc refuses to parse unnest(x, y)
const getTalkgroupsWithLearnedByPackedIDs = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many
SELECT
const getTalkgroupsWithLearnedBySysTGID = `SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
FROM talkgroups tg
@ -59,9 +43,8 @@ type GetTalkgroupsRow struct {
Learned bool `json:"learned"`
}
func (q *Queries) GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, ids TalkgroupTs) ([]GetTalkgroupsRow, error) {
sysAr, tgAr := ids.Nest()
rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedByPackedIDs, sysAr, tgAr)
func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedBySysTGID, ids[0], ids[1])
if err != nil {
return nil, err
}
@ -96,15 +79,12 @@ func (q *Queries) GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, ids T
return items, nil
}
const getTalkgroupsByPackedIDs = `-- name: GetTalkgroupsByPackedIDs :many
SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
`
JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
func (q *Queries) GetTalkgroupsByPackedIDs(ctx context.Context, ids TalkgroupTs) ([]GetTalkgroupsRow, error) {
sysAr, tgAr := ids.Nest()
rows, err := q.db.Query(ctx, getTalkgroupsByPackedIDs, sysAr, tgAr)
func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
rows, err := q.db.Query(ctx, getTalkgroupsBySysTGID, ids[0], ids[1])
if err != nil {
return nil, err
}
@ -137,3 +117,10 @@ func (q *Queries) GetTalkgroupsByPackedIDs(ctx context.Context, ids TalkgroupTs)
}
return items, nil
}
const bulkSetTalkgroupTags = `UPDATE talkgroups tg SET tags = $3 FROM UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) WHERE (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, tgs TGTuples, tags []string) error {
_, err := q.db.Exec(ctx, bulkSetTalkgroupTags, tgs[0], tgs[1], tags)
return err
}

31
pkg/database/talkgroups.sql.go
View file

@ -10,20 +10,9 @@ import (
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/alerting/rules"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
const bulkSetTalkgroupTags = `-- name: BulkSetTalkgroupTags :exec
UPDATE talkgroups SET tags = $2
WHERE id = ANY($1)
`
func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, iD uuid.UUID, tags []string) error {
_, err := q.db.Exec(ctx, bulkSetTalkgroupTags, iD, tags)
return err
}
const getSystemName = `-- name: GetSystemName :one
SELECT name FROM systems WHERE id = $1
`
@ -44,8 +33,8 @@ type GetTalkgroupRow struct {
Talkgroup Talkgroup `json:"talkgroup"`
}
func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupRow, error) {
row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgid)
func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) {
row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgID)
var i GetTalkgroupRow
err := row.Scan(
&i.Talkgroup.ID,
@ -98,11 +87,11 @@ func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, a
const getTalkgroupTags = `-- name: GetTalkgroupTags :one
SELECT tags FROM talkgroups
WHERE id = systg2id($1, $2)
WHERE system_id = $1 AND tgid = $2
`
func (q *Queries) GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error) {
row := q.db.QueryRow(ctx, getTalkgroupTags, sys, tg)
func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) {
row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tgID)
var tags []string
err := row.Scan(&tags)
return tags, err
@ -117,7 +106,7 @@ JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = ($1, $2)
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@ -362,12 +351,12 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
}
const setTalkgroupTags = `-- name: SetTalkgroupTags :exec
UPDATE talkgroups SET tags = $3
WHERE id = systg2id($1, $2)
UPDATE talkgroups SET tags = $1
WHERE system_id = $2 AND tgid = $3
`
func (q *Queries) SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error {
_, err := q.db.Exec(ctx, setTalkgroupTags, sys, tg, tags)
func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error {
_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tgID)
return err
}

27
pkg/database/talkgroups.sql_test.go
View file

@ -6,34 +6,16 @@ import (
"github.com/stretchr/testify/require"
)
const getTalkgroupsWithLearnedByPackedIDsTest = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.id = ANY($1::INT8[])
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE systg2id(tgl.system_id, tgl.tgid) = ANY($1::INT8[]) AND ignored IS NOT TRUE
`
const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
SELECT
tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
FALSE learned
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE tg.id = systg2id($1, $2)
WHERE (tg.system_id, tg.tgid) = ($1, $2)
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@ -52,7 +34,7 @@ JOIN systems sys ON tg.system_id = sys.id
WHERE tg.system_id = $1
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@ -70,7 +52,7 @@ FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@ -81,7 +63,6 @@ WHERE ignored IS NOT TRUE
`
func TestQueryColumnsMatch(t *testing.T) {
require.Equal(t, getTalkgroupsWithLearnedByPackedIDsTest, getTalkgroupsWithLearnedByPackedIDs)
require.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
require.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
require.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)

33
pkg/talkgroups/cache.go
View file

@ -39,8 +39,8 @@ type Store interface {
// Hint hints the Store that the provided talkgroups will be asked for.
Hint(ctx context.Context, tgs []ID) error
// Load loads the provided packed talkgroup IDs into the Store.
Load(ctx context.Context, tgs []database.TalkgroupT) error
// Load loads the provided talkgroup ID tuples into the Store.
Load(ctx context.Context, tgs database.TGTuples) error
// Invalidate invalidates any caching in the Store.
Invalidate()
@ -98,19 +98,20 @@ func NewCache() Store {
func (t *cache) Hint(ctx context.Context, tgs []ID) error {
t.RLock()
var toLoad []database.TalkgroupT
var toLoad database.TGTuples
if len(t.tgs) > len(tgs)/2 { // TODO: instrument this
for _, tg := range tgs {
_, ok := t.tgs[tg]
if !ok {
toLoad = append(toLoad, tg.Tuple())
toLoad.Append(tg.System, tg.Talkgroup)
}
}
} else {
toLoad = make([]database.TalkgroupT, 0, len(tgs))
toLoad[0] = make([]uint32, 0, len(tgs))
toLoad[1] = make([]uint32, 0, len(tgs))
for _, g := range tgs {
toLoad = append(toLoad, g.Tuple())
toLoad.Append(g.System, g.Talkgroup)
}
}
@ -136,7 +137,7 @@ func (t *cache) add(rec *Talkgroup) error {
type row interface {
database.GetTalkgroupsRow | database.GetTalkgroupsWithLearnedRow |
database.GetTalkgroupsWithLearnedBySystemRow
database.GetTalkgroupsWithLearnedBySystemRow | database.GetTalkgroupWithLearnedRow
GetTalkgroup() database.Talkgroup
GetSystem() database.System
GetLearned() bool
@ -180,7 +181,7 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
}
t.RUnlock()
tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, toGet.Tuples())
tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, toGet.Tuples())
if err != nil {
return nil, err
}
@ -196,8 +197,8 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
return addToRowList(t, r, tgRecords)
}
func (t *cache) Load(ctx context.Context, tgs []database.TalkgroupT) error {
tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, tgs)
func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error {
tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, tgs)
if err != nil {
return err
}
@ -245,7 +246,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
return rec, nil
}
recs, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, []database.TalkgroupT{tg.Tuple()})
record, err := database.FromCtx(ctx).GetTalkgroupWithLearned(ctx, int32(tg.System), int32(tg.Talkgroup))
switch err {
case nil:
case pgx.ErrNoRows:
@ -255,17 +256,13 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
return nil, errors.Join(ErrNotFound, err)
}
if len(recs) < 1 {
return nil, ErrNotFound
}
err = t.add(rowToTalkgroup(recs[0]))
err = t.add(rowToTalkgroup(record))
if err != nil {
log.Error().Err(err).Msg("TG() cache add")
return rowToTalkgroup(recs[0]), errors.Join(ErrNotFound, err)
return rowToTalkgroup(record), errors.Join(ErrNotFound, err)
}
return rowToTalkgroup(recs[0]), nil
return rowToTalkgroup(record), nil
}
func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool) {

20
pkg/talkgroups/talkgroup.go
View file

@ -26,13 +26,16 @@ type ID struct {
type IDs []ID
func (ids *IDs) Tuples() []database.TalkgroupT {
r := make([]database.TalkgroupT, len(*ids))
for i := range *ids {
r[i] = (*ids)[i].Tuple()
func (t IDs) Tuples() database.TGTuples {
sys := make([]uint32, len(t))
tg := make([]uint32, len(t))
for i := range t {
sys[i] = t[i].System
tg[i] = t[i].Talkgroup
}
return r
return database.TGTuples{sys, tg}
}
type intId interface {
@ -46,13 +49,6 @@ func TG[T intId, U intId](sys T, tgid U) ID {
}
}
func (t ID) Tuple() database.TalkgroupT {
return database.TalkgroupT{
System: t.System,
Talkgroup: t.Talkgroup,
}
}
func (t ID) String() string {
return fmt.Sprintf("%d:%d", t.System, t.Talkgroup)

32
sql/postgres/migrations/001_initial.up.sql
View file

@ -23,31 +23,10 @@ CREATE TABLE IF NOT EXISTS systems(
name TEXT NOT NULL
);
CREATE OR REPLACE FUNCTION systg2id(_sys INTEGER, _tg INTEGER) RETURNS INT8 LANGUAGE plpgsql AS
$$
BEGIN
RETURN ((_sys::BIGINT << 32) | _tg);
END
$$;
CREATE OR REPLACE FUNCTION tgfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
$$
BEGIN
RETURN (_id & x'ffffffff'::BIGINT);
END
$$;
CREATE OR REPLACE FUNCTION sysfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
$$
BEGIN
RETURN (_id >> 32);
END
$$;
CREATE TABLE IF NOT EXISTS talkgroups(
id INT8 PRIMARY KEY,
system_id INT4 REFERENCES systems(id) NOT NULL GENERATED ALWAYS AS (id >> 32) STORED,
tgid INT4 NOT NULL GENERATED ALWAYS AS (id & x'ffffffff'::BIGINT) STORED,
id UUID PRIMARY KEY,
system_id INT4 REFERENCES systems(id) NOT NULL,
tgid INT4 NOT NULL,
name TEXT,
alpha_tag TEXT,
tg_group TEXT,
@ -56,9 +35,12 @@ CREATE TABLE IF NOT EXISTS talkgroups(
tags TEXT[] NOT NULL DEFAULT '{}',
alert BOOLEAN NOT NULL DEFAULT 'true',
alert_config JSONB,
weight REAL NOT NULL DEFAULT 1.0
weight REAL NOT NULL DEFAULT 1.0,
UNIQUE (system_id, tgid)
);
CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
CREATE TABLE IF NOT EXISTS talkgroups_learned(

9
sql/postgres/migrations/002_reid.down.sql
View file

@ -1,9 +0,0 @@
DROP INDEX IF EXISTS talkgroups_system_tgid_idx;
ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE INT8 USING (systg2id(system_id, tgid));
ALTER TABLE talkgroups DROP COLUMN IF EXISTS tgid;
ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS tgid INT4 NOT NULL GENERATED ALWAYS AS (id & x'ffffffff'::BIGINT) STORED,
ALTER TABLE talkgroups DROP COLUMN IF EXISTS system_id;
ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS system_id INT4 REFERENCES systems(id) NOT NULL GENERATED ALWAYS AS (id >> 32) STORED;

7
sql/postgres/migrations/002_reid.up.sql
View file

@ -1,7 +0,0 @@
ALTER TABLE talkgroups ALTER COLUMN system_id DROP EXPRESSION;
ALTER TABLE talkgroups ALTER COLUMN tgid DROP EXPRESSION;
ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid());
CREATE INDEX IF NOT EXISTS talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);

157
sql/postgres/migrations/flattened_initial.sql
View file

@ -1,157 +0,0 @@
CREATE TABLE IF NOT EXISTS users(
id SERIAL PRIMARY KEY,
username VARCHAR (255) UNIQUE NOT NULL,
password TEXT NOT NULL,
email TEXT NOT NULL,
is_admin BOOLEAN NOT NULL,
prefs JSONB
);
CREATE INDEX IF NOT EXISTS users_username_idx ON users(username);
CREATE TABLE IF NOT EXISTS api_keys(
id SERIAL PRIMARY KEY,
owner INTEGER REFERENCES users(id) NOT NULL,
created_at TIMESTAMP NOT NULL,
expires TIMESTAMP,
disabled BOOLEAN,
api_key TEXT UNIQUE NOT NULL
);
CREATE TABLE IF NOT EXISTS systems(
id INTEGER PRIMARY KEY,
name TEXT NOT NULL
);
CREATE OR REPLACE FUNCTION systg2id(_sys INTEGER, _tg INTEGER) RETURNS INT8 LANGUAGE plpgsql AS
$$
BEGIN
RETURN ((_sys::BIGINT << 32) | _tg);
END
$$;
CREATE OR REPLACE FUNCTION tgfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
$$
BEGIN
RETURN (_id & x'ffffffff'::BIGINT);
END
$$;
CREATE OR REPLACE FUNCTION sysfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
$$
BEGIN
RETURN (_id >> 32);
END
$$;
CREATE TABLE IF NOT EXISTS talkgroups(
id UUID PRIMARY KEY,
system_id INT4 REFERENCES systems(id) NOT NULL,
tgid INT4 NOT NULL,
name TEXT,
alpha_tag TEXT,
tg_group TEXT,
frequency INTEGER,
metadata JSONB,
tags TEXT[] NOT NULL DEFAULT '{}',
alert BOOLEAN NOT NULL DEFAULT 'true',
alert_config JSONB,
weight REAL NOT NULL DEFAULT 1.0
);
CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
CREATE TABLE IF NOT EXISTS talkgroups_learned(
id SERIAL PRIMARY KEY,
system_id INTEGER REFERENCES systems(id) NOT NULL,
tgid INTEGER NOT NULL,
name TEXT NOT NULL,
alpha_tag TEXT,
ignored BOOLEAN,
UNIQUE (system_id, tgid, name)
);
CREATE TABLE IF NOT EXISTS alerts(
id UUID PRIMARY KEY,
time TIMESTAMPTZ NOT NULL,
tgid INTEGER NOT NULL,
system_id INTEGER REFERENCES systems(id) NOT NULL,
weight REAL,
score REAL,
orig_score REAL,
notified BOOLEAN NOT NULL DEFAULT 'false',
metadata JSONB
);
CREATE OR REPLACE FUNCTION learn_talkgroup()
RETURNS TRIGGER AS $$
BEGIN
IF NOT EXISTS (
SELECT tg.system_id, tg.tgid, tg.name, tg.alpha_tag FROM talkgroups tg WHERE tg.system_id = NEW.system AND tg.tgid = NEW.talkgroup
UNION
SELECT tgl.system_id, tgl.tgid, tgl.name, tgl.alpha_tag FROM talkgroups_learned tgl WHERE tgl.system_id = NEW.system AND tgl.tgid = NEW.talkgroup
) THEN
INSERT INTO talkgroups_learned(system_id, tgid, name, alpha_tag) VALUES(
NEW.system, NEW.talkgroup, NEW.tg_label, NEW.tg_alpha_tag
) ON CONFLICT DO NOTHING;
END IF;
RETURN NEW;
END
$$ LANGUAGE plpgsql;
CREATE TABLE IF NOT EXISTS calls(
id UUID PRIMARY KEY,
submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL,
system INTEGER NOT NULL,
talkgroup INTEGER NOT NULL,
call_date TIMESTAMPTZ NOT NULL,
audio_name TEXT,
audio_blob BYTEA,
duration INTEGER,
audio_type TEXT,
audio_url TEXT,
frequency INTEGER NOT NULL,
frequencies INTEGER[],
patches INTEGER[],
tg_label TEXT,
tg_alpha_tag TEXT,
tg_group TEXT,
source INTEGER NOT NULL,
transcript TEXT
);
CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls
FOR EACH ROW EXECUTE FUNCTION learn_talkgroup();
CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript));
CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date);
CREATE TABLE IF NOT EXISTS settings(
name TEXT PRIMARY KEY,
updated_by INTEGER REFERENCES users(id),
value JSONB
);
CREATE TABLE IF NOT EXISTS incidents(
id UUID PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
start_time TIMESTAMP,
end_time TIMESTAMP,
location JSONB,
metadata JSONB
);
CREATE INDEX IF NOT EXISTS incidents_name_description_idx ON incidents USING GIN (
(to_tsvector('english', name) || to_tsvector('english', coalesce(description, ''))
)
);
CREATE TABLE IF NOT EXISTS incidents_calls(
incident_id UUID REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE,
call_id UUID REFERENCES calls(id) ON UPDATE CASCADE,
notes JSONB,
PRIMARY KEY (incident_id, call_id)
);

26
sql/postgres/queries/talkgroups.sql
View file

@ -8,25 +8,21 @@ WHERE tags && ARRAY[$1];
-- name: GetTalkgroupIDsByTags :many
SELECT system_id, tgid FROM talkgroups
WHERE (tags @> ARRAY[sqlc.arg(anyTags)])
AND (tags && ARRAY[sqlc.arg(allTags)])
AND NOT (tags @> ARRAY[sqlc.arg(notTags)]);
WHERE (tags @> ARRAY[@anyTags])
AND (tags && ARRAY[@allTags])
AND NOT (tags @> ARRAY[@notTags]);
-- name: GetTalkgroupTags :one
SELECT tags FROM talkgroups
WHERE id = systg2id($1, $2);
WHERE system_id = @system_id AND tgid = @tg_id;
-- name: SetTalkgroupTags :exec
UPDATE talkgroups SET tags = $3
WHERE id = systg2id($1, $2);
-- name: BulkSetTalkgroupTags :exec
UPDATE talkgroups SET tags = $2
WHERE id = ANY($1);
UPDATE talkgroups SET tags = @tags
WHERE system_id = @system_id AND tgid = @tg_id;
-- name: GetTalkgroup :one
SELECT sqlc.embed(talkgroups) FROM talkgroups
WHERE (system_id, tgid) = (@system_id, @tgid);
WHERE (system_id, tgid) = (@system_id, @tg_id);
-- name: GetTalkgroupWithLearned :one
SELECT
@ -34,17 +30,17 @@ sqlc.embed(tg), sqlc.embed(sys),
FALSE learned
FROM talkgroups tg
JOIN systems sys ON tg.system_id = sys.id
WHERE (tg.system_id, tg.tgid) = (sqlc.arg(system_id), sqlc.arg(tgid))
WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid)
UNION
SELECT
tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
TRUE learned
FROM talkgroups_learned tgl
JOIN systems sys ON tgl.system_id = sys.id
WHERE tgl.system_id = sqlc.arg(system_id) AND tgl.tgid = sqlc.arg(tgid) AND ignored IS NOT TRUE;
WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;
-- name: GetTalkgroupsWithLearnedBySystem :many
SELECT
@ -82,7 +78,7 @@ JOIN systems sys ON tgl.system_id = sys.id
WHERE ignored IS NOT TRUE;
-- name: GetSystemName :one
SELECT name FROM systems WHERE id = sqlc.arg(system_id);
SELECT name FROM systems WHERE id = @system_id;
-- name: UpdateTalkgroup :one
UPDATE talkgroups