diff --git a/pkg/alerting/alerting.go b/pkg/alerting/alerting.go index ee3ec5f..316f266 100644 --- a/pkg/alerting/alerting.go +++ b/pkg/alerting/alerting.go @@ -228,11 +228,11 @@ func (as *alerter) scoredTGs() []talkgroups.ID { return tgs } -// packedScoredTGs gets a list of packed TGIDs. -func (as *alerter) scoredTGsTuple() []database.TalkgroupT { - tgs := make([]database.TalkgroupT, 0, len(as.scores)) +// packedScoredTGs gets a list of TGID tuples. +func (as *alerter) scoredTGsTuple() (tgs database.TGTuples) { + tgs = database.MakeTGTuples(len(as.scores)) for _, s := range as.scores { - tgs = append(tgs, s.ID.Tuple()) + tgs.Append(s.ID.System, s.ID.Talkgroup) } return tgs diff --git a/pkg/alerting/stats.go b/pkg/alerting/stats.go index 5dea62e..3ace635 100644 --- a/pkg/alerting/stats.go +++ b/pkg/alerting/stats.go @@ -40,7 +40,7 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() db := database.FromCtx(ctx) - tgs, err := db.GetTalkgroupsWithLearnedByPackedIDs(ctx, as.scoredTGsTuple()) + tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple()) if err != nil { log.Error().Err(err).Msg("stats TG get failed") http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/pkg/config/config.go b/pkg/config/config.go index 47e288a..fb2c589 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,7 +37,8 @@ type CORS struct { } type DB struct { - Connect string `yaml:"connect"` + Connect string `yaml:"connect"` + LogQueries bool `yaml:"logQueries"` } type Logger struct { diff --git a/pkg/database/database.go b/pkg/database/database.go index ebc5a6f..d1a7e06 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -21,9 +21,9 @@ type DB struct { *Queries } -type myLogger struct{} +type dbLogger struct{} -func (m myLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) { +func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) { log.Debug().Fields(data).Msg(msg) } @@ -51,12 +51,13 @@ func NewClient(ctx context.Context, conf config.DB) (*DB, error) { return nil, err } - logger := myLogger{} - tracer := &tracelog.TraceLog{ - Logger: logger, - LogLevel: tracelog.LogLevelTrace, + if conf.LogQueries { + pgConf.ConnConfig.Tracer = &tracelog.TraceLog{ + Logger: dbLogger{}, + LogLevel: tracelog.LogLevelTrace, + } } - pgConf.ConnConfig.Tracer = tracer + pool, err := pgxpool.NewWithConfig(ctx, pgConf) if err != nil { return nil, err diff --git a/pkg/database/extend.go b/pkg/database/extend.go index 61619d4..d885991 100644 --- a/pkg/database/extend.go +++ b/pkg/database/extend.go @@ -1,14 +1,17 @@ package database -func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup { return d.Talkgroup } -func (d GetTalkgroupsRow) GetSystem() System { return d.System } -func (d GetTalkgroupsRow) GetLearned() bool { return d.Learned } -func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup } -func (g GetTalkgroupsWithLearnedRow) GetSystem() System { return g.System } -func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Learned } -func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup } -func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System { return g.System } -func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Learned } -func (g Talkgroup) GetTalkgroup() Talkgroup { return g } -func (g Talkgroup) GetSystem() System { return System{ID: int(g.SystemID)} } -func (g Talkgroup) GetLearned() bool { return false } +func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup { return d.Talkgroup } +func (d GetTalkgroupsRow) GetSystem() System { return d.System } +func (d GetTalkgroupsRow) GetLearned() bool { return d.Learned } +func (g GetTalkgroupWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup } +func (g GetTalkgroupWithLearnedRow) GetSystem() System { return g.System } +func (g GetTalkgroupWithLearnedRow) GetLearned() bool { return g.Learned } +func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup { return g.Talkgroup } +func (g GetTalkgroupsWithLearnedRow) GetSystem() System { return g.System } +func (g GetTalkgroupsWithLearnedRow) GetLearned() bool { return g.Learned } +func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup } +func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System { return g.System } +func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool { return g.Learned } +func (g Talkgroup) GetTalkgroup() Talkgroup { return g } +func (g Talkgroup) GetSystem() System { return System{ID: int(g.SystemID)} } +func (g Talkgroup) GetLearned() bool { return false } diff --git a/pkg/database/querier.go b/pkg/database/querier.go index 8d91e22..0f72f67 100644 --- a/pkg/database/querier.go +++ b/pkg/database/querier.go @@ -14,7 +14,6 @@ import ( type Querier interface { AddAlert(ctx context.Context, arg AddAlertParams) error AddCall(ctx context.Context, arg AddCallParams) error - BulkSetTalkgroupTags(ctx context.Context, iD uuid.UUID, tags []string) error CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) DeleteAPIKey(ctx context.Context, apiKey string) error @@ -22,9 +21,9 @@ type Querier interface { GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error) GetDatabaseSize(ctx context.Context) (string, error) GetSystemName(ctx context.Context, systemID int) (string, error) - GetTalkgroup(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupRow, error) + GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, alltags []string, nottags []string) ([]GetTalkgroupIDsByTagsRow, error) - GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error) + GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) GetTalkgroupWithLearned(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupWithLearnedRow, error) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAllTagsRow, error) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAnyTagsRow, error) @@ -35,7 +34,7 @@ type Querier interface { GetUserByUsername(ctx context.Context, username string) (User, error) GetUsers(ctx context.Context) ([]User, error) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error - SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error + SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error UpdatePassword(ctx context.Context, username string, password string) error UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) } diff --git a/pkg/database/talkgroups.manual.sql.go b/pkg/database/talkgroups.go similarity index 61% rename from pkg/database/talkgroups.manual.sql.go rename to pkg/database/talkgroups.go index debebe5..0870892 100644 --- a/pkg/database/talkgroups.manual.sql.go +++ b/pkg/database/talkgroups.go @@ -2,41 +2,25 @@ package database import ( "context" - "database/sql/driver" - "fmt" - - "github.com/jackc/pgx/v5/pgtype" ) -type TalkgroupT struct { - System uint32 `json:"system_id"` - Talkgroup uint32 `json:"tgid"` -} +type TGTuples [2][]uint32 -type TalkgroupTs []TalkgroupT - -func (t TalkgroupTs) Nest() (sys []uint32, tg []uint32) { - sys = make([]uint32, len(t)) - tg = make([]uint32, len(t)) - - for i := range t { - sys[i] = t[i].System - tg[i] = t[i].Talkgroup +func MakeTGTuples(cap int) TGTuples { + return [2][]uint32{ + make([]uint32, 0, cap), + make([]uint32, 0, cap), } - - return } -func (t TalkgroupT) Value() (driver.Value, error) { - return [2]uint32{t.System, t.Talkgroup}, nil +func (t *TGTuples) Append(sys, tg uint32) { + t[0] = append(t[0], sys) + t[1] = append(t[1], tg) } -func (t TalkgroupT) TextValue() (pgtype.Text, error) { - return pgtype.Text{String: fmt.Sprintf("%d:%d", t.System, t.Talkgroup)}, nil -} +// Below queries are here because sqlc refuses to parse unnest(x, y) -const getTalkgroupsWithLearnedByPackedIDs = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many -SELECT +const getTalkgroupsWithLearnedBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, FALSE learned FROM talkgroups tg @@ -59,9 +43,8 @@ type GetTalkgroupsRow struct { Learned bool `json:"learned"` } -func (q *Queries) GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, ids TalkgroupTs) ([]GetTalkgroupsRow, error) { - sysAr, tgAr := ids.Nest() - rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedByPackedIDs, sysAr, tgAr) +func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) { + rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedBySysTGID, ids[0], ids[1]) if err != nil { return nil, err } @@ -96,15 +79,12 @@ func (q *Queries) GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, ids T return items, nil } -const getTalkgroupsByPackedIDs = `-- name: GetTalkgroupsByPackedIDs :many -SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg +const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg JOIN systems sys ON tg.system_id = sys.id -JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg) -` +JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);` -func (q *Queries) GetTalkgroupsByPackedIDs(ctx context.Context, ids TalkgroupTs) ([]GetTalkgroupsRow, error) { - sysAr, tgAr := ids.Nest() - rows, err := q.db.Query(ctx, getTalkgroupsByPackedIDs, sysAr, tgAr) +func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) { + rows, err := q.db.Query(ctx, getTalkgroupsBySysTGID, ids[0], ids[1]) if err != nil { return nil, err } @@ -137,3 +117,10 @@ func (q *Queries) GetTalkgroupsByPackedIDs(ctx context.Context, ids TalkgroupTs) } return items, nil } + +const bulkSetTalkgroupTags = `UPDATE talkgroups tg SET tags = $3 FROM UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) WHERE (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);` + +func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, tgs TGTuples, tags []string) error { + _, err := q.db.Exec(ctx, bulkSetTalkgroupTags, tgs[0], tgs[1], tags) + return err +} diff --git a/pkg/database/talkgroups.sql.go b/pkg/database/talkgroups.sql.go index 1a2e37c..a3836f9 100644 --- a/pkg/database/talkgroups.sql.go +++ b/pkg/database/talkgroups.sql.go @@ -10,20 +10,9 @@ import ( "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/alerting/rules" - "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) -const bulkSetTalkgroupTags = `-- name: BulkSetTalkgroupTags :exec -UPDATE talkgroups SET tags = $2 -WHERE id = ANY($1) -` - -func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, iD uuid.UUID, tags []string) error { - _, err := q.db.Exec(ctx, bulkSetTalkgroupTags, iD, tags) - return err -} - const getSystemName = `-- name: GetSystemName :one SELECT name FROM systems WHERE id = $1 ` @@ -44,8 +33,8 @@ type GetTalkgroupRow struct { Talkgroup Talkgroup `json:"talkgroup"` } -func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgid int32) (GetTalkgroupRow, error) { - row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgid) +func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) { + row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgID) var i GetTalkgroupRow err := row.Scan( &i.Talkgroup.ID, @@ -98,11 +87,11 @@ func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, a const getTalkgroupTags = `-- name: GetTalkgroupTags :one SELECT tags FROM talkgroups -WHERE id = systg2id($1, $2) +WHERE system_id = $1 AND tgid = $2 ` -func (q *Queries) GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error) { - row := q.db.QueryRow(ctx, getTalkgroupTags, sys, tg) +func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) { + row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tgID) var tags []string err := row.Scan(&tags) return tags, err @@ -117,7 +106,7 @@ JOIN systems sys ON tg.system_id = sys.id WHERE (tg.system_id, tg.tgid) = ($1, $2) UNION SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, +NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, TRUE, NULL::JSONB, 1.0, sys.id, sys.name, @@ -362,12 +351,12 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i } const setTalkgroupTags = `-- name: SetTalkgroupTags :exec -UPDATE talkgroups SET tags = $3 -WHERE id = systg2id($1, $2) +UPDATE talkgroups SET tags = $1 +WHERE system_id = $2 AND tgid = $3 ` -func (q *Queries) SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error { - _, err := q.db.Exec(ctx, setTalkgroupTags, sys, tg, tags) +func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error { + _, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tgID) return err } diff --git a/pkg/database/talkgroups.sql_test.go b/pkg/database/talkgroups.sql_test.go index 10f9888..14c9215 100644 --- a/pkg/database/talkgroups.sql_test.go +++ b/pkg/database/talkgroups.sql_test.go @@ -6,34 +6,16 @@ import ( "github.com/stretchr/testify/require" ) -const getTalkgroupsWithLearnedByPackedIDsTest = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many -SELECT -tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, -FALSE learned -FROM talkgroups tg -JOIN systems sys ON tg.system_id = sys.id -WHERE tg.id = ANY($1::INT8[]) -UNION -SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, -tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, -CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, -TRUE, NULL::JSONB, 1.0, sys.id, sys.name, -TRUE learned -FROM talkgroups_learned tgl -JOIN systems sys ON tgl.system_id = sys.id -WHERE systg2id(tgl.system_id, tgl.tgid) = ANY($1::INT8[]) AND ignored IS NOT TRUE -` const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, FALSE learned FROM talkgroups tg JOIN systems sys ON tg.system_id = sys.id -WHERE tg.id = systg2id($1, $2) +WHERE (tg.system_id, tg.tgid) = ($1, $2) UNION SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, +NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, TRUE, NULL::JSONB, 1.0, sys.id, sys.name, @@ -52,7 +34,7 @@ JOIN systems sys ON tg.system_id = sys.id WHERE tg.system_id = $1 UNION SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, +NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, TRUE, NULL::JSONB, 1.0, sys.id, sys.name, @@ -70,7 +52,7 @@ FROM talkgroups tg JOIN systems sys ON tg.system_id = sys.id UNION SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, +NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, TRUE, NULL::JSONB, 1.0, sys.id, sys.name, @@ -81,7 +63,6 @@ WHERE ignored IS NOT TRUE ` func TestQueryColumnsMatch(t *testing.T) { - require.Equal(t, getTalkgroupsWithLearnedByPackedIDsTest, getTalkgroupsWithLearnedByPackedIDs) require.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned) require.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem) require.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned) diff --git a/pkg/talkgroups/cache.go b/pkg/talkgroups/cache.go index 0e86b2c..84bc070 100644 --- a/pkg/talkgroups/cache.go +++ b/pkg/talkgroups/cache.go @@ -39,8 +39,8 @@ type Store interface { // Hint hints the Store that the provided talkgroups will be asked for. Hint(ctx context.Context, tgs []ID) error - // Load loads the provided packed talkgroup IDs into the Store. - Load(ctx context.Context, tgs []database.TalkgroupT) error + // Load loads the provided talkgroup ID tuples into the Store. + Load(ctx context.Context, tgs database.TGTuples) error // Invalidate invalidates any caching in the Store. Invalidate() @@ -98,19 +98,20 @@ func NewCache() Store { func (t *cache) Hint(ctx context.Context, tgs []ID) error { t.RLock() - var toLoad []database.TalkgroupT + var toLoad database.TGTuples if len(t.tgs) > len(tgs)/2 { // TODO: instrument this for _, tg := range tgs { _, ok := t.tgs[tg] if !ok { - toLoad = append(toLoad, tg.Tuple()) + toLoad.Append(tg.System, tg.Talkgroup) } } } else { - toLoad = make([]database.TalkgroupT, 0, len(tgs)) + toLoad[0] = make([]uint32, 0, len(tgs)) + toLoad[1] = make([]uint32, 0, len(tgs)) for _, g := range tgs { - toLoad = append(toLoad, g.Tuple()) + toLoad.Append(g.System, g.Talkgroup) } } @@ -136,7 +137,7 @@ func (t *cache) add(rec *Talkgroup) error { type row interface { database.GetTalkgroupsRow | database.GetTalkgroupsWithLearnedRow | - database.GetTalkgroupsWithLearnedBySystemRow + database.GetTalkgroupsWithLearnedBySystemRow | database.GetTalkgroupWithLearnedRow GetTalkgroup() database.Talkgroup GetSystem() database.System GetLearned() bool @@ -180,7 +181,7 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) { } t.RUnlock() - tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, toGet.Tuples()) + tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, toGet.Tuples()) if err != nil { return nil, err } @@ -196,8 +197,8 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) { return addToRowList(t, r, tgRecords) } -func (t *cache) Load(ctx context.Context, tgs []database.TalkgroupT) error { - tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, tgs) +func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error { + tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, tgs) if err != nil { return err } @@ -245,7 +246,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) { return rec, nil } - recs, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, []database.TalkgroupT{tg.Tuple()}) + record, err := database.FromCtx(ctx).GetTalkgroupWithLearned(ctx, int32(tg.System), int32(tg.Talkgroup)) switch err { case nil: case pgx.ErrNoRows: @@ -255,17 +256,13 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) { return nil, errors.Join(ErrNotFound, err) } - if len(recs) < 1 { - return nil, ErrNotFound - } - - err = t.add(rowToTalkgroup(recs[0])) + err = t.add(rowToTalkgroup(record)) if err != nil { log.Error().Err(err).Msg("TG() cache add") - return rowToTalkgroup(recs[0]), errors.Join(ErrNotFound, err) + return rowToTalkgroup(record), errors.Join(ErrNotFound, err) } - return rowToTalkgroup(recs[0]), nil + return rowToTalkgroup(record), nil } func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool) { diff --git a/pkg/talkgroups/talkgroup.go b/pkg/talkgroups/talkgroup.go index 5849ebb..c609f42 100644 --- a/pkg/talkgroups/talkgroup.go +++ b/pkg/talkgroups/talkgroup.go @@ -26,13 +26,16 @@ type ID struct { type IDs []ID -func (ids *IDs) Tuples() []database.TalkgroupT { - r := make([]database.TalkgroupT, len(*ids)) - for i := range *ids { - r[i] = (*ids)[i].Tuple() +func (t IDs) Tuples() database.TGTuples { + sys := make([]uint32, len(t)) + tg := make([]uint32, len(t)) + + for i := range t { + sys[i] = t[i].System + tg[i] = t[i].Talkgroup } - return r + return database.TGTuples{sys, tg} } type intId interface { @@ -46,13 +49,6 @@ func TG[T intId, U intId](sys T, tgid U) ID { } } -func (t ID) Tuple() database.TalkgroupT { - return database.TalkgroupT{ - System: t.System, - Talkgroup: t.Talkgroup, - } -} - func (t ID) String() string { return fmt.Sprintf("%d:%d", t.System, t.Talkgroup) diff --git a/sql/postgres/migrations/001_initial.up.sql b/sql/postgres/migrations/001_initial.up.sql index 4706fe9..f412f28 100644 --- a/sql/postgres/migrations/001_initial.up.sql +++ b/sql/postgres/migrations/001_initial.up.sql @@ -23,31 +23,10 @@ CREATE TABLE IF NOT EXISTS systems( name TEXT NOT NULL ); -CREATE OR REPLACE FUNCTION systg2id(_sys INTEGER, _tg INTEGER) RETURNS INT8 LANGUAGE plpgsql AS -$$ -BEGIN - RETURN ((_sys::BIGINT << 32) | _tg); -END -$$; - -CREATE OR REPLACE FUNCTION tgfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS -$$ -BEGIN - RETURN (_id & x'ffffffff'::BIGINT); -END -$$; - -CREATE OR REPLACE FUNCTION sysfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS -$$ -BEGIN - RETURN (_id >> 32); -END -$$; - CREATE TABLE IF NOT EXISTS talkgroups( - id INT8 PRIMARY KEY, - system_id INT4 REFERENCES systems(id) NOT NULL GENERATED ALWAYS AS (id >> 32) STORED, - tgid INT4 NOT NULL GENERATED ALWAYS AS (id & x'ffffffff'::BIGINT) STORED, + id UUID PRIMARY KEY, + system_id INT4 REFERENCES systems(id) NOT NULL, + tgid INT4 NOT NULL, name TEXT, alpha_tag TEXT, tg_group TEXT, @@ -56,9 +35,12 @@ CREATE TABLE IF NOT EXISTS talkgroups( tags TEXT[] NOT NULL DEFAULT '{}', alert BOOLEAN NOT NULL DEFAULT 'true', alert_config JSONB, - weight REAL NOT NULL DEFAULT 1.0 + weight REAL NOT NULL DEFAULT 1.0, + UNIQUE (system_id, tgid) ); +CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid); + CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags); CREATE TABLE IF NOT EXISTS talkgroups_learned( diff --git a/sql/postgres/migrations/002_reid.down.sql b/sql/postgres/migrations/002_reid.down.sql deleted file mode 100644 index b27dba3..0000000 --- a/sql/postgres/migrations/002_reid.down.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP INDEX IF EXISTS talkgroups_system_tgid_idx; - -ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE INT8 USING (systg2id(system_id, tgid)); - -ALTER TABLE talkgroups DROP COLUMN IF EXISTS tgid; -ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS tgid INT4 NOT NULL GENERATED ALWAYS AS (id & x'ffffffff'::BIGINT) STORED, - -ALTER TABLE talkgroups DROP COLUMN IF EXISTS system_id; -ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS system_id INT4 REFERENCES systems(id) NOT NULL GENERATED ALWAYS AS (id >> 32) STORED; diff --git a/sql/postgres/migrations/002_reid.up.sql b/sql/postgres/migrations/002_reid.up.sql deleted file mode 100644 index b0d91d7..0000000 --- a/sql/postgres/migrations/002_reid.up.sql +++ /dev/null @@ -1,7 +0,0 @@ -ALTER TABLE talkgroups ALTER COLUMN system_id DROP EXPRESSION; - -ALTER TABLE talkgroups ALTER COLUMN tgid DROP EXPRESSION; - -ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid()); - -CREATE INDEX IF NOT EXISTS talkgroups_system_tgid_idx ON talkgroups (system_id, tgid); diff --git a/sql/postgres/migrations/flattened_initial.sql b/sql/postgres/migrations/flattened_initial.sql deleted file mode 100644 index 1c4397d..0000000 --- a/sql/postgres/migrations/flattened_initial.sql +++ /dev/null @@ -1,157 +0,0 @@ -CREATE TABLE IF NOT EXISTS users( - id SERIAL PRIMARY KEY, - username VARCHAR (255) UNIQUE NOT NULL, - password TEXT NOT NULL, - email TEXT NOT NULL, - is_admin BOOLEAN NOT NULL, - prefs JSONB -); - -CREATE INDEX IF NOT EXISTS users_username_idx ON users(username); - -CREATE TABLE IF NOT EXISTS api_keys( - id SERIAL PRIMARY KEY, - owner INTEGER REFERENCES users(id) NOT NULL, - created_at TIMESTAMP NOT NULL, - expires TIMESTAMP, - disabled BOOLEAN, - api_key TEXT UNIQUE NOT NULL -); - -CREATE TABLE IF NOT EXISTS systems( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL -); - -CREATE OR REPLACE FUNCTION systg2id(_sys INTEGER, _tg INTEGER) RETURNS INT8 LANGUAGE plpgsql AS -$$ -BEGIN - RETURN ((_sys::BIGINT << 32) | _tg); -END -$$; - -CREATE OR REPLACE FUNCTION tgfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS -$$ -BEGIN - RETURN (_id & x'ffffffff'::BIGINT); -END -$$; - -CREATE OR REPLACE FUNCTION sysfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS -$$ -BEGIN - RETURN (_id >> 32); -END -$$; - -CREATE TABLE IF NOT EXISTS talkgroups( - id UUID PRIMARY KEY, - system_id INT4 REFERENCES systems(id) NOT NULL, - tgid INT4 NOT NULL, - name TEXT, - alpha_tag TEXT, - tg_group TEXT, - frequency INTEGER, - metadata JSONB, - tags TEXT[] NOT NULL DEFAULT '{}', - alert BOOLEAN NOT NULL DEFAULT 'true', - alert_config JSONB, - weight REAL NOT NULL DEFAULT 1.0 -); - -CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid); - -CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags); - -CREATE TABLE IF NOT EXISTS talkgroups_learned( - id SERIAL PRIMARY KEY, - system_id INTEGER REFERENCES systems(id) NOT NULL, - tgid INTEGER NOT NULL, - name TEXT NOT NULL, - alpha_tag TEXT, - ignored BOOLEAN, - UNIQUE (system_id, tgid, name) -); - -CREATE TABLE IF NOT EXISTS alerts( - id UUID PRIMARY KEY, - time TIMESTAMPTZ NOT NULL, - tgid INTEGER NOT NULL, - system_id INTEGER REFERENCES systems(id) NOT NULL, - weight REAL, - score REAL, - orig_score REAL, - notified BOOLEAN NOT NULL DEFAULT 'false', - metadata JSONB -); - -CREATE OR REPLACE FUNCTION learn_talkgroup() -RETURNS TRIGGER AS $$ -BEGIN - IF NOT EXISTS ( - SELECT tg.system_id, tg.tgid, tg.name, tg.alpha_tag FROM talkgroups tg WHERE tg.system_id = NEW.system AND tg.tgid = NEW.talkgroup - UNION - SELECT tgl.system_id, tgl.tgid, tgl.name, tgl.alpha_tag FROM talkgroups_learned tgl WHERE tgl.system_id = NEW.system AND tgl.tgid = NEW.talkgroup - ) THEN - INSERT INTO talkgroups_learned(system_id, tgid, name, alpha_tag) VALUES( - NEW.system, NEW.talkgroup, NEW.tg_label, NEW.tg_alpha_tag - ) ON CONFLICT DO NOTHING; - END IF; - RETURN NEW; -END -$$ LANGUAGE plpgsql; - -CREATE TABLE IF NOT EXISTS calls( - id UUID PRIMARY KEY, - submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, - system INTEGER NOT NULL, - talkgroup INTEGER NOT NULL, - call_date TIMESTAMPTZ NOT NULL, - audio_name TEXT, - audio_blob BYTEA, - duration INTEGER, - audio_type TEXT, - audio_url TEXT, - frequency INTEGER NOT NULL, - frequencies INTEGER[], - patches INTEGER[], - tg_label TEXT, - tg_alpha_tag TEXT, - tg_group TEXT, - source INTEGER NOT NULL, - transcript TEXT -); - -CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls -FOR EACH ROW EXECUTE FUNCTION learn_talkgroup(); - -CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript)); -CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date); - -CREATE TABLE IF NOT EXISTS settings( - name TEXT PRIMARY KEY, - updated_by INTEGER REFERENCES users(id), - value JSONB -); - -CREATE TABLE IF NOT EXISTS incidents( - id UUID PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - start_time TIMESTAMP, - end_time TIMESTAMP, - location JSONB, - metadata JSONB -); - -CREATE INDEX IF NOT EXISTS incidents_name_description_idx ON incidents USING GIN ( - (to_tsvector('english', name) || to_tsvector('english', coalesce(description, '')) - ) -); - -CREATE TABLE IF NOT EXISTS incidents_calls( - incident_id UUID REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, - call_id UUID REFERENCES calls(id) ON UPDATE CASCADE, - notes JSONB, - PRIMARY KEY (incident_id, call_id) -); diff --git a/sql/postgres/queries/talkgroups.sql b/sql/postgres/queries/talkgroups.sql index f2e7a1e..894f4d8 100644 --- a/sql/postgres/queries/talkgroups.sql +++ b/sql/postgres/queries/talkgroups.sql @@ -8,25 +8,21 @@ WHERE tags && ARRAY[$1]; -- name: GetTalkgroupIDsByTags :many SELECT system_id, tgid FROM talkgroups -WHERE (tags @> ARRAY[sqlc.arg(anyTags)]) -AND (tags && ARRAY[sqlc.arg(allTags)]) -AND NOT (tags @> ARRAY[sqlc.arg(notTags)]); +WHERE (tags @> ARRAY[@anyTags]) +AND (tags && ARRAY[@allTags]) +AND NOT (tags @> ARRAY[@notTags]); -- name: GetTalkgroupTags :one SELECT tags FROM talkgroups -WHERE id = systg2id($1, $2); +WHERE system_id = @system_id AND tgid = @tg_id; -- name: SetTalkgroupTags :exec -UPDATE talkgroups SET tags = $3 -WHERE id = systg2id($1, $2); - --- name: BulkSetTalkgroupTags :exec -UPDATE talkgroups SET tags = $2 -WHERE id = ANY($1); +UPDATE talkgroups SET tags = @tags +WHERE system_id = @system_id AND tgid = @tg_id; -- name: GetTalkgroup :one SELECT sqlc.embed(talkgroups) FROM talkgroups -WHERE (system_id, tgid) = (@system_id, @tgid); +WHERE (system_id, tgid) = (@system_id, @tg_id); -- name: GetTalkgroupWithLearned :one SELECT @@ -34,17 +30,17 @@ sqlc.embed(tg), sqlc.embed(sys), FALSE learned FROM talkgroups tg JOIN systems sys ON tg.system_id = sys.id -WHERE (tg.system_id, tg.tgid) = (sqlc.arg(system_id), sqlc.arg(tgid)) +WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid) UNION SELECT -tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, +NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END, TRUE, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned FROM talkgroups_learned tgl JOIN systems sys ON tgl.system_id = sys.id -WHERE tgl.system_id = sqlc.arg(system_id) AND tgl.tgid = sqlc.arg(tgid) AND ignored IS NOT TRUE; +WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE; -- name: GetTalkgroupsWithLearnedBySystem :many SELECT @@ -82,7 +78,7 @@ JOIN systems sys ON tgl.system_id = sys.id WHERE ignored IS NOT TRUE; -- name: GetSystemName :one -SELECT name FROM systems WHERE id = sqlc.arg(system_id); +SELECT name FROM systems WHERE id = @system_id; -- name: UpdateTalkgroup :one UPDATE talkgroups