diff --git a/pkg/alerting/stats.go b/pkg/alerting/stats.go index 2daa497..642a8b3 100644 --- a/pkg/alerting/stats.go +++ b/pkg/alerting/stats.go @@ -40,7 +40,8 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() db := database.FromCtx(ctx) - tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple()) + tgt := as.scoredTGsTuple() + tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, tgt) if err != nil { log.Error().Err(err).Msg("stats TG get failed") http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/pkg/calls/call.go b/pkg/calls/call.go index aaec385..d76b530 100644 --- a/pkg/calls/call.go +++ b/pkg/calls/call.go @@ -124,6 +124,7 @@ func (c *Call) LearnTG(ctx context.Context, db database.Store) (learnedId int, e TGID: c.Talkgroup, Name: c.TalkgroupLabel, AlphaTag: c.TGAlphaTag, + TGGroup: c.TalkgroupGroup, }) } diff --git a/pkg/database/talkgroups.go b/pkg/database/talkgroups.go index b233c62..a461dcd 100644 --- a/pkg/database/talkgroups.go +++ b/pkg/database/talkgroups.go @@ -15,7 +15,7 @@ type talkgroupQuerier interface { type TGTuples [2][]uint32 -const TGConstraintName = "" +const TGConstraintName = "calls_system_talkgroup_fkey" func IsTGConstraintViolation(e error) bool { var err *pgconn.PgError @@ -51,7 +51,7 @@ SELECT tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name, tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB, CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END, -TRUE, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name +TRUE, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned FROM talkgroups_learned tgl JOIN systems sys ON tgl.system_id = sys.id JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);` diff --git a/pkg/sinks/database.go b/pkg/sinks/database.go index a9e0cee..9ced102 100644 --- a/pkg/sinks/database.go +++ b/pkg/sinks/database.go @@ -27,21 +27,12 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error { return nil } - return s.db.InTx(ctx, func(tx database.Store) error { - params := s.toAddCallParams(call) + params := s.toAddCallParams(call) + + err := s.db.InTx(ctx, func(tx database.Store) error { err := tx.AddCall(ctx, params) if err != nil { - if database.IsTGConstraintViolation(err) { - _, err := call.LearnTG(ctx, tx) - if err != nil { - return fmt.Errorf("add call: learn tg: %w", err) - } - err = tx.AddCall(ctx, params) - if err != nil { - return fmt.Errorf("add call: retry: %w", err) - } - } return fmt.Errorf("add call: %w", err) } @@ -49,6 +40,24 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error { return nil }, pgx.TxOptions{}) + + if err != nil && database.IsTGConstraintViolation(err) { + return s.db.InTx(ctx, func(tx database.Store) error { + _, err := call.LearnTG(ctx, tx) + if err != nil { + return fmt.Errorf("add call: learn tg: %w", err) + } + + err = tx.AddCall(ctx, params) + if err != nil { + return fmt.Errorf("add call: retry: %w", err) + } + + return nil + }, pgx.TxOptions{}) + } + + return err } func (s *DatabaseSink) SinkType() string { diff --git a/pkg/talkgroups/cache.go b/pkg/talkgroups/cache.go index 9d0184b..0d58a16 100644 --- a/pkg/talkgroups/cache.go +++ b/pkg/talkgroups/cache.go @@ -252,7 +252,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) { case pgx.ErrNoRows: return nil, ErrNotFound default: - log.Error().Err(err).Msg("TG() cache add db get") + log.Error().Err(err).Uint32("sys", tg.System).Uint32("tg", tg.Talkgroup).Msg("TG() cache add db get") return nil, errors.Join(ErrNotFound, err) } diff --git a/sql/postgres/migrations/001_initial.up.sql b/sql/postgres/migrations/001_initial.up.sql index ff926bf..40d49ee 100644 --- a/sql/postgres/migrations/001_initial.up.sql +++ b/sql/postgres/migrations/001_initial.up.sql @@ -44,7 +44,7 @@ CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid); CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags); CREATE TABLE IF NOT EXISTS talkgroups_learned( - id UUID PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, system_id INTEGER REFERENCES systems(id) NOT NULL, tgid INTEGER NOT NULL, name TEXT NOT NULL, @@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS talkgroups_learned( ); CREATE TABLE IF NOT EXISTS alerts( - id UUID PRIMARY KEY, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, time TIMESTAMPTZ NOT NULL, tgid INTEGER NOT NULL, system_id INTEGER REFERENCES systems(id) NOT NULL, diff --git a/sql/postgres/migrations/002_tglearned.down.sql b/sql/postgres/migrations/002_tglearned.down.sql index 846075d..7355bf1 100644 --- a/sql/postgres/migrations/002_tglearned.down.sql +++ b/sql/postgres/migrations/002_tglearned.down.sql @@ -1,11 +1,11 @@ -ALTER TABLE calls DROP CONSTRAINT IF EXISTS calls_talkgroup_id_fkey; - -ALTER TABLE talkgroups ALTER COLUMN id DROP IDENTITY IF EXISTS; -ALTER TABLE talkgroups_learned ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid()); -DROP SEQUENCE IF EXISTS talkgroups_id_seq; +ALTER TABLE calls DROP CONSTRAINT IF EXISTS calls_system_talkgroup_fkey; ALTER TABLE talkgroups DROP COLUMN IF EXISTS learned; +ALTER TABLE talkgroups ALTER COLUMN id DROP IDENTITY IF EXISTS; +ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid()); +DROP SEQUENCE IF EXISTS talkgroups_id_seq; + CREATE OR REPLACE FUNCTION learn_talkgroup() RETURNS TRIGGER AS $$ BEGIN @@ -23,12 +23,3 @@ END $$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls FOR EACH ROW EXECUTE FUNCTION learn_talkgroup(); - -ALTER TABLE talkgroups_learned ALTER COLUMN id DROP IDENTITY IF EXISTS; -ALTER TABLE talkgroups_learned ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid()); -DROP SEQUENCE IF EXISTS talkgroups_learned_id_seq; - -ALTER TABLE alerts ALTER COLUMN id DROP IDENTITY IF EXISTS; -ALTER TABLE alerts ALTER COLUMN id SET DATA TYPE UUID USING (gen_random_uuid()); -DROP SEQUENCE IF EXISTS alerts_id_seq; - diff --git a/sql/postgres/migrations/002_tglearned.up.sql b/sql/postgres/migrations/002_tglearned.up.sql index 6fd1f00..21f5cd2 100644 --- a/sql/postgres/migrations/002_tglearned.up.sql +++ b/sql/postgres/migrations/002_tglearned.up.sql @@ -1,21 +1,24 @@ -CREATE SEQUENCE IF NOT EXISTS alerts_id_seq START WITH 1; -ALTER TABLE alerts ALTER COLUMN id SET DATA TYPE INTEGER USING (nextval('alerts_id_seq')); -ALTER TABLE alerts ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY; -DROP SEQUENCE IF EXISTS alerts_id_seq; - -CREATE SEQUENCE IF NOT EXISTS talkgroups_learned_id_seq START WITH 1; -ALTER TABLE talkgroups_learned ALTER COLUMN id SET DATA TYPE INTEGER USING (nextval('talkgroups_learned_id_seq')); -ALTER TABLE talkgroups_learned ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY; -DROP SEQUENCE IF EXISTS talkgroup_learned_id_seq; - DROP TRIGGER IF EXISTS learn_tg ON calls; DROP FUNCTION IF EXISTS learn_talkgroup(); -ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS learned BOOLEAN NOT NULL DEFAULT FALSE; - CREATE SEQUENCE IF NOT EXISTS talkgroups_id_seq START WITH 1; ALTER TABLE talkgroups ALTER COLUMN id SET DATA TYPE INTEGER USING (nextval('talkgroups_id_seq')); ALTER TABLE talkgroups ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY; DROP SEQUENCE IF EXISTS talkgroups_id_seq; -ALTER TABLE calls ADD CONSTRAINT calls_talkgroup_id_fkey FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid); +ALTER TABLE talkgroups ADD COLUMN IF NOT EXISTS learned BOOLEAN NOT NULL DEFAULT FALSE; + +-- calls fkey constraint requires us to migrate all calls' talkgroup tuples to exist +INSERT INTO talkgroups (system_id, tgid, learned) +SELECT DISTINCT system_id, tgid, TRUE FROM talkgroups_learned ON CONFLICT DO NOTHING; + +INSERT INTO talkgroups (system_id, tgid, learned) +SELECT DISTINCT system, talkgroup, TRUE FROM calls ON CONFLICT DO NOTHING; + +INSERT INTO talkgroups_learned (system_id, tgid, name, tg_group, alpha_tag) +SELECT DISTINCT c.system, c.talkgroup, c.tg_label, c.tg_group, c.tg_alpha_tag +FROM calls c +JOIN talkgroups t ON (t.system_id = c.system AND t.tgid = c.talkgroup AND t.learned IS TRUE) +ON CONFLICT DO NOTHING; + +ALTER TABLE calls ADD CONSTRAINT calls_system_talkgroup_fkey FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid); diff --git a/sql/postgres/migrations/initial.sql b/sql/postgres/migrations/initial.sql index 1006883..445fa4f 100644 --- a/sql/postgres/migrations/initial.sql +++ b/sql/postgres/migrations/initial.sql @@ -85,7 +85,8 @@ CREATE TABLE IF NOT EXISTS calls( tg_alpha_tag TEXT, tg_group TEXT, source INTEGER NOT NULL, - transcript TEXT + transcript TEXT, + FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) ); CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript));