Merge pull request 'callsnormalize' (#40 ) from callsnormalize into trunk

Reviewed-on: #40
don't alert ignored
2024-11-19 10:18:12 -05:00 · 2024-11-19 10:18:12 -05:00 · 2024-11-19 10:18:12 -05:00 · 2024-11-19 10:18:12 -05:00 · 2024-11-19 10:18:12 -05:00 · 2024-11-19 10:18:12 -05:00
30 changed files with 2123 additions and 1847 deletions
--- a/.mockery.yaml
+++ b/.mockery.yaml
@ -7,4 +7,4 @@ packages:
  dynatron.me/x/stillbox/pkg/database:
    config:
    interfaces:
-      DB:
+      Store:
--- a/cmd/stillbox/main.go
+++ b/cmd/stillbox/main.go
@ -35,8 +35,5 @@ func main() {
 	cmds := append([]*cobra.Command{serve.Command(cfg)}, admin.Command(cfg)...)
 	rootCmd.AddCommand(cmds...)

-	err := rootCmd.Execute()
-	if err != nil {
-		log.Fatal().Err(err).Msg("Dying")
-	}
+	rootCmd.Execute()
 }
--- a/pkg/alerting/alert/alert.go
+++ b/pkg/alerting/alert/alert.go
@ -10,12 +10,11 @@ import (
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"

-	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
 )

 type Alert struct {
-	ID         uuid.UUID
+	ID         int
 	Timestamp  time.Time
 	TGName     string
 	Score      trending.Score[talkgroups.ID]
@ -34,7 +33,6 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
 	}

 	return database.AddAlertParams{
-		ID:        a.ID,
 		Time:      pgtype.Timestamptz{Time: a.Timestamp, Valid: true},
 		SystemID:  int(a.Score.ID.System),
 		TGID:      int(a.Score.ID.Talkgroup),
@ -48,7 +46,6 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
 // Make creates an alert for later rendering or storage.
 func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
 	d := Alert{
-		ID:        uuid.New(),
 		Score:     score,
 		Timestamp: time.Now(),
 		Weight:    1.0,
--- a/pkg/alerting/alerting.go
+++ b/pkg/alerting/alerting.go
@ -161,7 +161,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
 	for _, s := range as.scores {
 		origScore := s.Score
 		tgr, err := as.tgCache.TG(ctx, s.ID)
-		if err == nil && !tgr.Talkgroup.Alert {
+		if err != nil || !tgr.Talkgroup.Alert {
 			continue
 		}

--- a/pkg/alerting/stats.go
+++ b/pkg/alerting/stats.go
@ -40,7 +40,8 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	db := database.FromCtx(ctx)

-	tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple())
+	tgt := as.scoredTGsTuple()
+	tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, tgt)
 	if err != nil {
 		log.Error().Err(err).Msg("stats TG get failed")
 		http.Error(w, err.Error(), http.StatusInternalServerError)
--- a/pkg/auth/jwt.go
+++ b/pkg/auth/jwt.go
@ -98,7 +98,7 @@ func (a *Auth) Login(ctx context.Context, username, password string) (token stri
 	return a.newToken(found.ID), nil
 }

-func (a *Auth) newToken(uid int32) string {
+func (a *Auth) newToken(uid int) string {
 	claims := claims{
 		"sub": strconv.Itoa(int(uid)),
 	}
@ -134,7 +134,7 @@ func (a *Auth) routeRefresh(w http.ResponseWriter, r *http.Request) {
 		return
 	}

-	tok := a.newToken(int32(uid))
+	tok := a.newToken(uid)

 	cookie := &http.Cookie{
 		Name:     "jwt",
--- a/pkg/calls/call.go
+++ b/pkg/calls/call.go
@ -1,11 +1,13 @@
 package calls

 import (
+	"context"
 	"fmt"
 	"time"

 	"dynatron.me/x/stillbox/internal/audio"
 	"dynatron.me/x/stillbox/pkg/auth"
+	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/pb"
 	"dynatron.me/x/stillbox/pkg/talkgroups"

@ -111,6 +113,21 @@ func (c *Call) ToPB() *pb.Call {
 	}
 }

+func (c *Call) LearnTG(ctx context.Context, db database.Store) (learnedId int, err error) {
+	err = db.AddTalkgroupWithLearnedFlag(ctx, int32(c.System), int32(c.Talkgroup))
+	if err != nil {
+		return 0, fmt.Errorf("addTalkgroupWithLearnedFlag: %w", err)
+	}
+
+	return db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
+		SystemID: c.System,
+		TGID:     c.Talkgroup,
+		Name:     c.TalkgroupLabel,
+		AlphaTag: c.TGAlphaTag,
+		TGGroup:  c.TalkgroupGroup,
+	})
+}
+
 func (c *Call) computeLength() (err error) {
 	var td time.Duration

--- a/pkg/database/calls.sql.go
+++ b/pkg/database/calls.sql.go
@ -13,7 +13,7 @@ import (
 )

 const addAlert = `-- name: AddAlert :exec
-INSERT INTO alerts (id, time, tgid, system_id, weight, score, orig_score, notified, metadata)
+INSERT INTO alerts (time, tgid, system_id, weight, score, orig_score, notified, metadata)
 VALUES
 (
 	$1,
@ -23,13 +23,11 @@ VALUES
 	$5,
 	$6,
 	$7,
-	$8,
-	$9
+	$8
 )
 `

 type AddAlertParams struct {
-	ID        uuid.UUID          `json:"id"`
 	Time      pgtype.Timestamptz `json:"time"`
 	TGID      int                `json:"tgid"`
 	SystemID  int                `json:"system_id"`
@ -42,7 +40,6 @@ type AddAlertParams struct {

 func (q *Queries) AddAlert(ctx context.Context, arg AddAlertParams) error {
 	_, err := q.db.Exec(ctx, addAlert,
-		arg.ID,
 		arg.Time,
 		arg.TGID,
 		arg.SystemID,
@ -109,9 +106,9 @@ type AddCallParams struct {
 	Frequency   int                `json:"frequency"`
 	Frequencies []int              `json:"frequencies"`
 	Patches     []int              `json:"patches"`
-	TgLabel     *string            `json:"tg_label"`
-	TgAlphaTag  *string            `json:"tg_alpha_tag"`
-	TgGroup     *string            `json:"tg_group"`
+	TGLabel     *string            `json:"tg_label"`
+	TGAlphaTag  *string            `json:"tg_alpha_tag"`
+	TGGroup     *string            `json:"tg_group"`
 	Source      int                `json:"source"`
 }

@ -130,9 +127,9 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) error {
 		arg.Frequency,
 		arg.Frequencies,
 		arg.Patches,
-		arg.TgLabel,
-		arg.TgAlphaTag,
-		arg.TgGroup,
+		arg.TGLabel,
+		arg.TGAlphaTag,
+		arg.TGGroup,
 		arg.Source,
 	)
 	return err
--- a/pkg/database/database.go
+++ b/pkg/database/database.go
@ -3,6 +3,7 @@ package database
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"

 	"dynatron.me/x/stillbox/pkg/config"
@ -19,11 +20,12 @@ import (
 // DB is a database handle.

 //go:generate mockery
-type DB interface {
+type Store interface {
 	Querier
 	talkgroupQuerier

 	DB() *Database
+	InTx(context.Context, func(Store) error, pgx.TxOptions) error
 }

 type Database struct {
@ -35,14 +37,41 @@ func (db *Database) DB() *Database {
 	return db
 }

+func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOptions) error {
+	tx, err := db.DB().Pool.BeginTx(ctx, opts)
+	if err != nil {
+		return fmt.Errorf("Tx begin: %w", err)
+	}
+
+	defer tx.Rollback(ctx)
+
+	dbtx := &Database{Pool: db.Pool, Queries: db.Queries.WithTx(tx)}
+
+	err = f(dbtx)
+	if err != nil {
+		return fmt.Errorf("Tx: %w", err)
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return fmt.Errorf("Tx commit: %w", err)
+	}
+
+	return nil
+}
+
 type dbLogger struct{}

 func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
 	log.Debug().Fields(data).Msg(msg)
 }

+func Close(c Store) {
+	c.(*Database).Pool.Close()
+}
+
 // NewClient creates a new DB using the provided config.
-func NewClient(ctx context.Context, conf config.DB) (DB, error) {
+func NewClient(ctx context.Context, conf config.DB) (Store, error) {
 	dir, err := iofs.New(sqlembed.Migrations, "postgres/migrations")
 	if err != nil {
 		return nil, err
@ -90,8 +119,8 @@ type dBCtxKey string
 const DBCtxKey dBCtxKey = "dbctx"

 // FromCtx returns the database handle from the provided Context.
-func FromCtx(ctx context.Context) DB {
-	c, ok := ctx.Value(DBCtxKey).(DB)
+func FromCtx(ctx context.Context) Store {
+	c, ok := ctx.Value(DBCtxKey).(Store)
 	if !ok {
 		panic("no DB in context")
 	}
@ -100,7 +129,7 @@ func FromCtx(ctx context.Context) DB {
 }

 // CtxWithDB returns a Context with the provided database handle.
-func CtxWithDB(ctx context.Context, conn DB) context.Context {
+func CtxWithDB(ctx context.Context, conn Store) context.Context {
 	return context.WithValue(ctx, DBCtxKey, conn)
 }

--- a/pkg/database/extend.go
+++ b/pkg/database/extend.go
@ -2,16 +2,16 @@ package database

 func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup                    { return d.Talkgroup }
 func (d GetTalkgroupsRow) GetSystem() System                          { return d.System }
-func (d GetTalkgroupsRow) GetLearned() bool                           { return d.Learned }
+func (d GetTalkgroupsRow) GetLearned() bool                           { return d.Talkgroup.Learned }
 func (g GetTalkgroupWithLearnedRow) GetTalkgroup() Talkgroup          { return g.Talkgroup }
 func (g GetTalkgroupWithLearnedRow) GetSystem() System                { return g.System }
-func (g GetTalkgroupWithLearnedRow) GetLearned() bool                 { return g.Learned }
+func (g GetTalkgroupWithLearnedRow) GetLearned() bool                 { return g.Talkgroup.Learned }
 func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup         { return g.Talkgroup }
 func (g GetTalkgroupsWithLearnedRow) GetSystem() System               { return g.System }
-func (g GetTalkgroupsWithLearnedRow) GetLearned() bool                { return g.Learned }
+func (g GetTalkgroupsWithLearnedRow) GetLearned() bool                { return g.Talkgroup.Learned }
 func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
 func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System       { return g.System }
-func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool        { return g.Learned }
+func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool        { return g.Talkgroup.Learned }
 func (g Talkgroup) GetTalkgroup() Talkgroup                           { return g }
 func (g Talkgroup) GetSystem() System                                 { return System{ID: int(g.SystemID)} }
 func (g Talkgroup) GetLearned() bool                                  { return false }
--- a/pkg/database/mocks/DB.go
+++ b/pkg/database/mocks/DB.go
--- a/pkg/database/mocks/Store.go
+++ b/pkg/database/mocks/Store.go
--- a/pkg/database/models.go
+++ b/pkg/database/models.go
@ -14,7 +14,7 @@ import (
 )

 type Alert struct {
-	ID        uuid.UUID          `json:"id"`
+	ID        int                `json:"id"`
 	Time      pgtype.Timestamptz `json:"time"`
 	TGID      int                `json:"tgid"`
 	SystemID  int                `json:"system_id"`
@ -26,7 +26,7 @@ type Alert struct {
 }

 type ApiKey struct {
-	ID        int32            `json:"id"`
+	ID        int              `json:"id"`
 	Owner     int              `json:"owner"`
 	CreatedAt time.Time        `json:"created_at"`
 	Expires   pgtype.Timestamp `json:"expires"`
@ -48,9 +48,9 @@ type Call struct {
 	Frequency   int                `json:"frequency"`
 	Frequencies []int              `json:"frequencies"`
 	Patches     []int              `json:"patches"`
-	TgLabel     *string            `json:"tg_label"`
-	TgAlphaTag  *string            `json:"tg_alpha_tag"`
-	TgGroup     *string            `json:"tg_group"`
+	TGLabel     *string            `json:"tg_label"`
+	TGAlphaTag  *string            `json:"tg_alpha_tag"`
+	TGGroup     *string            `json:"tg_group"`
 	Source      int                `json:"source"`
 	Transcript  *string            `json:"transcript"`
 }
@ -83,31 +83,33 @@ type System struct {
 }

 type Talkgroup struct {
-	ID          uuid.UUID          `json:"id"`
+	ID          int                `json:"id"`
 	SystemID    int32              `json:"system_id"`
 	TGID        int32              `json:"tgid"`
 	Name        *string            `json:"name"`
 	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string            `json:"tg_group"`
+	TGGroup     *string            `json:"tg_group"`
 	Frequency   *int32             `json:"frequency"`
 	Metadata    jsontypes.Metadata `json:"metadata"`
 	Tags        []string           `json:"tags"`
 	Alert       bool               `json:"alert"`
 	AlertConfig rules.AlertRules   `json:"alert_config"`
 	Weight      float32            `json:"weight"`
+	Learned     bool               `json:"learned"`
 }

 type TalkgroupsLearned struct {
-	ID       uuid.UUID `json:"id"`
-	SystemID int       `json:"system_id"`
-	TGID     int       `json:"tgid"`
-	Name     string    `json:"name"`
-	AlphaTag *string   `json:"alpha_tag"`
-	Ignored  *bool     `json:"ignored"`
+	ID       int     `json:"id"`
+	SystemID int     `json:"system_id"`
+	TGID     int     `json:"tgid"`
+	Name     string  `json:"name"`
+	AlphaTag *string `json:"alpha_tag"`
+	TGGroup  *string `json:"tg_group"`
+	Ignored  *bool   `json:"ignored"`
 }

 type User struct {
-	ID       int32  `json:"id"`
+	ID       int    `json:"id"`
 	Username string `json:"username"`
 	Password string `json:"password"`
 	Email    string `json:"email"`
--- a/pkg/database/querier.go
+++ b/pkg/database/querier.go
@ -14,6 +14,8 @@ import (
 type Querier interface {
 	AddAlert(ctx context.Context, arg AddAlertParams) error
 	AddCall(ctx context.Context, arg AddCallParams) error
+	AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error)
+	AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error
 	CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
 	CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
 	DeleteAPIKey(ctx context.Context, apiKey string) error
@ -21,20 +23,20 @@ type Querier interface {
 	GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error)
 	GetDatabaseSize(ctx context.Context) (string, error)
 	GetSystemName(ctx context.Context, systemID int) (string, error)
-	GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error)
+	GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error)
 	GetTalkgroupIDsByTags(ctx context.Context, anyTags []string, allTags []string, notTags []string) ([]GetTalkgroupIDsByTagsRow, error)
-	GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error)
+	GetTalkgroupTags(ctx context.Context, systemID int32, tGID int32) ([]string, error)
 	GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error)
 	GetTalkgroupsWithAllTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAllTagsRow, error)
 	GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAnyTagsRow, error)
 	GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroupsWithLearnedRow, error)
 	GetTalkgroupsWithLearnedBySystem(ctx context.Context, system int32) ([]GetTalkgroupsWithLearnedBySystemRow, error)
-	GetUserByID(ctx context.Context, id int32) (User, error)
-	GetUserByUID(ctx context.Context, id int32) (User, error)
+	GetUserByID(ctx context.Context, id int) (User, error)
+	GetUserByUID(ctx context.Context, id int) (User, error)
 	GetUserByUsername(ctx context.Context, username string) (User, error)
 	GetUsers(ctx context.Context) ([]User, error)
 	SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
-	SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error
+	SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error
 	UpdatePassword(ctx context.Context, username string, password string) error
 	UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
 }
--- a/pkg/database/talkgroups.go
+++ b/pkg/database/talkgroups.go
@ -2,6 +2,9 @@ package database

 import (
 	"context"
+	"errors"
+
+	"github.com/jackc/pgx/v5/pgconn"
 )

 type talkgroupQuerier interface {
@ -12,6 +15,17 @@ type talkgroupQuerier interface {

 type TGTuples [2][]uint32

+const TGConstraintName = "calls_system_talkgroup_fkey"
+
+func IsTGConstraintViolation(e error) bool {
+	var err *pgconn.PgError
+	if errors.As(e, &err) && err.Code == "23503" && err.ConstraintName == TGConstraintName {
+		return true
+	}
+
+	return false
+}
+
 func MakeTGTuples(cap int) TGTuples {
 	return [2][]uint32{
 		make([]uint32, 0, cap),
@ -27,18 +41,17 @@ func (t *TGTuples) Append(sys, tg uint32) {
 // Below queries are here because sqlc refuses to parse unnest(x, y)

 const getTalkgroupsWithLearnedBySysTGID = `SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
 JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
+WHERE tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);`
@ -46,7 +59,6 @@ JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys
 type GetTalkgroupsRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 	System    System    `json:"system"`
-	Learned   bool      `json:"learned"`
 }

 func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
@ -64,7 +76,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
@ -73,7 +85,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
 			&i.Talkgroup.Weight,
 			&i.System.ID,
 			&i.System.Name,
-			&i.Learned,
+			&i.Talkgroup.Learned,
 		); err != nil {
 			return nil, err
 		}
@ -87,7 +99,8 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT

 const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
+JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
+WHERE tg.learned IS NOT TRUE;`

 func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
 	rows, err := q.db.Query(ctx, getTalkgroupsBySysTGID, ids[0], ids[1])
@ -104,7 +117,7 @@ func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]G
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
--- a/pkg/database/talkgroups.sql.go
+++ b/pkg/database/talkgroups.sql.go
@ -10,9 +10,62 @@ import (

 	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/alerting/rules"
-	"github.com/jackc/pgx/v5/pgtype"
 )

+const addLearnedTalkgroup = `-- name: AddLearnedTalkgroup :one
+INSERT INTO talkgroups_learned(
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group
+) VALUES (
+	$1,
+	$2,
+	$3,
+	$4,
+	$5
+) RETURNING id
+`
+
+type AddLearnedTalkgroupParams struct {
+	SystemID int     `json:"system_id"`
+	TGID     int     `json:"tgid"`
+	Name     *string `json:"name"`
+	AlphaTag *string `json:"alpha_tag"`
+	TGGroup  *string `json:"tg_group"`
+}
+
+func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error) {
+	row := q.db.QueryRow(ctx, addLearnedTalkgroup,
+		arg.SystemID,
+		arg.TGID,
+		arg.Name,
+		arg.AlphaTag,
+		arg.TGGroup,
+	)
+	var id int
+	err := row.Scan(&id)
+	return id, err
+}
+
+const addTalkgroupWithLearnedFlag = `-- name: AddTalkgroupWithLearnedFlag :exec
+INSERT INTO talkgroups (
+	system_id,
+	tgid,
+	learned
+) VALUES(
+	$1,
+	$2,
+	't'
+)
+`
+
+func (q *Queries) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
+	_, err := q.db.Exec(ctx, addTalkgroupWithLearnedFlag, systemID, tGID)
+	return err
+}
+
 const getSystemName = `-- name: GetSystemName :one
 SELECT name FROM systems WHERE id = $1
 `
@ -25,7 +78,7 @@ func (q *Queries) GetSystemName(ctx context.Context, systemID int) (string, erro
 }

 const getTalkgroup = `-- name: GetTalkgroup :one
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
 WHERE (system_id, tgid) = ($1, $2)
 `

@ -33,8 +86,8 @@ type GetTalkgroupRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 }

-func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) {
-	row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgID)
+func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error) {
+	row := q.db.QueryRow(ctx, getTalkgroup, systemID, tGID)
 	var i GetTalkgroupRow
 	err := row.Scan(
 		&i.Talkgroup.ID,
@ -42,13 +95,14 @@ func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32)
 		&i.Talkgroup.TGID,
 		&i.Talkgroup.Name,
 		&i.Talkgroup.AlphaTag,
-		&i.Talkgroup.TgGroup,
+		&i.Talkgroup.TGGroup,
 		&i.Talkgroup.Frequency,
 		&i.Talkgroup.Metadata,
 		&i.Talkgroup.Tags,
 		&i.Talkgroup.Alert,
 		&i.Talkgroup.AlertConfig,
 		&i.Talkgroup.Weight,
+		&i.Talkgroup.Learned,
 	)
 	return i, err
 }
@ -90,8 +144,8 @@ SELECT tags FROM talkgroups
 WHERE system_id = $1 AND tgid = $2
 `

-func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) {
-	row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tgID)
+func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tGID int32) ([]string, error) {
+	row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tGID)
 	var tags []string
 	err := row.Scan(&tags)
 	return tags, err
@ -99,18 +153,16 @@ func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int

 const getTalkgroupWithLearned = `-- name: GetTalkgroupWithLearned :one
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = ($1, $2)
+WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
@ -119,7 +171,6 @@ WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
 type GetTalkgroupWithLearnedRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 	System    System    `json:"system"`
-	Learned   bool      `json:"learned"`
 }

 func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error) {
@ -131,22 +182,22 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
 		&i.Talkgroup.TGID,
 		&i.Talkgroup.Name,
 		&i.Talkgroup.AlphaTag,
-		&i.Talkgroup.TgGroup,
+		&i.Talkgroup.TGGroup,
 		&i.Talkgroup.Frequency,
 		&i.Talkgroup.Metadata,
 		&i.Talkgroup.Tags,
 		&i.Talkgroup.Alert,
 		&i.Talkgroup.AlertConfig,
 		&i.Talkgroup.Weight,
+		&i.Talkgroup.Learned,
 		&i.System.ID,
 		&i.System.Name,
-		&i.Learned,
 	)
 	return i, err
 }

 const getTalkgroupsWithAllTags = `-- name: GetTalkgroupsWithAllTags :many
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
 WHERE tags && ARRAY[$1]
 `

@ -169,13 +220,14 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
 			&i.Talkgroup.Alert,
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
+			&i.Talkgroup.Learned,
 		); err != nil {
 			return nil, err
 		}
@ -188,7 +240,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
 }

 const getTalkgroupsWithAnyTags = `-- name: GetTalkgroupsWithAnyTags :many
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
 WHERE tags @> ARRAY[$1]
 `

@ -211,13 +263,14 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
 			&i.Talkgroup.Alert,
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
+			&i.Talkgroup.Learned,
 		); err != nil {
 			return nil, err
 		}
@ -231,17 +284,16 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (

 const getTalkgroupsWithLearned = `-- name: GetTalkgroupsWithLearned :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
+WHERE tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE
@ -250,7 +302,6 @@ WHERE ignored IS NOT TRUE
 type GetTalkgroupsWithLearnedRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 	System    System    `json:"system"`
-	Learned   bool      `json:"learned"`
 }

 func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroupsWithLearnedRow, error) {
@ -268,16 +319,16 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
 			&i.Talkgroup.Alert,
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
+			&i.Talkgroup.Learned,
 			&i.System.ID,
 			&i.System.Name,
-			&i.Learned,
 		); err != nil {
 			return nil, err
 		}
@ -291,18 +342,16 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups

 const getTalkgroupsWithLearnedBySystem = `-- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = $1
+WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
@ -311,7 +360,6 @@ WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
 type GetTalkgroupsWithLearnedBySystemRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 	System    System    `json:"system"`
-	Learned   bool      `json:"learned"`
 }

 func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system int32) ([]GetTalkgroupsWithLearnedBySystemRow, error) {
@ -329,16 +377,16 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
 			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.TGGroup,
 			&i.Talkgroup.Frequency,
 			&i.Talkgroup.Metadata,
 			&i.Talkgroup.Tags,
 			&i.Talkgroup.Alert,
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
+			&i.Talkgroup.Learned,
 			&i.System.ID,
 			&i.System.Name,
-			&i.Learned,
 		); err != nil {
 			return nil, err
 		}
@ -355,8 +403,8 @@ UPDATE talkgroups SET tags = $1
 WHERE system_id = $2 AND tgid = $3
 `

-func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error {
-	_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tgID)
+func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error {
+	_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tGID)
 	return err
 }

@ -373,20 +421,20 @@ SET
 	alert_config = COALESCE($8, alert_config),
 	weight = COALESCE($9, weight)
 WHERE id = $10 OR (system_id = $11 AND tgid = $12)
-RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight
+RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
 `

 type UpdateTalkgroupParams struct {
 	Name        *string            `json:"name"`
 	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string            `json:"tg_group"`
+	TGGroup     *string            `json:"tg_group"`
 	Frequency   *int32             `json:"frequency"`
 	Metadata    jsontypes.Metadata `json:"metadata"`
 	Tags        []string           `json:"tags"`
 	Alert       *bool              `json:"alert"`
 	AlertConfig rules.AlertRules   `json:"alert_config"`
 	Weight      *float32           `json:"weight"`
-	ID          pgtype.UUID        `json:"id"`
+	ID          *int32             `json:"id"`
 	SystemID    *int32             `json:"system_id"`
 	TGID        *int32             `json:"tgid"`
 }
@ -395,7 +443,7 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
 	row := q.db.QueryRow(ctx, updateTalkgroup,
 		arg.Name,
 		arg.AlphaTag,
-		arg.TgGroup,
+		arg.TGGroup,
 		arg.Frequency,
 		arg.Metadata,
 		arg.Tags,
@ -413,13 +461,14 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
 		&i.TGID,
 		&i.Name,
 		&i.AlphaTag,
-		&i.TgGroup,
+		&i.TGGroup,
 		&i.Frequency,
 		&i.Metadata,
 		&i.Tags,
 		&i.Alert,
 		&i.AlertConfig,
 		&i.Weight,
+		&i.Learned,
 	)
 	return i, err
 }
--- a/pkg/database/talkgroups.sql_test.go
+++ b/pkg/database/talkgroups.sql_test.go
@ -3,23 +3,21 @@ package database
 import (
 	"testing"

-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
 )

 const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = ($1, $2)
+WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
 UNION
 SELECT
-NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
@ -27,18 +25,16 @@ WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE

 const getTalkgroupsWithLearnedBySystemTest = `-- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = $1
+WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
 UNION
 SELECT
-NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
@ -46,24 +42,23 @@ WHERE tgl.system_id = $1 AND ignored IS NOT TRUE

 const getTalkgroupsWithLearnedTest = `-- name: GetTalkgroupsWithLearned :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
+WHERE tg.learned IS NOT TRUE
 UNION
 SELECT
-NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE
 `

 func TestQueryColumnsMatch(t *testing.T) {
-	require.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
-	require.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
-	require.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
+	assert.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
+	assert.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
+	assert.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
 }
--- a/pkg/database/users.sql.go
+++ b/pkg/database/users.sql.go
@ -113,7 +113,7 @@ SELECT id, username, password, email, is_admin, prefs FROM users
 WHERE id = $1 LIMIT 1
 `

-func (q *Queries) GetUserByID(ctx context.Context, id int32) (User, error) {
+func (q *Queries) GetUserByID(ctx context.Context, id int) (User, error) {
 	row := q.db.QueryRow(ctx, getUserByID, id)
 	var i User
 	err := row.Scan(
@ -132,7 +132,7 @@ SELECT id, username, password, email, is_admin, prefs FROM users
 WHERE id = $1 LIMIT 1
 `

-func (q *Queries) GetUserByUID(ctx context.Context, id int32) (User, error) {
+func (q *Queries) GetUserByUID(ctx context.Context, id int) (User, error) {
 	row := q.db.QueryRow(ctx, getUserByUID, id)
 	var i User
 	err := row.Scan(
--- a/pkg/nexus/commands.go
+++ b/pkg/nexus/commands.go
@ -78,7 +78,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
 	resp := &pb.TalkgroupInfo{
 		Tg:         tg,
 		Name:       tgi.Talkgroup.Name,
-		Group:      tgi.Talkgroup.TgGroup,
+		Group:      tgi.Talkgroup.TGGroup,
 		Frequency:  tgi.Talkgroup.Frequency,
 		Metadata:   md,
 		Tags:       tgi.Talkgroup.Tags,
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@ -27,7 +27,7 @@ const shutdownTimeout = 5 * time.Second
 type Server struct {
 	auth     *auth.Auth
 	conf     *config.Config
-	db       database.DB
+	db       database.Store
 	r        *chi.Mux
 	sources  sources.Sources
 	sinks    sinks.Sinks
@ -112,7 +112,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 }

 func (s *Server) Go(ctx context.Context) error {
-	defer s.db.DB().Close()
+	defer database.Close(s.db)

 	s.installHupHandler()

--- a/pkg/sinks/database.go
+++ b/pkg/sinks/database.go
@ -8,16 +8,17 @@ import (
 	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/database"

+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/rs/zerolog/log"
 )

 type DatabaseSink struct {
-	db database.DB
+	db database.Store
 }

-func NewDatabaseSink(db database.DB) *DatabaseSink {
-	return &DatabaseSink{db: db}
+func NewDatabaseSink(store database.Store) *DatabaseSink {
+	return &DatabaseSink{store}
 }

 func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
@ -26,14 +27,37 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
 		return nil
 	}

-	err := s.db.AddCall(ctx, s.toAddCallParams(call))
-	if err != nil {
-		return fmt.Errorf("add call: %w", err)
+	params := s.toAddCallParams(call)
+
+	err := s.db.InTx(ctx, func(tx database.Store) error {
+		err := tx.AddCall(ctx, params)
+		if err != nil {
+
+			return fmt.Errorf("add call: %w", err)
+		}
+
+		log.Debug().Str("id", call.ID.String()).Int("system", call.System).Int("tgid", call.Talkgroup).Msg("stored")
+
+		return nil
+	}, pgx.TxOptions{})
+
+	if err != nil && database.IsTGConstraintViolation(err) {
+		return s.db.InTx(ctx, func(tx database.Store) error {
+			_, err := call.LearnTG(ctx, tx)
+			if err != nil {
+				return fmt.Errorf("add call: learn tg: %w", err)
+			}
+
+			err = tx.AddCall(ctx, params)
+			if err != nil {
+				return fmt.Errorf("add call: retry: %w", err)
+			}
+
+			return nil
+		}, pgx.TxOptions{})
 	}

-	log.Debug().Str("id", call.ID.String()).Int("system", call.System).Int("tgid", call.Talkgroup).Msg("stored")
-
-	return nil
+	return err
 }

 func (s *DatabaseSink) SinkType() string {
@ -54,9 +78,9 @@ func (s *DatabaseSink) toAddCallParams(call *calls.Call) database.AddCallParams
 		Frequency:   call.Frequency,
 		Frequencies: call.Frequencies,
 		Patches:     call.Patches,
-		TgLabel:     call.TalkgroupLabel,
-		TgAlphaTag:  call.TGAlphaTag,
-		TgGroup:     call.TalkgroupGroup,
+		TGLabel:     call.TalkgroupLabel,
+		TGAlphaTag:  call.TGAlphaTag,
+		TGGroup:     call.TalkgroupGroup,
 		Source:      call.Source,
 	}
 }
--- a/pkg/sinks/relay.go
+++ b/pkg/sinks/relay.go
@ -44,21 +44,20 @@ func NewRelayManager(s Sinks, cfgs []config.Relay) (*RelayManager, error) {
 	}

 	for i, cfg := range cfgs {
-		rs, err := rm.newRelay(cfg)
+		rs, err := rm.newRelay(i, cfg)
 		if err != nil {
 			return nil, err
 		}

 		rm.relays = append(rm.relays, rs)

-		sinkName := fmt.Sprintf("relay%d:%s", i, rs.url.Host)
-		s.Register(sinkName, rs, cfg.Required)
+		s.Register(rs.Name, rs, cfg.Required)
 	}

 	return rm, nil
 }

-func (rs *RelayManager) newRelay(cfg config.Relay) (*Relay, error) {
+func (rs *RelayManager) newRelay(idx int, cfg config.Relay) (*Relay, error) {
 	u, err := url.Parse(cfg.URL)
 	if err != nil {
 		return nil, err
@ -71,6 +70,7 @@ func (rs *RelayManager) newRelay(cfg config.Relay) (*Relay, error) {
 	u = u.JoinPath("/api/call-upload")

 	return &Relay{
+		Name:  fmt.Sprintf("relay%d:%s", idx, u.Host),
 		Relay: cfg,
 		url:   u,
 		mgr:   rs,
--- a/pkg/talkgroups/cache.go
+++ b/pkg/talkgroups/cache.go
@ -252,7 +252,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
 	case pgx.ErrNoRows:
 		return nil, ErrNotFound
 	default:
-		log.Error().Err(err).Msg("TG() cache add db get")
+		log.Error().Err(err).Uint32("sys", tg.System).Uint32("tg", tg.Talkgroup).Msg("TG() cache add db get")
 		return nil, errors.Join(ErrNotFound, err)
 	}

--- a/pkg/talkgroups/importer/import.go
+++ b/pkg/talkgroups/importer/import.go
@ -10,8 +10,6 @@ import (
 	"strconv"
 	"strings"

-	"github.com/google/uuid"
-
 	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
@ -112,12 +110,11 @@ func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int,
 			gn := groupName // must take a copy
 			tgs = append(tgs, talkgroups.Talkgroup{
 				Talkgroup: database.Talkgroup{
-					ID:       uuid.New(),
 					TGID:     int32(tgt.Talkgroup),
 					SystemID: int32(tgt.System),
 					Name:     &fields[4],
 					AlphaTag: &fields[3],
-					TgGroup:  &gn,
+					TGGroup:  &gn,
 					Metadata: metadata,
 					Tags:     tags,
 					Weight:   1.0,
--- a/pkg/talkgroups/importer/import_test.go
+++ b/pkg/talkgroups/importer/import_test.go
@ -57,7 +57,7 @@ func TestImport(t *testing.T) {

 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			dbMock := mocks.NewDB(t)
+			dbMock := mocks.NewStore(t)
 			if tc.expectErr == nil {
 				dbMock.EXPECT().GetSystemName(mock.AnythingOfType("*context.valueCtx"), tc.sysID).Return(tc.sysName, nil)
 			}
--- a/pkg/talkgroups/importer/testdata/riscon.json
+++ b/pkg/talkgroups/importer/testdata/riscon.json
--- a/sql/postgres/migrations/001_initial.up.sql
+++ b/sql/postgres/migrations/001_initial.up.sql
@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS users(
-	id SERIAL PRIMARY KEY,
+	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
 	username VARCHAR (255) UNIQUE NOT NULL,
 	password TEXT NOT NULL,
 	email TEXT NOT NULL,
@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS users(
 CREATE INDEX IF NOT EXISTS users_username_idx ON users(username);

 CREATE TABLE IF NOT EXISTS api_keys(
-	id SERIAL PRIMARY KEY,
+	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
 	owner INTEGER REFERENCES users(id) NOT NULL,
 	created_at TIMESTAMP NOT NULL,
 	expires TIMESTAMP,
@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS systems(
 );

 CREATE TABLE IF NOT EXISTS talkgroups(
-	id UUID PRIMARY KEY,
+	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
 	system_id INT4 REFERENCES systems(id) NOT NULL,
 	tgid INT4 NOT NULL,
 	name TEXT,
@ -36,6 +36,7 @@ CREATE TABLE IF NOT EXISTS talkgroups(
 	alert BOOLEAN NOT NULL DEFAULT 'true',
 	alert_config JSONB,
 	weight REAL NOT NULL DEFAULT 1.0,
+	learned BOOLEAN NOT NULL DEFAULT FALSE,
 	UNIQUE (system_id, tgid)
 );

@ -44,17 +45,18 @@ CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
 CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);

 CREATE TABLE IF NOT EXISTS talkgroups_learned(
-	id UUID PRIMARY KEY,
+	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
 	system_id INTEGER REFERENCES systems(id) NOT NULL,
 	tgid INTEGER NOT NULL,
 	name TEXT NOT NULL,
 	alpha_tag TEXT,
+	tg_group TEXT,
 	ignored BOOLEAN,
 	UNIQUE (system_id, tgid, name)
 );

 CREATE TABLE IF NOT EXISTS alerts(
-	id UUID PRIMARY KEY,
+	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
 	time TIMESTAMPTZ NOT NULL, 
 	tgid INTEGER NOT NULL,
 	system_id INTEGER REFERENCES systems(id) NOT NULL,
@ -65,22 +67,6 @@ CREATE TABLE IF NOT EXISTS alerts(
 	metadata JSONB
 );

-CREATE OR REPLACE FUNCTION learn_talkgroup()
-RETURNS TRIGGER AS $$
-BEGIN
-	IF NOT EXISTS (
-		SELECT tg.system_id, tg.tgid, tg.name, tg.alpha_tag FROM talkgroups tg WHERE tg.system_id = NEW.system AND tg.tgid = NEW.talkgroup
-		UNION
-		SELECT tgl.system_id, tgl.tgid, tgl.name, tgl.alpha_tag FROM talkgroups_learned tgl WHERE tgl.system_id = NEW.system AND tgl.tgid = NEW.talkgroup
-	) THEN
-		INSERT INTO talkgroups_learned(system_id, tgid, name, alpha_tag) VALUES(
-			NEW.system, NEW.talkgroup, NEW.tg_label, NEW.tg_alpha_tag
-		) ON CONFLICT DO NOTHING;
-	END IF;
-	RETURN NEW;
-END
-$$ LANGUAGE plpgsql;
-
 CREATE TABLE IF NOT EXISTS calls(
 	id UUID PRIMARY KEY,
 	submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL,
@ -99,12 +85,10 @@ CREATE TABLE IF NOT EXISTS calls(
 	tg_alpha_tag TEXT,
 	tg_group TEXT,
 	source INTEGER NOT NULL,
-	transcript TEXT
+	transcript TEXT,
+	FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid)
 );

-CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls
-FOR EACH ROW EXECUTE FUNCTION learn_talkgroup();
-
 CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript));
 CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date);

--- a/sql/postgres/queries/calls.sql
+++ b/sql/postgres/queries/calls.sql
@ -41,10 +41,9 @@ source
 UPDATE calls SET transcript = $2 WHERE id = $1;

 -- name: AddAlert :exec
-INSERT INTO alerts (id, time, tgid, system_id, weight, score, orig_score, notified, metadata)
+INSERT INTO alerts (time, tgid, system_id, weight, score, orig_score, notified, metadata)
 VALUES
 (
-	sqlc.arg(id),
 	sqlc.arg(time),
 	sqlc.arg(tgid),
 	sqlc.arg(system_id),
--- a/sql/postgres/queries/talkgroups.sql
+++ b/sql/postgres/queries/talkgroups.sql
@ -26,53 +26,48 @@ WHERE (system_id, tgid) = (@system_id, @tg_id);

 -- name: GetTalkgroupWithLearned :one
 SELECT
-sqlc.embed(tg), sqlc.embed(sys),
-FALSE learned
+sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid)
+WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid) AND tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;

 -- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
-sqlc.embed(tg), sqlc.embed(sys),
-FALSE learned
+sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = @system
+WHERE tg.system_id = @system AND tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE tgl.system_id = @system AND ignored IS NOT TRUE;

 -- name: GetTalkgroupsWithLearned :many
 SELECT
-sqlc.embed(tg), sqlc.embed(sys),
-FALSE learned
+sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
+WHERE tg.learned IS NOT TRUE
 UNION
 SELECT
 tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
+tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
+NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE;
@ -94,3 +89,29 @@ SET
 	weight = COALESCE(sqlc.narg('weight'), weight)
 WHERE id = sqlc.narg('id') OR (system_id = sqlc.narg('system_id') AND tgid = sqlc.narg('tgid'))
 RETURNING *;
+
+-- name: AddTalkgroupWithLearnedFlag :exec
+INSERT INTO talkgroups (
+	system_id,
+	tgid,
+	learned
+) VALUES(
+	@system_id,
+	@tgid,
+	't'
+);
+
+-- name: AddLearnedTalkgroup :one
+INSERT INTO talkgroups_learned(
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group
+) VALUES (
+	@system_id,
+	@tgid,
+	sqlc.narg('name'),
+	sqlc.narg('alpha_tag'),
+	sqlc.narg('tg_group')
+) RETURNING id;
--- a/sql/sqlc.yaml
+++ b/sql/sqlc.yaml
@ -14,6 +14,7 @@ sql:
        initialisms:
          - id
          - tgid
+          - tg
        emit_pointers_for_null_types: true
        overrides:
          - db_type: "uuid"
@ -22,10 +23,6 @@ sql:
              type: "UUID"
          - db_type: "pg_catalog.int4"
            go_type: "int"
-          - db_type: "pg_catalog.serial4"
-            go_type: "int"
-          - db_type: "integer"
-            go_type: "int"
          - db_type: "pg_catalog.timestamp"
            go_type: "time.Time"
          - db_type: "pg_catalog.text"
Author	SHA1	Message	Date
amigan	f779242995	Merge pull request 'callsnormalize' (#40 ) from callsnormalize into trunk Reviewed-on: #40	2024-11-19 10:18:12 -05:00
Daniel Ponte	f7fdaf9867	don't alert ignored	2024-11-19 10:18:12 -05:00
Daniel Ponte	711c8fbad0	tests	2024-11-19 10:18:12 -05:00
Daniel Ponte	f6cb8b4d17	fixes	2024-11-19 10:18:12 -05:00
Daniel Ponte	78f6df5920	Flatten migrations	2024-11-19 10:18:12 -05:00
Daniel Ponte	ef69d95583	Initial calls normalization fix wip	2024-11-19 10:18:12 -05:00