Fix query excluding learned

Reviewed-on: #46

cmd/stillbox/main.go

@@ -35,5 +35,6 @@ func main() {
 	cmds := append([]*cobra.Command{serve.Command(cfg)}, admin.Command(cfg)...)
 	rootCmd.AddCommand(cmds...)
 
-	rootCmd.Execute()
+	// cobra is already checking for errors and will print them
+	_ = rootCmd.Execute()
 }

config.sample.yaml

@@ -36,7 +36,7 @@ notify:
 #   subjectTemplate: "Stillbox Alert ({{ highest . }})"
 #   bodyTemplate: |
 #     {{ range . -}}
-#     {{ .TGName }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
+#     {{ .TGName }}{{ if (and .Talkgroup .Talkgroup.AlphaTag) }} ({{ .Talkgroup.StringTag false -}}){{ end }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
 #
 #     {{ end -}}
     config:

internal/common/common.go

@@ -44,3 +44,12 @@ func PtrOrNull[T comparable](val T) *T {
 
 	return &val
 }
+
+func ZeroOr[T any](v *T) T {
+	var zero T
+	if v == nil {
+		return zero
+	}
+
+	return *v
+}

pkg/alerting/alert/alert.go

@@ -3,12 +3,12 @@ package alert
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"time"
 
 	"dynatron.me/x/stillbox/internal/trending"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"github.com/jackc/pgx/v5/pgtype"
 )
@@ -17,6 +17,7 @@ type Alert struct {
 	ID         int
 	Timestamp  time.Time
 	TGName     string
+	Talkgroup  *talkgroups.Talkgroup
 	Score      trending.Score[talkgroups.ID]
 	OrigScore  float64
 	Weight     float32
@@ -44,7 +45,8 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
 }
 
 // Make creates an alert for later rendering or storage.
-func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
+func Make(ctx context.Context, score trending.Score[talkgroups.ID], origScore float64) (Alert, error) {
+	store := tgstore.FromCtx(ctx)
 	d := Alert{
 		Score:     score,
 		Timestamp: time.Now(),
@@ -56,15 +58,8 @@ func Make(ctx context.Context, store talkgroups.Store, score trending.Score[talk
 	switch err {
 	case nil:
 		d.Weight = tgRecord.Talkgroup.Weight
-		if tgRecord.System.Name == "" {
-			tgRecord.System.Name = strconv.Itoa(int(score.ID.System))
-		}
-
-		if tgRecord.Talkgroup.Name != nil {
-			d.TGName = fmt.Sprintf("%s %s [%d]", tgRecord.System.Name, *tgRecord.Talkgroup.Name, score.ID.Talkgroup)
-		} else {
-			d.TGName = fmt.Sprintf("%s:%d", tgRecord.System.Name, int(score.ID.Talkgroup))
-		}
+		d.TGName = tgRecord.String()
+		d.Talkgroup = tgRecord
 	default:
 		system, has := store.SystemName(ctx, int(score.ID.System))
 		if has {

pkg/alerting/alerting.go

@@ -3,6 +3,7 @@ package alerting
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"sort"
 	"sync"
@@ -14,7 +15,8 @@ import (
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/notify"
 	"dynatron.me/x/stillbox/pkg/sinks"
-	talkgroups "dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"dynatron.me/x/stillbox/internal/timeseries"
 	"dynatron.me/x/stillbox/internal/trending"
@@ -50,7 +52,7 @@ type alerter struct {
 	alertCache map[talkgroups.ID]alert.Alert
 	renotify   time.Duration
 	notifier   notify.Notifier
-	tgCache    talkgroups.Store
+	tgCache    tgstore.Store
 }
 
 type offsetClock time.Duration
@@ -85,7 +87,7 @@ func WithNotifier(n notify.Notifier) AlertOption {
 }
 
 // New creates a new Alerter using the provided configuration.
-func New(cfg config.Alerting, tgCache talkgroups.Store, opts ...AlertOption) Alerter {
+func New(cfg config.Alerting, tgCache tgstore.Store, opts ...AlertOption) Alerter {
 	if !cfg.Enable {
 		return &noopAlerter{}
 	}
@@ -168,7 +170,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
 		if s.Score > as.cfg.AlertThreshold || testMode {
 			if old, inCache := as.alertCache[s.ID]; !inCache || now.Sub(old.Timestamp) > as.renotify {
 				s.Score *= as.tgCache.Weight(ctx, s.ID, now)
-				a, err := alert.Make(ctx, as.tgCache, s, origScore)
+				a, err := alert.Make(ctx, s, origScore)
 				if err != nil {
 					return nil, fmt.Errorf("makeAlert: %w", err)
 				}
@@ -198,9 +200,16 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al
 }
 
 func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) {
-	alerts := make([]alert.Alert, 0, len(as.scores))
 	ctx := r.Context()
 
+	ridx := rand.Intn(len(as.scores))
+	a, err := alert.Make(ctx, as.scores[ridx], 1.0)
+	if err != nil {
+		log.Error().Err(err).Msg("test notify make alert fail")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
 	alerts, err := as.eval(ctx, time.Now(), true)
 	if err != nil {
 		log.Error().Err(err).Msg("test notification eval")
@@ -208,6 +217,8 @@ func (as *alerter) testNotifyHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	alerts = append(alerts, a)
+
 	err = as.notifier.Send(ctx, alerts)
 	if err != nil {
 		log.Error().Err(err).Msg("test notification send")

pkg/alerting/simulate.go

@@ -13,6 +13,7 @@ import (
 	"dynatron.me/x/stillbox/internal/trending"
 	"dynatron.me/x/stillbox/pkg/config"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"github.com/rs/zerolog/log"
 )
@@ -59,7 +60,7 @@ func (s *Simulation) stepClock(t time.Time) {
 // Simulate begins the simulation using the DB handle from ctx. It returns final scores.
 func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.ID], error) {
 	now := time.Now()
-	tgc := talkgroups.NewCache()
+	tgc := tgstore.NewCache()
 
 	s.Enable = true
 	s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter)

pkg/alerting/stats.html

@@ -86,7 +86,7 @@
       {{ $tg := (index $.TGs .ID) }}
       <tr>
 	<td>{{ $tg.System.Name}}</td>
-	<td>{{ $tg.Talkgroup.Name}}</td>
+	<td>{{ $tg.Talkgroup }}</td>
 	<td>{{  .ID.Talkgroup }}</td>
 	<td>{{ f .Count 0 }}</td>
 	<td>{{ f .RecentCount 0 }}</td>

pkg/auth/jwt.go

@@ -40,6 +40,23 @@ type jwtAuth interface {
 
 type claims map[string]interface{}
 
+func UIDFrom(ctx context.Context) *int32 {
+	tok, _, err := jwtauth.FromContext(ctx)
+	if err != nil {
+		return nil
+	}
+
+	uidStr := tok.Subject()
+	uidInt, err := strconv.Atoi(uidStr)
+	if err != nil {
+		return nil
+	}
+
+	uid := int32(uidInt)
+
+	return &uid
+}
+
 func (a *Auth) Authenticated(r *http.Request) (claims, bool) {
 	// TODO: check IP against ACL, or conf.Public, and against map of routes
 	tok, cl, err := jwtauth.FromContext(r.Context())

pkg/calls/call.go

@@ -1,13 +1,11 @@
 package calls
 
 import (
-	"context"
 	"fmt"
 	"time"
 
 	"dynatron.me/x/stillbox/internal/audio"
 	"dynatron.me/x/stillbox/pkg/auth"
-	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/pb"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
 
@@ -113,21 +111,6 @@ func (c *Call) ToPB() *pb.Call {
 	}
 }
 
-func (c *Call) LearnTG(ctx context.Context, db database.Store) (learnedId int, err error) {
-	err = db.AddTalkgroupWithLearnedFlag(ctx, int32(c.System), int32(c.Talkgroup))
-	if err != nil {
-		return 0, fmt.Errorf("addTalkgroupWithLearnedFlag: %w", err)
-	}
-
-	return db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
-		SystemID: c.System,
-		TGID:     c.Talkgroup,
-		Name:     c.TalkgroupLabel,
-		AlphaTag: c.TGAlphaTag,
-		TGGroup:  c.TalkgroupGroup,
-	})
-}
-
 func (c *Call) computeLength() (err error) {
 	var td time.Duration
 

pkg/calls/filter.go

@@ -1,113 +0,0 @@
-package calls
-
-import (
-	"context"
-
-	"dynatron.me/x/stillbox/pkg/database"
-	"dynatron.me/x/stillbox/pkg/pb"
-
-	tgs "dynatron.me/x/stillbox/pkg/talkgroups"
-)
-
-type TalkgroupFilter struct {
-	Talkgroups       []tgs.ID `json:"talkgroups,omitempty"`
-	TalkgroupsNot    []tgs.ID `json:"talkgroupsNot,omitempty"`
-	TalkgroupTagsAll []string `json:"talkgroupTagsAll,omitempty"`
-	TalkgroupTagsAny []string `json:"talkgroupTagsAny,omitempty"`
-	TalkgroupTagsNot []string `json:"talkgroupTagsNot,omitempty"`
-
-	talkgroups map[tgs.ID]bool
-}
-
-func TalkgroupFilterFromPB(ctx context.Context, p *pb.Filter) (*TalkgroupFilter, error) {
-	tgf := &TalkgroupFilter{
-		TalkgroupTagsAll: p.TalkgroupTagsAll,
-		TalkgroupTagsAny: p.TalkgroupTagsAny,
-		TalkgroupTagsNot: p.TalkgroupTagsNot,
-	}
-
-	if l := len(p.Talkgroups); l > 0 {
-		tgf.Talkgroups = make([]tgs.ID, l)
-		for i, t := range p.Talkgroups {
-			tgf.Talkgroups[i] = tgs.ID{
-				System:    uint32(t.System),
-				Talkgroup: uint32(t.Talkgroup),
-			}
-		}
-	}
-
-	if l := len(p.TalkgroupsNot); l > 0 {
-		tgf.TalkgroupsNot = make([]tgs.ID, l)
-		for i, t := range p.TalkgroupsNot {
-			tgf.TalkgroupsNot[i] = tgs.ID{
-				System:    uint32(t.System),
-				Talkgroup: uint32(t.Talkgroup),
-			}
-		}
-	}
-
-	return tgf, tgf.compile(ctx)
-}
-
-func (f *TalkgroupFilter) hasTags() bool {
-	return len(f.TalkgroupTagsAny) > 0 || len(f.TalkgroupTagsAll) > 0 || len(f.TalkgroupTagsNot) > 0
-}
-
-func (f *TalkgroupFilter) GetFinalTalkgroups() map[tgs.ID]bool {
-	return f.talkgroups
-}
-
-func (f *TalkgroupFilter) compile(ctx context.Context) error {
-	f.talkgroups = make(map[tgs.ID]bool)
-	for _, tg := range f.Talkgroups {
-		f.talkgroups[tg] = true
-	}
-
-	if f.hasTags() { // don't bother with DB if no tags
-		db := database.FromCtx(ctx)
-		tagTGs, err := db.GetTalkgroupIDsByTags(ctx, f.TalkgroupTagsAny, f.TalkgroupTagsAll, f.TalkgroupTagsNot)
-		if err != nil {
-			return err
-		}
-
-		for _, tg := range tagTGs {
-			f.talkgroups[tgs.ID{System: uint32(tg.SystemID), Talkgroup: uint32(tg.TGID)}] = true
-		}
-	}
-
-	for _, tg := range f.TalkgroupsNot {
-		f.talkgroups[tg] = false
-	}
-
-	return nil
-}
-
-func (f *TalkgroupFilter) Test(ctx context.Context, call *Call) bool {
-	if f == nil { // no filter means all calls
-		return true
-	}
-
-	if f.talkgroups == nil {
-		err := f.compile(ctx)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	tg := call.TalkgroupTuple()
-
-	tgRes, have := f.talkgroups[tg]
-	if have {
-		return tgRes
-	}
-
-	for _, patch := range call.Patches {
-		tg.Talkgroup = uint32(patch)
-		tgRes, have := f.talkgroups[tg]
-		if have {
-			return tgRes
-		}
-	}
-
-	return false
-}

pkg/database/batch.go

@@ -0,0 +1,210 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.27.0
+// source: batch.go
+
+package database
+
+import (
+	"context"
+	"errors"
+
+	"dynatron.me/x/stillbox/internal/jsontypes"
+	"dynatron.me/x/stillbox/pkg/alerting/rules"
+	"github.com/jackc/pgx/v5"
+)
+
+var (
+	ErrBatchAlreadyClosed = errors.New("batch already closed")
+)
+
+const storeTGVersion = `-- name: StoreTGVersion :batchexec
+INSERT INTO talkgroup_versions(time, created_by,
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned
+) SELECT NOW(), $1,
+	tg.system_id,
+	tg.tgid,
+	tg.name,
+	tg.alpha_tag,
+	tg.tg_group,
+	tg.frequency,
+	tg.metadata,
+	tg.tags,
+	tg.alert,
+	tg.alert_config,
+	tg.weight,
+	tg.learned
+FROM talkgroups tg WHERE tg.system_id = $2 AND tg.tgid = $3
+`
+
+type StoreTGVersionBatchResults struct {
+	br     pgx.BatchResults
+	tot    int
+	closed bool
+}
+
+type StoreTGVersionParams struct {
+	Submitter *int32 `json:"submitter"`
+	SystemID  int32  `json:"system_id"`
+	TGID      int32  `json:"tgid"`
+}
+
+func (q *Queries) StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults {
+	batch := &pgx.Batch{}
+	for _, a := range arg {
+		vals := []interface{}{
+			a.Submitter,
+			a.SystemID,
+			a.TGID,
+		}
+		batch.Queue(storeTGVersion, vals...)
+	}
+	br := q.db.SendBatch(ctx, batch)
+	return &StoreTGVersionBatchResults{br, len(arg), false}
+}
+
+func (b *StoreTGVersionBatchResults) Exec(f func(int, error)) {
+	defer b.br.Close()
+	for t := 0; t < b.tot; t++ {
+		if b.closed {
+			if f != nil {
+				f(t, ErrBatchAlreadyClosed)
+			}
+			continue
+		}
+		_, err := b.br.Exec()
+		if f != nil {
+			f(t, err)
+		}
+	}
+}
+
+func (b *StoreTGVersionBatchResults) Close() error {
+	b.closed = true
+	return b.br.Close()
+}
+
+const upsertTalkgroup = `-- name: UpsertTalkgroup :batchone
+INSERT INTO talkgroups AS tg (
+	system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
+) VALUES (
+	$1,
+	$2,
+	$3,
+	$4,
+	$5,
+	$6,
+	$7,
+	$8,
+	$9,
+	$10,
+	$11,
+	$12
+)
+ON CONFLICT (system_id, tgid) DO UPDATE
+SET
+	name = COALESCE($3, tg.name),
+	alpha_tag = COALESCE($4, tg.alpha_tag),
+	tg_group = COALESCE($5, tg.tg_group),
+	frequency = COALESCE($6, tg.frequency),
+	metadata = COALESCE($7, tg.metadata),
+	tags = COALESCE($8, tg.tags),
+	alert = COALESCE($9, tg.alert),
+	alert_config = COALESCE($10, tg.alert_config),
+	weight = COALESCE($11, tg.weight),
+	learned = COALESCE($12, tg.learned)
+RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
+`
+
+type UpsertTalkgroupBatchResults struct {
+	br     pgx.BatchResults
+	tot    int
+	closed bool
+}
+
+type UpsertTalkgroupParams struct {
+	SystemID    int32              `json:"system_id"`
+	TGID        int32              `json:"tgid"`
+	Name        *string            `json:"name"`
+	AlphaTag    *string            `json:"alpha_tag"`
+	TGGroup     *string            `json:"tg_group"`
+	Frequency   *int32             `json:"frequency"`
+	Metadata    jsontypes.Metadata `json:"metadata"`
+	Tags        []string           `json:"tags"`
+	Alert       *bool              `json:"alert"`
+	AlertConfig rules.AlertRules   `json:"alert_config"`
+	Weight      *float32           `json:"weight"`
+	Learned     *bool              `json:"learned"`
+}
+
+func (q *Queries) UpsertTalkgroup(ctx context.Context, arg []UpsertTalkgroupParams) *UpsertTalkgroupBatchResults {
+	batch := &pgx.Batch{}
+	for _, a := range arg {
+		vals := []interface{}{
+			a.SystemID,
+			a.TGID,
+			a.Name,
+			a.AlphaTag,
+			a.TGGroup,
+			a.Frequency,
+			a.Metadata,
+			a.Tags,
+			a.Alert,
+			a.AlertConfig,
+			a.Weight,
+			a.Learned,
+		}
+		batch.Queue(upsertTalkgroup, vals...)
+	}
+	br := q.db.SendBatch(ctx, batch)
+	return &UpsertTalkgroupBatchResults{br, len(arg), false}
+}
+
+func (b *UpsertTalkgroupBatchResults) QueryRow(f func(int, Talkgroup, error)) {
+	defer b.br.Close()
+	for t := 0; t < b.tot; t++ {
+		var i Talkgroup
+		if b.closed {
+			if f != nil {
+				f(t, i, ErrBatchAlreadyClosed)
+			}
+			continue
+		}
+		row := b.br.QueryRow()
+		err := row.Scan(
+			&i.ID,
+			&i.SystemID,
+			&i.TGID,
+			&i.Name,
+			&i.AlphaTag,
+			&i.TGGroup,
+			&i.Frequency,
+			&i.Metadata,
+			&i.Tags,
+			&i.Alert,
+			&i.AlertConfig,
+			&i.Weight,
+			&i.Learned,
+			&i.Ignored,
+		)
+		if f != nil {
+			f(t, i, err)
+		}
+	}
+}
+
+func (b *UpsertTalkgroupBatchResults) Close() error {
+	b.closed = true
+	return b.br.Close()
+}

pkg/database/database.go

@@ -43,6 +43,7 @@ func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOp
 		return fmt.Errorf("Tx begin: %w", err)
 	}
 
+	//nolint:errcheck
 	defer tx.Rollback(ctx)
 
 	dbtx := &Database{Pool: db.Pool, Queries: db.Queries.WithTx(tx)}

pkg/database/db.go

@@ -15,6 +15,7 @@ type DBTX interface {
 	Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
 	Query(context.Context, string, ...interface{}) (pgx.Rows, error)
 	QueryRow(context.Context, string, ...interface{}) pgx.Row
+	SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
 }
 
 func New(db DBTX) *Queries {

pkg/database/extend.go

@@ -1,5 +1,9 @@
 package database
 
+import (
+	"strconv"
+)
+
 func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup                    { return d.Talkgroup }
 func (d GetTalkgroupsRow) GetSystem() System                          { return d.System }
 func (d GetTalkgroupsRow) GetLearned() bool                           { return d.Talkgroup.Learned }
@@ -15,3 +19,22 @@ func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool        { return g
 func (g Talkgroup) GetTalkgroup() Talkgroup                           { return g }
 func (g Talkgroup) GetSystem() System                                 { return System{ID: int(g.SystemID)} }
 func (g Talkgroup) GetLearned() bool                                  { return false }
+
+func (g Talkgroup) String() string {
+	return g.StringTag(true)
+}
+
+func (g Talkgroup) StringTag(withTag bool) string {
+	switch {
+	case withTag && g.AlphaTag != nil:
+		return *g.AlphaTag
+	case g.Name != nil && g.TGGroup != nil:
+		return *g.TGGroup + " " + *g.Name
+	case g.Name != nil:
+		return *g.Name + " [" + strconv.Itoa(int(g.TGID)) + "]"
+	case g.TGGroup != nil:
+		return *g.TGGroup + " [" + strconv.Itoa(int(g.TGID)) + "]"
+	}
+
+	return strconv.Itoa(int(g.TGID))
+}

pkg/database/mocks/Store.go

@@ -123,22 +123,22 @@ func (_c *Store_AddCall_Call) RunAndReturn(run func(context.Context, database.Ad
 }
 
 // AddLearnedTalkgroup provides a mock function with given fields: ctx, arg
-func (_m *Store) AddLearnedTalkgroup(ctx context.Context, arg database.AddLearnedTalkgroupParams) (int, error) {
+func (_m *Store) AddLearnedTalkgroup(ctx context.Context, arg database.AddLearnedTalkgroupParams) (database.Talkgroup, error) {
 	ret := _m.Called(ctx, arg)
 
 	if len(ret) == 0 {
 		panic("no return value specified for AddLearnedTalkgroup")
 	}
 
-	var r0 int
+	var r0 database.Talkgroup
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) (int, error)); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) (database.Talkgroup, error)); ok {
 		return rf(ctx, arg)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) int); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, database.AddLearnedTalkgroupParams) database.Talkgroup); ok {
 		r0 = rf(ctx, arg)
 	} else {
-		r0 = ret.Get(0).(int)
+		r0 = ret.Get(0).(database.Talkgroup)
 	}
 
 	if rf, ok := ret.Get(1).(func(context.Context, database.AddLearnedTalkgroupParams) error); ok {
@@ -169,60 +169,12 @@ func (_c *Store_AddLearnedTalkgroup_Call) Run(run func(ctx context.Context, arg
 	return _c
 }
 
-func (_c *Store_AddLearnedTalkgroup_Call) Return(_a0 int, _a1 error) *Store_AddLearnedTalkgroup_Call {
+func (_c *Store_AddLearnedTalkgroup_Call) Return(_a0 database.Talkgroup, _a1 error) *Store_AddLearnedTalkgroup_Call {
 	_c.Call.Return(_a0, _a1)
 	return _c
 }
 
-func (_c *Store_AddLearnedTalkgroup_Call) RunAndReturn(run func(context.Context, database.AddLearnedTalkgroupParams) (int, error)) *Store_AddLearnedTalkgroup_Call {
-	_c.Call.Return(run)
-	return _c
-}
-
-// AddTalkgroupWithLearnedFlag provides a mock function with given fields: ctx, systemID, tGID
-func (_m *Store) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
-	ret := _m.Called(ctx, systemID, tGID)
-
-	if len(ret) == 0 {
-		panic("no return value specified for AddTalkgroupWithLearnedFlag")
-	}
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, int32, int32) error); ok {
-		r0 = rf(ctx, systemID, tGID)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Store_AddTalkgroupWithLearnedFlag_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddTalkgroupWithLearnedFlag'
-type Store_AddTalkgroupWithLearnedFlag_Call struct {
-	*mock.Call
-}
-
-// AddTalkgroupWithLearnedFlag is a helper method to define mock.On call
-//   - ctx context.Context
-//   - systemID int32
-//   - tGID int32
-func (_e *Store_Expecter) AddTalkgroupWithLearnedFlag(ctx interface{}, systemID interface{}, tGID interface{}) *Store_AddTalkgroupWithLearnedFlag_Call {
-	return &Store_AddTalkgroupWithLearnedFlag_Call{Call: _e.mock.On("AddTalkgroupWithLearnedFlag", ctx, systemID, tGID)}
-}
-
-func (_c *Store_AddTalkgroupWithLearnedFlag_Call) Run(run func(ctx context.Context, systemID int32, tGID int32)) *Store_AddTalkgroupWithLearnedFlag_Call {
-	_c.Call.Run(func(args mock.Arguments) {
-		run(args[0].(context.Context), args[1].(int32), args[2].(int32))
-	})
-	return _c
-}
-
-func (_c *Store_AddTalkgroupWithLearnedFlag_Call) Return(_a0 error) *Store_AddTalkgroupWithLearnedFlag_Call {
-	_c.Call.Return(_a0)
-	return _c
-}
-
-func (_c *Store_AddTalkgroupWithLearnedFlag_Call) RunAndReturn(run func(context.Context, int32, int32) error) *Store_AddTalkgroupWithLearnedFlag_Call {
+func (_c *Store_AddLearnedTalkgroup_Call) RunAndReturn(run func(context.Context, database.AddLearnedTalkgroupParams) (database.Talkgroup, error)) *Store_AddLearnedTalkgroup_Call {
 	_c.Call.Return(run)
 	return _c
 }
@@ -1569,6 +1521,63 @@ func (_c *Store_InTx_Call) RunAndReturn(run func(context.Context, func(database.
 	return _c
 }
 
+// RestoreTalkgroupVersion provides a mock function with given fields: ctx, versionIds
+func (_m *Store) RestoreTalkgroupVersion(ctx context.Context, versionIds int) (database.Talkgroup, error) {
+	ret := _m.Called(ctx, versionIds)
+
+	if len(ret) == 0 {
+		panic("no return value specified for RestoreTalkgroupVersion")
+	}
+
+	var r0 database.Talkgroup
+	var r1 error
+	if rf, ok := ret.Get(0).(func(context.Context, int) (database.Talkgroup, error)); ok {
+		return rf(ctx, versionIds)
+	}
+	if rf, ok := ret.Get(0).(func(context.Context, int) database.Talkgroup); ok {
+		r0 = rf(ctx, versionIds)
+	} else {
+		r0 = ret.Get(0).(database.Talkgroup)
+	}
+
+	if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
+		r1 = rf(ctx, versionIds)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Store_RestoreTalkgroupVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RestoreTalkgroupVersion'
+type Store_RestoreTalkgroupVersion_Call struct {
+	*mock.Call
+}
+
+// RestoreTalkgroupVersion is a helper method to define mock.On call
+//   - ctx context.Context
+//   - versionIds int
+func (_e *Store_Expecter) RestoreTalkgroupVersion(ctx interface{}, versionIds interface{}) *Store_RestoreTalkgroupVersion_Call {
+	return &Store_RestoreTalkgroupVersion_Call{Call: _e.mock.On("RestoreTalkgroupVersion", ctx, versionIds)}
+}
+
+func (_c *Store_RestoreTalkgroupVersion_Call) Run(run func(ctx context.Context, versionIds int)) *Store_RestoreTalkgroupVersion_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run(args[0].(context.Context), args[1].(int))
+	})
+	return _c
+}
+
+func (_c *Store_RestoreTalkgroupVersion_Call) Return(_a0 database.Talkgroup, _a1 error) *Store_RestoreTalkgroupVersion_Call {
+	_c.Call.Return(_a0, _a1)
+	return _c
+}
+
+func (_c *Store_RestoreTalkgroupVersion_Call) RunAndReturn(run func(context.Context, int) (database.Talkgroup, error)) *Store_RestoreTalkgroupVersion_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
 // SetCallTranscript provides a mock function with given fields: ctx, iD, transcript
 func (_m *Store) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error {
 	ret := _m.Called(ctx, iD, transcript)
@@ -1666,6 +1675,55 @@ func (_c *Store_SetTalkgroupTags_Call) RunAndReturn(run func(context.Context, []
 	return _c
 }
 
+// StoreTGVersion provides a mock function with given fields: ctx, arg
+func (_m *Store) StoreTGVersion(ctx context.Context, arg []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults {
+	ret := _m.Called(ctx, arg)
+
+	if len(ret) == 0 {
+		panic("no return value specified for StoreTGVersion")
+	}
+
+	var r0 *database.StoreTGVersionBatchResults
+	if rf, ok := ret.Get(0).(func(context.Context, []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults); ok {
+		r0 = rf(ctx, arg)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*database.StoreTGVersionBatchResults)
+		}
+	}
+
+	return r0
+}
+
+// Store_StoreTGVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StoreTGVersion'
+type Store_StoreTGVersion_Call struct {
+	*mock.Call
+}
+
+// StoreTGVersion is a helper method to define mock.On call
+//   - ctx context.Context
+//   - arg []database.StoreTGVersionParams
+func (_e *Store_Expecter) StoreTGVersion(ctx interface{}, arg interface{}) *Store_StoreTGVersion_Call {
+	return &Store_StoreTGVersion_Call{Call: _e.mock.On("StoreTGVersion", ctx, arg)}
+}
+
+func (_c *Store_StoreTGVersion_Call) Run(run func(ctx context.Context, arg []database.StoreTGVersionParams)) *Store_StoreTGVersion_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run(args[0].(context.Context), args[1].([]database.StoreTGVersionParams))
+	})
+	return _c
+}
+
+func (_c *Store_StoreTGVersion_Call) Return(_a0 *database.StoreTGVersionBatchResults) *Store_StoreTGVersion_Call {
+	_c.Call.Return(_a0)
+	return _c
+}
+
+func (_c *Store_StoreTGVersion_Call) RunAndReturn(run func(context.Context, []database.StoreTGVersionParams) *database.StoreTGVersionBatchResults) *Store_StoreTGVersion_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
 // UpdatePassword provides a mock function with given fields: ctx, username, password
 func (_m *Store) UpdatePassword(ctx context.Context, username string, password string) error {
 	ret := _m.Called(ctx, username, password)
@@ -1771,6 +1829,55 @@ func (_c *Store_UpdateTalkgroup_Call) RunAndReturn(run func(context.Context, dat
 	return _c
 }
 
+// UpsertTalkgroup provides a mock function with given fields: ctx, arg
+func (_m *Store) UpsertTalkgroup(ctx context.Context, arg []database.UpsertTalkgroupParams) *database.UpsertTalkgroupBatchResults {
+	ret := _m.Called(ctx, arg)
+
+	if len(ret) == 0 {
+		panic("no return value specified for UpsertTalkgroup")
+	}
+
+	var r0 *database.UpsertTalkgroupBatchResults
+	if rf, ok := ret.Get(0).(func(context.Context, []database.UpsertTalkgroupParams) *database.UpsertTalkgroupBatchResults); ok {
+		r0 = rf(ctx, arg)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*database.UpsertTalkgroupBatchResults)
+		}
+	}
+
+	return r0
+}
+
+// Store_UpsertTalkgroup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpsertTalkgroup'
+type Store_UpsertTalkgroup_Call struct {
+	*mock.Call
+}
+
+// UpsertTalkgroup is a helper method to define mock.On call
+//   - ctx context.Context
+//   - arg []database.UpsertTalkgroupParams
+func (_e *Store_Expecter) UpsertTalkgroup(ctx interface{}, arg interface{}) *Store_UpsertTalkgroup_Call {
+	return &Store_UpsertTalkgroup_Call{Call: _e.mock.On("UpsertTalkgroup", ctx, arg)}
+}
+
+func (_c *Store_UpsertTalkgroup_Call) Run(run func(ctx context.Context, arg []database.UpsertTalkgroupParams)) *Store_UpsertTalkgroup_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run(args[0].(context.Context), args[1].([]database.UpsertTalkgroupParams))
+	})
+	return _c
+}
+
+func (_c *Store_UpsertTalkgroup_Call) Return(_a0 *database.UpsertTalkgroupBatchResults) *Store_UpsertTalkgroup_Call {
+	_c.Call.Return(_a0)
+	return _c
+}
+
+func (_c *Store_UpsertTalkgroup_Call) RunAndReturn(run func(context.Context, []database.UpsertTalkgroupParams) *database.UpsertTalkgroupBatchResults) *Store_UpsertTalkgroup_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
 // NewStore creates a new instance of Store. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
 // The first argument is typically a *testing.T value.
 func NewStore(t interface {

pkg/database/models.go

@@ -96,16 +96,26 @@ type Talkgroup struct {
 	AlertConfig rules.AlertRules   `json:"alert_config"`
 	Weight      float32            `json:"weight"`
 	Learned     bool               `json:"learned"`
+	Ignored     bool               `json:"ignored"`
 }
 
-type TalkgroupsLearned struct {
-	ID       int     `json:"id"`
-	SystemID int     `json:"system_id"`
-	TGID     int     `json:"tgid"`
-	Name     string  `json:"name"`
-	AlphaTag *string `json:"alpha_tag"`
-	TGGroup  *string `json:"tg_group"`
-	Ignored  bool    `json:"ignored"`
+type TalkgroupVersion struct {
+	ID          int                `json:"id"`
+	Time        pgtype.Timestamptz `json:"time"`
+	CreatedBy   *int32             `json:"created_by"`
+	SystemID    *int32             `json:"system_id"`
+	TGID        *int32             `json:"tgid"`
+	Name        *string            `json:"name"`
+	AlphaTag    *string            `json:"alpha_tag"`
+	TGGroup     *string            `json:"tg_group"`
+	Frequency   *int32             `json:"frequency"`
+	Metadata    []byte             `json:"metadata"`
+	Tags        []string           `json:"tags"`
+	Alert       *bool              `json:"alert"`
+	AlertConfig []byte             `json:"alert_config"`
+	Weight      *float32           `json:"weight"`
+	Learned     *bool              `json:"learned"`
+	Ignored     *bool              `json:"ignored"`
 }
 
 type User struct {

pkg/database/querier.go

@@ -14,8 +14,7 @@ import (
 type Querier interface {
 	AddAlert(ctx context.Context, arg AddAlertParams) error
 	AddCall(ctx context.Context, arg AddCallParams) error
-	AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error)
-	AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error
+	AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error)
 	CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
 	CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
 	DeleteAPIKey(ctx context.Context, apiKey string) error
@@ -35,10 +34,13 @@ type Querier interface {
 	GetUserByUID(ctx context.Context, id int) (User, error)
 	GetUserByUsername(ctx context.Context, username string) (User, error)
 	GetUsers(ctx context.Context) ([]User, error)
+	RestoreTalkgroupVersion(ctx context.Context, versionIds int) (Talkgroup, error)
 	SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
 	SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error
+	StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults
 	UpdatePassword(ctx context.Context, username string, password string) error
 	UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
+	UpsertTalkgroup(ctx context.Context, arg []UpsertTalkgroupParams) *UpsertTalkgroupBatchResults
 }
 
 var _ Querier = (*Queries)(nil)

pkg/database/talkgroups.go

@@ -41,20 +41,10 @@ func (t *TGTuples) Append(sys, tg uint32) {
 // Below queries are here because sqlc refuses to parse unnest(x, y)
 
 const getTalkgroupsWithLearnedBySysTGID = `SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name, tg.learned, tg.ignored
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
-WHERE tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, sys.id, sys.name, TRUE learned
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);`
+JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
 
 type GetTalkgroupsRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
@@ -86,6 +76,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
 			&i.System.ID,
 			&i.System.Name,
 			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 		); err != nil {
 			return nil, err
 		}
@@ -97,7 +88,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGT
 	return items, nil
 }
 
-const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
+const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
 JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
 WHERE tg.learned IS NOT TRUE;`
@@ -124,6 +115,8 @@ func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]G
 			&i.Talkgroup.Alert,
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
+			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 			&i.System.ID,
 			&i.System.Name,
 		); err != nil {

pkg/database/talkgroups.sql.go

@@ -13,30 +13,32 @@ import (
 )
 
 const addLearnedTalkgroup = `-- name: AddLearnedTalkgroup :one
-INSERT INTO talkgroups_learned(
+INSERT INTO talkgroups(
 	system_id,
 	tgid,
+	learned,
 	name,
 	alpha_tag,
 	tg_group
 ) VALUES (
 	$1,
 	$2,
+	TRUE,
 	$3,
 	$4,
 	$5
-) RETURNING id
+) RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
 `
 
 type AddLearnedTalkgroupParams struct {
-	SystemID int     `json:"system_id"`
-	TGID     int     `json:"tgid"`
+	SystemID int32   `json:"system_id"`
+	TGID     int32   `json:"tgid"`
 	Name     *string `json:"name"`
 	AlphaTag *string `json:"alpha_tag"`
 	TGGroup  *string `json:"tg_group"`
 }
 
-func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (int, error) {
+func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error) {
 	row := q.db.QueryRow(ctx, addLearnedTalkgroup,
 		arg.SystemID,
 		arg.TGID,
@@ -44,26 +46,24 @@ func (q *Queries) AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgro
 		arg.AlphaTag,
 		arg.TGGroup,
 	)
-	var id int
-	err := row.Scan(&id)
-	return id, err
-}
-
-const addTalkgroupWithLearnedFlag = `-- name: AddTalkgroupWithLearnedFlag :exec
-INSERT INTO talkgroups (
-	system_id,
-	tgid,
-	learned
-) VALUES(
-	$1,
-	$2,
-	't'
-)
-`
-
-func (q *Queries) AddTalkgroupWithLearnedFlag(ctx context.Context, systemID int32, tGID int32) error {
-	_, err := q.db.Exec(ctx, addTalkgroupWithLearnedFlag, systemID, tGID)
-	return err
+	var i Talkgroup
+	err := row.Scan(
+		&i.ID,
+		&i.SystemID,
+		&i.TGID,
+		&i.Name,
+		&i.AlphaTag,
+		&i.TGGroup,
+		&i.Frequency,
+		&i.Metadata,
+		&i.Tags,
+		&i.Alert,
+		&i.AlertConfig,
+		&i.Weight,
+		&i.Learned,
+		&i.Ignored,
+	)
+	return i, err
 }
 
 const getSystemName = `-- name: GetSystemName :one
@@ -78,7 +78,7 @@ func (q *Queries) GetSystemName(ctx context.Context, systemID int) (string, erro
 }
 
 const getTalkgroup = `-- name: GetTalkgroup :one
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
 WHERE (system_id, tgid) = ($1, $2)
 `
 
@@ -103,6 +103,7 @@ func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tGID int32)
 		&i.Talkgroup.AlertConfig,
 		&i.Talkgroup.Weight,
 		&i.Talkgroup.Learned,
+		&i.Talkgroup.Ignored,
 	)
 	return i, err
 }
@@ -153,19 +154,10 @@ func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tGID int
 
 const getTalkgroupWithLearned = `-- name: GetTalkgroupWithLearned :one
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
+WHERE (tg.system_id, tg.tgid) = ($1, $2)
 `
 
 type GetTalkgroupWithLearnedRow struct {
@@ -190,6 +182,7 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
 		&i.Talkgroup.AlertConfig,
 		&i.Talkgroup.Weight,
 		&i.Talkgroup.Learned,
+		&i.Talkgroup.Ignored,
 		&i.System.ID,
 		&i.System.Name,
 	)
@@ -197,7 +190,7 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, t
 }
 
 const getTalkgroupsWithAllTags = `-- name: GetTalkgroupsWithAllTags :many
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
 WHERE tags && ARRAY[$1]
 `
 
@@ -228,6 +221,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
 			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 		); err != nil {
 			return nil, err
 		}
@@ -240,7 +234,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
 }
 
 const getTalkgroupsWithAnyTags = `-- name: GetTalkgroupsWithAnyTags :many
-SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned FROM talkgroups
+SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight, talkgroups.learned, talkgroups.ignored FROM talkgroups
 WHERE tags @> ARRAY[$1]
 `
 
@@ -271,6 +265,7 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
 			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 		); err != nil {
 			return nil, err
 		}
@@ -284,18 +279,9 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
 
 const getTalkgroupsWithLearned = `-- name: GetTalkgroupsWithLearned :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE
 `
 
@@ -327,6 +313,7 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
 			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 			&i.System.ID,
 			&i.System.Name,
 		); err != nil {
@@ -342,19 +329,10 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
 
 const getTalkgroupsWithLearnedBySystem = `-- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, tg.ignored, sys.id, sys.name
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
+WHERE tg.system_id = $1
 `
 
 type GetTalkgroupsWithLearnedBySystemRow struct {
@@ -385,6 +363,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
 			&i.Talkgroup.AlertConfig,
 			&i.Talkgroup.Weight,
 			&i.Talkgroup.Learned,
+			&i.Talkgroup.Ignored,
 			&i.System.ID,
 			&i.System.Name,
 		); err != nil {
@@ -398,6 +377,73 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
 	return items, nil
 }
 
+const restoreTalkgroupVersion = `-- name: RestoreTalkgroupVersion :one
+INSERT INTO talkgroups(
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned,
+	ignored
+)
+SELECT
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned,
+	ignored
+FROM talkgroup_versions tgv ON CONFLICT (system_id, tgid) DO UPDATE SET
+	name = excluded.name,
+	alpha_tag = excluded.alpha_tag,
+	tg_group = excluded.tg_group,
+	metadata = excluded.metadata,
+	tags = excluded.tags,
+	alert = excluded.alert,
+	alert_config = excluded.alert_config,
+	weight = excluded.weight,
+	learned = excluded.learner,
+	ignored = excluded.ignored
+WHERE tgv.id = ANY($1)
+RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
+`
+
+func (q *Queries) RestoreTalkgroupVersion(ctx context.Context, versionIds int) (Talkgroup, error) {
+	row := q.db.QueryRow(ctx, restoreTalkgroupVersion, versionIds)
+	var i Talkgroup
+	err := row.Scan(
+		&i.ID,
+		&i.SystemID,
+		&i.TGID,
+		&i.Name,
+		&i.AlphaTag,
+		&i.TGGroup,
+		&i.Frequency,
+		&i.Metadata,
+		&i.Tags,
+		&i.Alert,
+		&i.AlertConfig,
+		&i.Weight,
+		&i.Learned,
+		&i.Ignored,
+	)
+	return i, err
+}
+
 const setTalkgroupTags = `-- name: SetTalkgroupTags :exec
 UPDATE talkgroups SET tags = $1
 WHERE system_id = $2 AND tgid = $3
@@ -419,9 +465,10 @@ SET
 	tags = COALESCE($6, tags),
 	alert = COALESCE($7, alert),
 	alert_config = COALESCE($8, alert_config),
-	weight = COALESCE($9, weight)
-WHERE id = $10 OR (system_id = $11 AND tgid = $12)
-RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
+	weight = COALESCE($9, weight),
+	learned = COALESCE($10, learned)
+WHERE id = $11 OR (system_id = $12 AND tgid = $13)
+RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned, ignored
 `
 
 type UpdateTalkgroupParams struct {
@@ -434,6 +481,7 @@ type UpdateTalkgroupParams struct {
 	Alert       *bool              `json:"alert"`
 	AlertConfig rules.AlertRules   `json:"alert_config"`
 	Weight      *float32           `json:"weight"`
+	Learned     *bool              `json:"learned"`
 	ID          *int32             `json:"id"`
 	SystemID    *int32             `json:"system_id"`
 	TGID        *int32             `json:"tgid"`
@@ -450,6 +498,7 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
 		arg.Alert,
 		arg.AlertConfig,
 		arg.Weight,
+		arg.Learned,
 		arg.ID,
 		arg.SystemID,
 		arg.TGID,
@@ -469,6 +518,7 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
 		&i.AlertConfig,
 		&i.Weight,
 		&i.Learned,
+		&i.Ignored,
 	)
 	return i, err
 }

pkg/database/talkgroups.sql_test.go

@@ -1,64 +0,0 @@
-package database
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
-SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = ($1, $2) AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = $1 AND tgl.tgid = $2 AND ignored IS NOT TRUE
-`
-
-const getTalkgroupsWithLearnedBySystemTest = `-- name: GetTalkgroupsWithLearnedBySystem :many
-SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = $1 AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = $1 AND ignored IS NOT TRUE
-`
-
-const getTalkgroupsWithLearnedTest = `-- name: GetTalkgroupsWithLearned :many
-SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, tg.learned, sys.id, sys.name
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE ignored IS NOT TRUE
-`
-
-func TestQueryColumnsMatch(t *testing.T) {
-	assert.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
-	assert.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
-	assert.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)
-}

pkg/nexus/client.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 
 	"dynatron.me/x/stillbox/internal/version"
-	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/pb"
+	tgfilter "dynatron.me/x/stillbox/pkg/talkgroups/filter"
 
 	"github.com/rs/zerolog/log"
 	"google.golang.org/protobuf/proto"
@@ -33,7 +33,7 @@ type client struct {
 	Connection
 
 	liveState pb.LiveState
-	filter    *calls.TalkgroupFilter
+	filter    *tgfilter.TalkgroupFilter
 
 	nexus *Nexus
 }

pkg/nexus/commands.go

@@ -3,9 +3,10 @@ package nexus
 import (
 	"context"
 
-	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/pb"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
+	tgfilter "dynatron.me/x/stillbox/pkg/talkgroups/filter"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"github.com/rs/zerolog/log"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -59,9 +60,9 @@ func (c *client) SendError(cmd *pb.Command, err error) {
 }
 
 func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
-	tgi, err := talkgroups.StoreFrom(ctx).TG(ctx, talkgroups.TG(tg.System, tg.Talkgroup))
+	tgi, err := tgstore.FromCtx(ctx).TG(ctx, talkgroups.TG(tg.System, tg.Talkgroup))
 	if err != nil {
-		if err != talkgroups.ErrNotFound {
+		if err != tgstore.ErrNotFound {
 			log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("get talkgroup fail")
 		}
 		return err
@@ -110,7 +111,7 @@ func (c *client) Live(ctx context.Context, cmd *pb.Live) error {
 	}
 
 	if cmd.Filter != nil {
-		filter, err := calls.TalkgroupFilterFromPB(ctx, cmd.Filter)
+		filter, err := tgfilter.TalkgroupFilterFromPB(ctx, cmd.Filter)
 		if err != nil {
 			log.Error().Err(err).Msg("filter create failed")
 			return err

pkg/notify/notify.go

@@ -52,7 +52,7 @@ var alertFm = template.FuncMap{
 
 const (
 	defaultBodyTemplStr = `{{ range . -}}
-{{ .TGName }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
+{{ .TGName }}{{ if (and .Talkgroup .Talkgroup.AlphaTag) }} ({{ .Talkgroup.StringTag false -}}){{ end }} is active with a score of {{ f .Score.Score 4 }}! ({{ f .Score.RecentCount 0 }}/{{ .Score.Count }} recent calls)
 
 {{ end -}}`
 	defaultSubjectTemplStr = `Stillbox Alert ({{ highest . }})`

pkg/rest/api.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"net/http"
 
-	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
@@ -42,7 +42,6 @@ type errResponse struct {
 
 func (e *errResponse) Render(w http.ResponseWriter, r *http.Request) error {
 	switch e.Code {
-	case http.StatusNotFound:
 	default:
 		log.Error().Str("path", r.URL.Path).Err(e.Err).Int("code", e.Code).Str("msg", e.Error).Msg("request failed")
 	}
@@ -68,10 +67,18 @@ func recordNotFound(err error) render.Renderer {
 	}
 }
 
-func internalError(err error) render.Renderer {
+func errTextNotFound(err error) render.Renderer {
 	return &errResponse{
 		Err:   err,
 		Code:  http.StatusNotFound,
+		Error: "Record not found: " + err.Error(),
+	}
+}
+
+func internalError(err error) render.Renderer {
+	return &errResponse{
+		Err:   err,
+		Code:  http.StatusInternalServerError,
 		Error: "Internal server error",
 	}
 }
@@ -79,9 +86,9 @@ func internalError(err error) render.Renderer {
 type errResponder func(error) render.Renderer
 
 var statusMapping = map[error]errResponder{
-	talkgroups.ErrNoSuchSystem: recordNotFound,
-	talkgroups.ErrNotFound:     recordNotFound,
-	pgx.ErrNoRows:              recordNotFound,
+	tgstore.ErrNoSuchSystem: errTextNotFound,
+	tgstore.ErrNotFound:     errTextNotFound,
+	pgx.ErrNoRows:           recordNotFound,
 }
 
 func autoError(err error) render.Renderer {

pkg/rest/talkgroups.go

@@ -6,7 +6,8 @@ import (
 	"dynatron.me/x/stillbox/internal/forms"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
-	"dynatron.me/x/stillbox/pkg/talkgroups/importer"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
+	"dynatron.me/x/stillbox/pkg/talkgroups/xport"
 
 	"github.com/go-chi/chi/v5"
 )
@@ -17,9 +18,10 @@ type talkgroupAPI struct {
 func (tga *talkgroupAPI) Subrouter() http.Handler {
 	r := chi.NewMux()
 
-	r.Get("/{system:\\d+}/{id:\\d+}", tga.get)
-	r.Put("/{system:\\d+}/{id:\\d+}", tga.put)
-	r.Get("/{system:\\d+}/", tga.get)
+	r.Get(`/{system:\d+}/{id:\d+}`, tga.get)
+	r.Put(`/{system:\d+}/{id:\d+}`, tga.put)
+	r.Put(`/{system:\d+}`, tga.putTalkgroups)
+	r.Get(`/{system:\d+}/`, tga.get)
 	r.Get("/", tga.get)
 	r.Post("/import", tga.tgImport)
 
@@ -31,7 +33,7 @@ type tgParams struct {
 	ID     *int `param:"id"`
 }
 
-func (t tgParams) haveBoth() bool {
+func (t tgParams) hasBoth() bool {
 	return t.System != nil && t.ID != nil
 }
 
@@ -52,7 +54,7 @@ func (t tgParams) ToID() talkgroups.ID {
 
 func (tga *talkgroupAPI) get(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
-	tgs := talkgroups.StoreFrom(ctx)
+	tgs := tgstore.FromCtx(ctx)
 
 	var p tgParams
 
@@ -64,11 +66,12 @@ func (tga *talkgroupAPI) get(w http.ResponseWriter, r *http.Request) {
 
 	var res interface{}
 	switch {
-	case p.System != nil && p.ID != nil:
+	case p.hasBoth():
 		res, err = tgs.TG(ctx, talkgroups.TG(*p.System, *p.ID))
 	case p.System != nil:
 		res, err = tgs.SystemTGs(ctx, int32(*p.System))
 	default:
+		// get all talkgroups
 		res, err = tgs.TGs(ctx, nil)
 	}
 
@@ -89,7 +92,7 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 	}
 
 	ctx := r.Context()
-	tgs := talkgroups.StoreFrom(ctx)
+	tgs := tgstore.FromCtx(ctx)
 
 	input := database.UpdateTalkgroupParams{}
 
@@ -99,6 +102,8 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	input.Learned = nil // ignore for this call
+
 	record, err := tgs.UpdateTG(ctx, input)
 	if err != nil {
 		wErr(w, r, autoError(err))
@@ -109,7 +114,7 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 }
 
 func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) {
-	var impJob importer.ImportJob
+	var impJob xport.ImportJob
 	err := forms.Unmarshal(r, &impJob, forms.WithTag("json"), forms.WithAcceptBlank(), forms.WithOmitEmpty())
 	if err != nil {
 		wErr(w, r, badRequest(err))
@@ -123,3 +128,36 @@ func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) {
 
 	respond(w, r, recs)
 }
+
+func (tga *talkgroupAPI) putTalkgroups(w http.ResponseWriter, r *http.Request) {
+	var id tgParams
+	err := decodeParams(&id, r)
+	if err != nil {
+		wErr(w, r, badRequest(err))
+		return
+	}
+
+	if id.System == nil { // don't think this would ever happen
+		wErr(w, r, badRequest(tgstore.ErrNoSuchSystem))
+		return
+	}
+
+	ctx := r.Context()
+	tgs := tgstore.FromCtx(ctx)
+
+	var input []database.UpsertTalkgroupParams
+
+	err = forms.Unmarshal(r, &input, forms.WithTag("json"), forms.WithAcceptBlank(), forms.WithOmitEmpty())
+	if err != nil {
+		wErr(w, r, badRequest(err))
+		return
+	}
+
+	record, err := tgs.UpsertTGs(ctx, *id.System, input)
+	if err != nil {
+		wErr(w, r, autoError(err))
+		return
+	}
+
+	respond(w, r, record)
+}

pkg/server/routes.go

@@ -9,7 +9,7 @@ import (
 	"dynatron.me/x/stillbox/internal/version"
 	"dynatron.me/x/stillbox/pkg/config"
 	"dynatron.me/x/stillbox/pkg/database"
-	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/httprate"
@@ -28,7 +28,7 @@ func (s *Server) setupRoutes() {
 
 	r := s.r
 	r.Use(middleware.WithValue(database.DBCtxKey, s.db))
-	r.Use(middleware.WithValue(talkgroups.StoreCtxKey, s.tgs))
+	r.Use(middleware.WithValue(tgstore.StoreCtxKey, s.tgs))
 
 	s.installPprof()
 

pkg/server/server.go

@@ -15,7 +15,8 @@ import (
 	"dynatron.me/x/stillbox/pkg/rest"
 	"dynatron.me/x/stillbox/pkg/sinks"
 	"dynatron.me/x/stillbox/pkg/sources"
-	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
+
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
@@ -37,7 +38,7 @@ type Server struct {
 	alerter  alerting.Alerter
 	notifier notify.Notifier
 	hup      chan os.Signal
-	tgs      talkgroups.Store
+	tgs      tgstore.Store
 	rest     rest.API
 }
 
@@ -61,7 +62,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 		return nil, err
 	}
 
-	tgCache := talkgroups.NewCache()
+	tgCache := tgstore.NewCache()
 	api := rest.New()
 
 	srv := &Server{
@@ -78,7 +79,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 		rest:     api,
 	}
 
-	srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db), true)
+	srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db, tgCache), true)
 	srv.sinks.Register("nexus", sinks.NewNexusSink(srv.nex), false)
 
 	if srv.alerter.Enabled() {
@@ -117,7 +118,7 @@ func (s *Server) Go(ctx context.Context) error {
 	s.installHupHandler()
 
 	ctx = database.CtxWithDB(ctx, s.db)
-	ctx = talkgroups.CtxWithStore(ctx, s.tgs)
+	ctx = tgstore.CtxWithStore(ctx, s.tgs)
 
 	httpSrv := &http.Server{
 		Addr:    s.conf.Listen,

pkg/sinks/database.go

@@ -7,6 +7,7 @@ import (
 	"dynatron.me/x/stillbox/internal/common"
 	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/database"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgtype"
@@ -14,11 +15,12 @@ import (
 )
 
 type DatabaseSink struct {
-	db database.Store
+	db  database.Store
+	tgs tgstore.Store
 }
 
-func NewDatabaseSink(store database.Store) *DatabaseSink {
-	return &DatabaseSink{store}
+func NewDatabaseSink(store database.Store, tgs tgstore.Store) *DatabaseSink {
+	return &DatabaseSink{store, tgs}
 }
 
 func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
@@ -43,14 +45,14 @@ func (s *DatabaseSink) Call(ctx context.Context, call *calls.Call) error {
 
 	if err != nil && database.IsTGConstraintViolation(err) {
 		return s.db.InTx(ctx, func(tx database.Store) error {
-			_, err := call.LearnTG(ctx, tx)
+			_, err := s.tgs.LearnTG(ctx, call)
 			if err != nil {
-				return fmt.Errorf("add call: learn tg: %w", err)
+				return fmt.Errorf("learn tg: %w", err)
 			}
 
 			err = tx.AddCall(ctx, params)
 			if err != nil {
-				return fmt.Errorf("add call: retry: %w", err)
+				return fmt.Errorf("learn tg retry: %w", err)
 			}
 
 			return nil

pkg/sources/http.go

@@ -131,7 +131,7 @@ func (h *RdioHTTP) routeCallUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Info().Int("system", cur.System).Int("tgid", cur.Talkgroup).Msg("ingested")
+	log.Info().Int("system", cur.System).Int("tgid", cur.Talkgroup).Str("duration", call.Duration.Duration().String()).Msg("ingested")
 
 	written, err := w.Write([]byte("Call imported successfully."))
 	if err != nil {

pkg/talkgroups/filter/filter.go

@@ -0,0 +1,168 @@
+package filter
+
+import (
+	"context"
+
+	"dynatron.me/x/stillbox/pkg/calls"
+	"dynatron.me/x/stillbox/pkg/database"
+	"dynatron.me/x/stillbox/pkg/pb"
+
+	tgsp "dynatron.me/x/stillbox/pkg/talkgroups"
+)
+
+type TalkgroupFilter struct {
+	Talkgroups       []tgsp.ID `json:"talkgroups,omitempty"`
+	TalkgroupsNot    []tgsp.ID `json:"talkgroupsNot,omitempty"`
+	TalkgroupTagsAll []string  `json:"talkgroupTagsAll,omitempty"`
+	TalkgroupTagsAny []string  `json:"talkgroupTagsAny,omitempty"`
+	TalkgroupTagsNot []string  `json:"talkgroupTagsNot,omitempty"`
+
+	talkgroups map[tgsp.ID]bool `json:"-"`
+}
+
+func (f *TalkgroupFilter) TGs(ctx context.Context) (tgsp.IDs, error) {
+	err := f.ensureCompiled(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	r := make(tgsp.IDs, 0, len(f.talkgroups))
+	for tg := range f.talkgroups {
+		r = append(r, tg)
+	}
+
+	return r, nil
+}
+
+func (f *TalkgroupFilter) Tuples(ctx context.Context) (database.TGTuples, error) {
+	err := f.ensureCompiled(ctx)
+	if err != nil {
+		return database.TGTuples{}, err
+	}
+
+	sys := make([]uint32, len(f.talkgroups))
+	tgs := make([]uint32, len(f.talkgroups))
+
+	i := 0
+	for tg := range f.talkgroups {
+		sys[i] = tg.System
+		tgs[i] = tg.Talkgroup
+	}
+
+	return database.TGTuples{sys, tgs}, nil
+}
+
+func (f *TalkgroupFilter) ensureCompiled(ctx context.Context) error {
+	if f.talkgroups == nil {
+		return f.compile(ctx)
+	}
+
+	return nil
+}
+
+func (tgf *TalkgroupFilter) IsEmpty() bool {
+	if tgf == nil {
+		return true
+	}
+
+	if len(tgf.Talkgroups) > 0 ||
+		len(tgf.TalkgroupsNot) > 0 ||
+		len(tgf.TalkgroupTagsAll) > 0 ||
+		len(tgf.TalkgroupTagsAny) > 0 ||
+		len(tgf.TalkgroupsNot) > 0 {
+		return false
+	}
+
+	return true
+}
+
+func TalkgroupFilterFromPB(ctx context.Context, p *pb.Filter) (*TalkgroupFilter, error) {
+	tgf := &TalkgroupFilter{
+		TalkgroupTagsAll: p.TalkgroupTagsAll,
+		TalkgroupTagsAny: p.TalkgroupTagsAny,
+		TalkgroupTagsNot: p.TalkgroupTagsNot,
+	}
+
+	if l := len(p.Talkgroups); l > 0 {
+		tgf.Talkgroups = make([]tgsp.ID, l)
+		for i, t := range p.Talkgroups {
+			tgf.Talkgroups[i] = tgsp.ID{
+				System:    uint32(t.System),
+				Talkgroup: uint32(t.Talkgroup),
+			}
+		}
+	}
+
+	if l := len(p.TalkgroupsNot); l > 0 {
+		tgf.TalkgroupsNot = make([]tgsp.ID, l)
+		for i, t := range p.TalkgroupsNot {
+			tgf.TalkgroupsNot[i] = tgsp.ID{
+				System:    uint32(t.System),
+				Talkgroup: uint32(t.Talkgroup),
+			}
+		}
+	}
+
+	return tgf, tgf.compile(ctx)
+}
+
+func (f *TalkgroupFilter) hasTags() bool {
+	return len(f.TalkgroupTagsAny) > 0 || len(f.TalkgroupTagsAll) > 0 || len(f.TalkgroupTagsNot) > 0
+}
+
+func (f *TalkgroupFilter) GetFinalTalkgroups() map[tgsp.ID]bool {
+	return f.talkgroups
+}
+
+func (f *TalkgroupFilter) compile(ctx context.Context) error {
+	f.talkgroups = make(map[tgsp.ID]bool)
+	for _, tg := range f.Talkgroups {
+		f.talkgroups[tg] = true
+	}
+
+	if f.hasTags() { // don't bother with DB if no tags
+		db := database.FromCtx(ctx)
+		tagTGs, err := db.GetTalkgroupIDsByTags(ctx, f.TalkgroupTagsAny, f.TalkgroupTagsAll, f.TalkgroupTagsNot)
+		if err != nil {
+			return err
+		}
+
+		for _, tg := range tagTGs {
+			f.talkgroups[tgsp.ID{System: uint32(tg.SystemID), Talkgroup: uint32(tg.TGID)}] = true
+		}
+	}
+
+	for _, tg := range f.TalkgroupsNot {
+		f.talkgroups[tg] = false
+	}
+
+	return nil
+}
+
+func (f *TalkgroupFilter) Test(ctx context.Context, call *calls.Call) bool {
+	if f == nil { // no filter means all calls
+		return true
+	}
+
+	err := f.ensureCompiled(ctx)
+	if err != nil {
+		panic(err)
+	}
+
+	tg := call.TalkgroupTuple()
+
+	tgRes, have := f.talkgroups[tg]
+	if have {
+		return tgRes
+	}
+
+	for _, patch := range call.Patches {
+		tg.Talkgroup = uint32(patch)
+		tgRes, have := f.talkgroups[tg]
+		if have {
+			return tgRes
+		}
+	}
+
+	return false
+}

pkg/talkgroups/talkgroup.go

@@ -2,6 +2,7 @@ package talkgroups
 
 import (
 	"fmt"
+	"strconv"
 
 	"dynatron.me/x/stillbox/pkg/database"
 )
@@ -12,13 +13,20 @@ type Talkgroup struct {
 	Learned bool            `json:"learned"`
 }
 
-type Metadata map[string]interface{}
+func (t Talkgroup) String() string {
+	if t.System.Name == "" {
+		t.System.Name = strconv.Itoa(int(t.Talkgroup.TGID))
+	}
 
-type Names struct {
-	System    string
-	Talkgroup string
+	if t.Talkgroup.Name != nil || t.Talkgroup.TGGroup != nil || t.Talkgroup.AlphaTag != nil {
+		return t.System.Name + " " + t.Talkgroup.String()
+	}
+
+	return fmt.Sprintf("%s:%d", t.System.Name, int(t.Talkgroup.TGID))
 }
 
+type Metadata map[string]interface{}
+
 type ID struct {
 	System    uint32 `json:"sys"`
 	Talkgroup uint32 `json:"tg"`

pkg/talkgroups/tgstore/store.go

@@ -1,19 +1,24 @@
-package talkgroups
+package tgstore
 
 import (
 	"context"
 	"errors"
+	"strings"
 	"sync"
 	"time"
 
+	"dynatron.me/x/stillbox/internal/common"
+	"dynatron.me/x/stillbox/pkg/auth"
+	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/config"
 	"dynatron.me/x/stillbox/pkg/database"
+	tgsp "dynatron.me/x/stillbox/pkg/talkgroups"
 
 	"github.com/jackc/pgx/v5"
 	"github.com/rs/zerolog/log"
 )
 
-type tgMap map[ID]*Talkgroup
+type tgMap map[tgsp.ID]*tgsp.Talkgroup
 
 var (
 	ErrNotFound     = errors.New("talkgroup not found")
@@ -22,22 +27,28 @@ var (
 
 type Store interface {
 	// UpdateTG updates a talkgroup record.
-	UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*Talkgroup, error)
+	UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*tgsp.Talkgroup, error)
+
+	// UpsertTGs upserts a slice of talkgroups.
+	UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*tgsp.Talkgroup, error)
 
 	// TG retrieves a Talkgroup from the Store.
-	TG(ctx context.Context, tg ID) (*Talkgroup, error)
+	TG(ctx context.Context, tg tgsp.ID) (*tgsp.Talkgroup, error)
 
 	// TGs retrieves many talkgroups from the Store.
-	TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error)
+	TGs(ctx context.Context, tgs tgsp.IDs) ([]*tgsp.Talkgroup, error)
+
+	// LearnTG learns the talkgroup from a Call.
+	LearnTG(ctx context.Context, call *calls.Call) (*tgsp.Talkgroup, error)
 
 	// SystemTGs retrieves all Talkgroups associated with a System.
-	SystemTGs(ctx context.Context, systemID int32) ([]*Talkgroup, error)
+	SystemTGs(ctx context.Context, systemID int32) ([]*tgsp.Talkgroup, error)
 
 	// SystemName retrieves a system name from the store. It returns the record and whether one was found.
 	SystemName(ctx context.Context, id int) (string, bool)
 
 	// Hint hints the Store that the provided talkgroups will be asked for.
-	Hint(ctx context.Context, tgs []ID) error
+	Hint(ctx context.Context, tgs []tgsp.ID) error
 
 	// Load loads the provided talkgroup ID tuples into the Store.
 	Load(ctx context.Context, tgs database.TGTuples) error
@@ -46,7 +57,7 @@ type Store interface {
 	Invalidate()
 
 	// Weight returns the final weight of this talkgroup, including its static and rules-derived weight.
-	Weight(ctx context.Context, id ID, t time.Time) float64
+	Weight(ctx context.Context, id tgsp.ID, t time.Time) float64
 
 	// Hupper
 	HUP(*config.Config)
@@ -60,7 +71,7 @@ func CtxWithStore(ctx context.Context, s Store) context.Context {
 	return context.WithValue(ctx, StoreCtxKey, s)
 }
 
-func StoreFrom(ctx context.Context) Store {
+func FromCtx(ctx context.Context) Store {
 	s, ok := ctx.Value(StoreCtxKey).(Store)
 	if !ok {
 		return NewCache()
@@ -96,7 +107,7 @@ func NewCache() Store {
 	return tgc
 }
 
-func (t *cache) Hint(ctx context.Context, tgs []ID) error {
+func (t *cache) Hint(ctx context.Context, tgs []tgsp.ID) error {
 	t.RLock()
 	var toLoad database.TGTuples
 	if len(t.tgs) > len(tgs)/2 { // TODO: instrument this
@@ -124,15 +135,22 @@ func (t *cache) Hint(ctx context.Context, tgs []ID) error {
 	return nil
 }
 
-func (t *cache) add(rec *Talkgroup) error {
+func (t *cache) get(id tgsp.ID) (*tgsp.Talkgroup, bool) {
+	t.RLock()
+	defer t.RUnlock()
+
+	tg, has := t.tgs[id]
+
+	return tg, has
+}
+
+func (t *cache) add(rec *tgsp.Talkgroup) {
 	t.Lock()
 	defer t.Unlock()
 
-	tg := TG(rec.System.ID, rec.Talkgroup.TGID)
+	tg := tgsp.TG(rec.System.ID, rec.Talkgroup.TGID)
 	t.tgs[tg] = rec
 	t.systems[int32(rec.System.ID)] = rec.System.Name
-
-	return nil
 }
 
 type row interface {
@@ -143,49 +161,44 @@ type row interface {
 	GetLearned() bool
 }
 
-func rowToTalkgroup[T row](r T) *Talkgroup {
-	return &Talkgroup{
+func rowToTalkgroup[T row](r T) *tgsp.Talkgroup {
+	return &tgsp.Talkgroup{
 		Talkgroup: r.GetTalkgroup(),
 		System:    r.GetSystem(),
 		Learned:   r.GetLearned(),
 	}
 }
 
-func addToRowList[T row](t *cache, r []*Talkgroup, tgRecords []T) ([]*Talkgroup, error) {
+func addToRowList[T row](t *cache, r []*tgsp.Talkgroup, tgRecords []T) []*tgsp.Talkgroup {
 	for _, rec := range tgRecords {
 		tg := rowToTalkgroup(rec)
-		err := t.add(tg)
-		if err != nil {
-			return nil, err
-		}
+		t.add(tg)
 
 		r = append(r, tg)
 	}
 
-	return r, nil
+	return r
 }
 
-func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
-	r := make([]*Talkgroup, 0, len(tgs))
+func (t *cache) TGs(ctx context.Context, tgs tgsp.IDs) ([]*tgsp.Talkgroup, error) {
+	r := make([]*tgsp.Talkgroup, 0, len(tgs))
 	var err error
 	if tgs != nil {
-		toGet := make(IDs, 0, len(tgs))
-		t.RLock()
+		toGet := make(tgsp.IDs, 0, len(tgs))
 		for _, id := range tgs {
-			rec, has := t.tgs[id]
+			rec, has := t.get(id)
 			if has {
 				r = append(r, rec)
 			} else {
 				toGet = append(toGet, id)
 			}
 		}
-		t.RUnlock()
 
 		tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, toGet.Tuples())
 		if err != nil {
 			return nil, err
 		}
-		return addToRowList(t, r, tgRecords)
+		return addToRowList(t, r, tgRecords), nil
 	}
 
 	// get all talkgroups
@@ -194,7 +207,7 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
 	if err != nil {
 		return nil, err
 	}
-	return addToRowList(t, r, tgRecords)
+	return addToRowList(t, r, tgRecords), nil
 }
 
 func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error {
@@ -204,17 +217,13 @@ func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error {
 	}
 
 	for _, rec := range tgRecords {
-		err := t.add(rowToTalkgroup(rec))
-
-		if err != nil {
-			log.Error().Err(err).Msg("add alert config fail")
-		}
+		t.add(rowToTalkgroup(rec))
 	}
 
 	return nil
 }
 
-func (t *cache) Weight(ctx context.Context, id ID, tm time.Time) float64 {
+func (t *cache) Weight(ctx context.Context, id tgsp.ID, tm time.Time) float64 {
 	tg, err := t.TG(ctx, id)
 	if err != nil {
 		return 1.0
@@ -227,20 +236,18 @@ func (t *cache) Weight(ctx context.Context, id ID, tm time.Time) float64 {
 	return float64(m)
 }
 
-func (t *cache) SystemTGs(ctx context.Context, systemID int32) ([]*Talkgroup, error) {
+func (t *cache) SystemTGs(ctx context.Context, systemID int32) ([]*tgsp.Talkgroup, error) {
 	recs, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySystem(ctx, systemID)
 	if err != nil {
 		return nil, err
 	}
 
-	r := make([]*Talkgroup, 0, len(recs))
-	return addToRowList(t, r, recs)
+	r := make([]*tgsp.Talkgroup, 0, len(recs))
+	return addToRowList(t, r, recs), nil
 }
 
-func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
-	t.RLock()
-	rec, has := t.tgs[tg]
-	t.RUnlock()
+func (t *cache) TG(ctx context.Context, tg tgsp.ID) (*tgsp.Talkgroup, error) {
+	rec, has := t.get(tg)
 
 	if has {
 		return rec, nil
@@ -256,11 +263,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
 		return nil, errors.Join(ErrNotFound, err)
 	}
 
-	err = t.add(rowToTalkgroup(record))
-	if err != nil {
-		log.Error().Err(err).Msg("TG() cache add")
-		return rowToTalkgroup(record), errors.Join(ErrNotFound, err)
-	}
+	t.add(rowToTalkgroup(record))
 
 	return rowToTalkgroup(record), nil
 }
@@ -286,7 +289,7 @@ func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool)
 	return n, has
 }
 
-func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*Talkgroup, error) {
+func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*tgsp.Talkgroup, error) {
 	sysName, has := t.SystemName(ctx, int(*input.SystemID))
 	if !has {
 		return nil, ErrNoSuchSystem
@@ -297,7 +300,7 @@ func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupPara
 		return nil, err
 	}
 
-	record := &Talkgroup{
+	record := &tgsp.Talkgroup{
 		Talkgroup: tg,
 		System:    database.System{ID: int(tg.SystemID), Name: sysName},
 	}
@@ -305,3 +308,113 @@ func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupPara
 
 	return record, nil
 }
+
+func (t *cache) LearnTG(ctx context.Context, c *calls.Call) (*tgsp.Talkgroup, error) {
+	db := database.FromCtx(ctx)
+
+	sys, has := t.SystemName(ctx, c.System)
+	if !has {
+		return nil, ErrNoSuchSystem
+	}
+
+	tgm, err := db.AddLearnedTalkgroup(ctx, database.AddLearnedTalkgroupParams{
+		SystemID: int32(c.System),
+		TGID:     int32(c.Talkgroup),
+		Name:     c.TalkgroupLabel,
+		AlphaTag: c.TGAlphaTag,
+		TGGroup:  c.TalkgroupGroup,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	tg := &tgsp.Talkgroup{
+		Talkgroup: tgm,
+		System: database.System{
+			ID:   c.System,
+			Name: sys,
+		},
+		Learned: tgm.Learned,
+	}
+
+	t.add(tg)
+
+	return tg, nil
+}
+
+func (t *cache) UpsertTGs(ctx context.Context, system int, input []database.UpsertTalkgroupParams) ([]*tgsp.Talkgroup, error) {
+	db := database.FromCtx(ctx)
+	sysName, hasSys := t.SystemName(ctx, system)
+	if !hasSys {
+		return nil, ErrNoSuchSystem
+	}
+	sys := database.System{
+		ID:   system,
+		Name: sysName,
+	}
+
+	tgs := make([]*tgsp.Talkgroup, 0, len(input))
+
+	err := db.InTx(ctx, func(db database.Store) error {
+		versionParams := make([]database.StoreTGVersionParams, 0, len(input))
+		for i := range input {
+			// normalize tags
+			for j, tag := range input[i].Tags {
+				input[i].Tags[j] = strings.ToLower(tag)
+			}
+
+			input[i].SystemID = int32(system)
+			input[i].Learned = common.PtrTo(false)
+
+		}
+
+		var oerr error
+
+		tgUpsertBatch := db.UpsertTalkgroup(ctx, input)
+		defer tgUpsertBatch.Close()
+
+		tgUpsertBatch.QueryRow(func(_ int, r database.Talkgroup, err error) {
+			if err != nil {
+				oerr = err
+				return
+			}
+			versionParams = append(versionParams, database.StoreTGVersionParams{
+				SystemID:  int32(system),
+				TGID:      r.TGID,
+				Submitter: auth.UIDFrom(ctx),
+			})
+			tgs = append(tgs, &tgsp.Talkgroup{
+				Talkgroup: r,
+				System:    sys,
+				Learned:   r.Learned,
+			})
+		})
+
+		if oerr != nil {
+			return oerr
+		}
+
+		versionBatch := db.StoreTGVersion(ctx, versionParams)
+		defer versionBatch.Close()
+
+		versionBatch.Exec(func(_ int, err error) {
+			if err != nil {
+				oerr = err
+				return
+			}
+		})
+
+		return oerr
+	}, pgx.TxOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// update the cache
+	for _, tg := range tgs {
+		t.add(tg)
+	}
+
+	return tgs, nil
+}

pkg/talkgroups/xport/export.go

@@ -0,0 +1,54 @@
+package xport
+
+import (
+	"context"
+	"io"
+
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/filter"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
+	"dynatron.me/x/stillbox/pkg/talkgroups/xport/sdrtrunk"
+)
+
+type Exporter interface {
+	ExportTalkgroups(ctx context.Context, w io.Writer, tgs []*talkgroups.Talkgroup, tmpl []byte) error
+}
+
+type ExportJob struct {
+	Type     Format `json:"type"`
+	SystemID int    `json:"systemID"`
+	Template []byte `json:"template"`
+
+	filter.TalkgroupFilter
+	Exporter
+}
+
+func (ej *ExportJob) Export(ctx context.Context, w io.Writer) error {
+	var tgs []*talkgroups.Talkgroup
+	var err error
+	tgst := tgstore.FromCtx(ctx)
+	if ej.TalkgroupFilter.IsEmpty() {
+		tgs, err = tgst.SystemTGs(ctx, int32(ej.SystemID))
+		if err != nil {
+			return err
+		}
+	} else {
+		ids, err := ej.TalkgroupFilter.TGs(ctx)
+		if err != nil {
+			return err
+		}
+		tgs, err = tgst.TGs(ctx, ids)
+		if err != nil {
+			return err
+		}
+	}
+
+	switch ej.Type {
+	case FormatSDRTrunk:
+		ej.Exporter = sdrtrunk.New()
+	default:
+		return ErrBadType
+	}
+
+	return ej.ExportTalkgroups(ctx, w, tgs, ej.Template)
+}

pkg/talkgroups/xport/format.go

@@ -0,0 +1,16 @@
+package xport
+
+import (
+	"errors"
+)
+
+type Format string
+
+const (
+	FormatRadioReference Format = "radioreference"
+	FormatSDRTrunk       Format = "sdrtrunk"
+)
+
+var (
+	ErrBadType = errors.New("unknown format type")
+)

pkg/talkgroups/xport/import.go

@@ -0,0 +1,35 @@
+package xport
+
+import (
+	"bytes"
+	"context"
+	"io"
+
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/xport/radioref"
+)
+
+type Importer interface {
+	ImportTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error)
+}
+
+type ImportJob struct {
+	Type     Format `json:"type"`
+	SystemID int    `json:"systemID"`
+	Body     string `json:"body"`
+
+	Importer `json:"-"`
+}
+
+func (ij *ImportJob) Import(ctx context.Context) ([]talkgroups.Talkgroup, error) {
+	r := bytes.NewReader([]byte(ij.Body))
+
+	switch ij.Type {
+	case FormatRadioReference:
+		ij.Importer = radioref.New()
+	default:
+		return nil, ErrBadType
+	}
+
+	return ij.ImportTalkgroups(ctx, ij.SystemID, r)
+}

pkg/talkgroups/xport/import_test.go

@@ -0,0 +1,51 @@
+package xport_test
+
+import (
+	"context"
+	"testing"
+
+	"dynatron.me/x/stillbox/pkg/talkgroups/xport"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImport(t *testing.T) {
+	tests := []struct {
+		name      string
+		impType   string
+		input     []byte
+		sysID     int
+		sysName   string
+		jsExpect  []byte
+		expectErr error
+	}{
+		{
+			name:      "unknown importer",
+			impType:   "nonexistent",
+			expectErr: xport.ErrBadType,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			ij := &xport.ImportJob{
+				Type:     xport.Format(tc.impType),
+				SystemID: tc.sysID,
+				Body:     string(tc.input),
+			}
+
+			_, err := ij.Import(ctx)
+
+			if tc.expectErr != nil {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tc.expectErr.Error())
+			} else {
+				require.NoError(t, err)
+
+			}
+
+		})
+	}
+}

pkg/talkgroups/xport/radioref/radioreference.go

@@ -1,10 +1,8 @@
-package importer
+package radioref
 
 import (
 	"bufio"
-	"bytes"
 	"context"
-	"errors"
 	"io"
 	"regexp"
 	"strconv"
@@ -13,44 +11,13 @@ import (
 	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
 )
 
-type ImportSource string
+type Driver struct{}
 
-const (
-	ImportSrcRadioReference ImportSource = "radioreference"
-)
-
-var (
-	ErrBadImportType = errors.New("unknown import type")
-)
-
-type importer interface {
-	importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error)
-}
-
-type ImportJob struct {
-	Type     ImportSource `json:"type"`
-	SystemID int          `json:"systemID"`
-	Body     string       `json:"body"`
-
-	importer `json:"-"`
-}
-
-func (ij *ImportJob) Import(ctx context.Context) ([]talkgroups.Talkgroup, error) {
-	r := bytes.NewReader([]byte(ij.Body))
-
-	switch ij.Type {
-	case ImportSrcRadioReference:
-		ij.importer = new(radioReferenceImporter)
-	default:
-		return nil, ErrBadImportType
-	}
-
-	return ij.importTalkgroups(ctx, ij.SystemID, r)
-}
-
-type radioReferenceImporter struct {
+func New() *Driver {
+	return new(Driver)
 }
 
 type rrState int
@@ -63,12 +30,12 @@ const (
 
 var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`)
 
-func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error) {
+func (rr *Driver) ImportTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error) {
 	sc := bufio.NewScanner(r)
 	tgs := make([]talkgroups.Talkgroup, 0, 8)
-	sysn, has := talkgroups.StoreFrom(ctx).SystemName(ctx, sys)
+	sysn, has := tgstore.FromCtx(ctx).SystemName(ctx, sys)
 	if !has {
-		return nil, talkgroups.ErrNoSuchSystem
+		return nil, tgstore.ErrNoSuchSystem
 	}
 
 	var groupName string
@@ -110,6 +77,7 @@ func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int,
 			gn := groupName // must take a copy
 			tgs = append(tgs, talkgroups.Talkgroup{
 				Talkgroup: database.Talkgroup{
+					ID:       len(tgs), // need unique ID for the UI to track
 					TGID:     int32(tgt.Talkgroup),
 					SystemID: int32(tgt.System),
 					Name:     &fields[4],
@@ -118,6 +86,7 @@ func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int,
 					Metadata: metadata,
 					Tags:     tags,
 					Weight:   1.0,
+					Alert:    true,
 				},
 				System: database.System{
 					ID:   sys,

pkg/talkgroups/xport/radioref/radioreference_test.go

@@ -1,4 +1,4 @@
-package importer_test
+package radioref_test
 
 import (
 	"context"
@@ -15,7 +15,8 @@ import (
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/database/mocks"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
-	"dynatron.me/x/stillbox/pkg/talkgroups/importer"
+	"dynatron.me/x/stillbox/pkg/talkgroups/tgstore"
+	"dynatron.me/x/stillbox/pkg/talkgroups/xport"
 )
 
 func getFixture(fixture string) []byte {
@@ -27,7 +28,7 @@ func getFixture(fixture string) []byte {
 	return fixt
 }
 
-func TestImport(t *testing.T) {
+func TestRadioRef(t *testing.T) {
 	// this is for deterministic UUIDs
 	uuid.SetRand(rand.New(rand.NewSource(1)))
 
@@ -41,18 +42,13 @@ func TestImport(t *testing.T) {
 		expectErr error
 	}{
 		{
-			name:     "radioreference",
+			name:     "radioreference import",
 			impType:  "radioreference",
 			input:    getFixture("riscon.txt"),
 			jsExpect: getFixture("riscon.json"),
 			sysID:    197,
 			sysName:  "RISCON",
 		},
-		{
-			name:      "unknown importer",
-			impType:   "nonexistent",
-			expectErr: importer.ErrBadImportType,
-		},
 	}
 
 	for _, tc := range tests {
@@ -62,9 +58,9 @@ func TestImport(t *testing.T) {
 				dbMock.EXPECT().GetSystemName(mock.AnythingOfType("*context.valueCtx"), tc.sysID).Return(tc.sysName, nil)
 			}
 			ctx := database.CtxWithDB(context.Background(), dbMock)
-			ctx = talkgroups.CtxWithStore(ctx, talkgroups.NewCache())
-			ij := &importer.ImportJob{
-				Type:     importer.ImportSource(tc.impType),
+			ctx = tgstore.CtxWithStore(ctx, tgstore.NewCache())
+			ij := &xport.ImportJob{
+				Type:     xport.Format(tc.impType),
 				SystemID: tc.sysID,
 				Body:     string(tc.input),
 			}

pkg/talkgroups/xport/sdrtrunk/sdrtrunk.go

@@ -0,0 +1,128 @@
+package sdrtrunk
+
+import (
+	"context"
+	"encoding/xml"
+	"io"
+
+	"dynatron.me/x/stillbox/internal/common"
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+)
+
+type Playlist struct {
+	XMLName  xml.Name  `xml:"playlist"`
+	Version  int       `xml:"version,attr"`
+	Aliases  []Alias   `xml:"alias"`
+	Channels []Channel `xml:"channel,omitempty"`
+	Streams  []Stream  `xml:"stream,omitempty"`
+}
+
+type Alias struct {
+	XMLName  xml.Name `xml:"alias"`
+	Name     string   `xml:"name,attr,omitempty"`
+	Color    int      `xml:"color,attr,omitempty"`
+	Group    string   `xml:"group,attr,omitempty"`
+	IconName string   `xml:"iconName,attr,omitempty"`
+	List     string   `xml:"list,attr,omitempty"`
+	IDs      []ID     `xml:"id"`
+}
+
+func tgToAlias(tg *talkgroups.Talkgroup) Alias {
+	return Alias{
+		XMLName: xml.Name{Local: "alias"},
+		Name:    common.ZeroOr(tg.Name),
+		Group:   common.ZeroOr(tg.TGGroup),
+		List:    "Stillbox",
+		IDs: []ID{
+			ID{
+				XMLName: xml.Name{Local: "id"},
+				Type:    "talkgroup",
+				Value:   common.PtrTo(int(tg.TGID)),
+			},
+		},
+	}
+}
+
+type ID struct {
+	XMLName  xml.Name `xml:"id"`
+	Type     string   `xml:"type,attr"`
+	Priority *int     `xml:"priority,attr,omitempty"`
+	Channel  *string  `xml:"channel,attr,omitempty"`
+	Protocol *string  `xml:"protocol,attr,omitempty"`
+	Value    *int     `xml:"value,attr,omitempty"`
+	Min      *int     `xml:"min,attr,omitempty"`
+	Max      *int     `xml:"max,attr,omitempty"`
+}
+
+type Channel struct {
+	XMLName xml.Name `xml:"channel"`
+	Name    string   `xml:"name,attr"`
+	System  string   `xml:"system,attr"`
+	Enabled bool     `xml:"enabled,attr"`
+	Site    string   `xml:"site,attr"`
+	Order   int      `xml:"order,attr"`
+
+	AliasListName   string          `xml:"alias_list_name"`
+	EventLogConfig  EventLogConfig  `xml:"event_log_configuration"`
+	SourceConfig    SourceConfig    `xml:"source_configuration"`
+	AuxDecodeConfig AuxDecodeConfig `xml:"aux_decode_configuration"`
+	DecodeConfig    DecodeConfig    `xml:"decode_configuration"`
+	RecordConfig    RecordConfig    `xml:"record_configuration"`
+}
+
+type EventLogConfig struct {
+	EventLogConfig []byte `xml:",innerxml"`
+}
+
+type SourceConfig struct {
+	SourceConfig []byte `xml:",innerxml"`
+}
+
+type AuxDecodeConfig struct {
+	AuxDecodeConfig []byte `xml:",innerxml"`
+}
+
+type DecodeConfig struct {
+	DecodeConfig []byte `xml:",innerxml"`
+}
+
+type RecordConfig struct {
+	RecordConfig []byte `xml:",innerxml"`
+}
+
+type Stream struct {
+	Attributes []xml.Attr `xml:",any,attr"`
+	Stream     []byte     `xml:",innerxml"`
+}
+
+func New() *Driver {
+	return new(Driver)
+}
+
+type Driver struct{}
+
+func (st *Driver) ExportTalkgroups(ctx context.Context, w io.Writer, tgs []*talkgroups.Talkgroup, tmpl []byte) error {
+	var pl Playlist
+
+	if tmpl != nil {
+		err := xml.Unmarshal(tmpl, &pl)
+		if err != nil {
+			return err
+		}
+
+		pl.Aliases = nil
+	}
+
+	for _, tg := range tgs {
+		pl.Aliases = append(pl.Aliases, tgToAlias(tg))
+	}
+
+	enc := xml.NewEncoder(w)
+	enc.Indent("", "  ")
+	err := enc.Encode(&pl)
+	if err != nil {
+		return err
+	}
+
+	return enc.Close()
+}

sql/postgres/migrations/001_initial.up.sql

@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS talkgroups(
 	alert_config JSONB,
 	weight REAL NOT NULL DEFAULT 1.0,
 	learned BOOLEAN NOT NULL DEFAULT FALSE,
+	ignored BOOLEAN NOT NULL DEFAULT FALSE,
 	UNIQUE (system_id, tgid)
 );
 
@@ -44,15 +45,25 @@ CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
 
 CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
 
-CREATE TABLE IF NOT EXISTS talkgroups_learned(
+CREATE TABLE IF NOT EXISTS talkgroup_versions(
+	-- version metadata
 	id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
-	system_id INTEGER REFERENCES systems(id) NOT NULL,
-	tgid INTEGER NOT NULL,
-	name TEXT NOT NULL,
+	time TIMESTAMPTZ NOT NULL,
+	created_by INTEGER REFERENCES users(id),
+	-- talkgroup snapshot
+	system_id INT4 REFERENCES systems(id),
+	tgid INT4,
+	name TEXT,
 	alpha_tag TEXT,
 	tg_group TEXT,
-	ignored BOOLEAN NOT NULL DEFAULT FALSE,
-	UNIQUE (system_id, tgid, name)
+	frequency INTEGER,
+	metadata JSONB,
+	tags TEXT[],
+	alert BOOLEAN,
+	alert_config JSONB,
+	weight REAL,
+	learned BOOLEAN,
+	ignored BOOLEAN
 );
 
 CREATE TABLE IF NOT EXISTS alerts(

sql/postgres/queries/talkgroups.sql

@@ -29,47 +29,20 @@ SELECT
 sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid) AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;
+WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid);
 
 -- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
 sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.system_id = @system AND tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = @system AND ignored IS NOT TRUE;
+WHERE tg.system_id = @system;
 
 -- name: GetTalkgroupsWithLearned :many
 SELECT
 sqlc.embed(tg), sqlc.embed(sys)
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.learned IS NOT TRUE
-UNION
-SELECT
-tgl.id, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.tg_group, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.tg_group IS NULL THEN NULL ELSE ARRAY[tgl.tg_group] END,
-NOT tgl.ignored, NULL::JSONB, 1.0, TRUE learned, sys.id, sys.name
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE;
 
 -- name: GetSystemName :one
@@ -86,32 +59,128 @@ SET
 	tags = COALESCE(sqlc.narg('tags'), tags),
 	alert = COALESCE(sqlc.narg('alert'), alert),
 	alert_config = COALESCE(sqlc.narg('alert_config'), alert_config),
-	weight = COALESCE(sqlc.narg('weight'), weight)
+	weight = COALESCE(sqlc.narg('weight'), weight),
+	learned = COALESCE(sqlc.narg('learned'), learned)
 WHERE id = sqlc.narg('id') OR (system_id = sqlc.narg('system_id') AND tgid = sqlc.narg('tgid'))
 RETURNING *;
 
--- name: AddTalkgroupWithLearnedFlag :exec
-INSERT INTO talkgroups (
-	system_id,
-	tgid,
-	learned
-) VALUES(
+-- name: UpsertTalkgroup :batchone
+INSERT INTO talkgroups AS tg (
+	system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight, learned
+) VALUES (
 	@system_id,
 	@tgid,
-	't'
-);
+	sqlc.narg('name'),
+	sqlc.narg('alpha_tag'),
+	sqlc.narg('tg_group'),
+	sqlc.narg('frequency'),
+	sqlc.narg('metadata'),
+	sqlc.narg('tags'),
+	sqlc.narg('alert'),
+	sqlc.narg('alert_config'),
+	sqlc.narg('weight'),
+	sqlc.narg('learned')
+)
+ON CONFLICT (system_id, tgid) DO UPDATE
+SET
+	name = COALESCE(sqlc.narg('name'), tg.name),
+	alpha_tag = COALESCE(sqlc.narg('alpha_tag'), tg.alpha_tag),
+	tg_group = COALESCE(sqlc.narg('tg_group'), tg.tg_group),
+	frequency = COALESCE(sqlc.narg('frequency'), tg.frequency),
+	metadata = COALESCE(sqlc.narg('metadata'), tg.metadata),
+	tags = COALESCE(sqlc.narg('tags'), tg.tags),
+	alert = COALESCE(sqlc.narg('alert'), tg.alert),
+	alert_config = COALESCE(sqlc.narg('alert_config'), tg.alert_config),
+	weight = COALESCE(sqlc.narg('weight'), tg.weight),
+	learned = COALESCE(sqlc.narg('learned'), tg.learned)
+RETURNING *;
 
--- name: AddLearnedTalkgroup :one
-INSERT INTO talkgroups_learned(
+-- name: StoreTGVersion :batchexec
+INSERT INTO talkgroup_versions(time, created_by,
 	system_id,
 	tgid,
 	name,
 	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned
+) SELECT NOW(), @submitter,
+	tg.system_id,
+	tg.tgid,
+	tg.name,
+	tg.alpha_tag,
+	tg.tg_group,
+	tg.frequency,
+	tg.metadata,
+	tg.tags,
+	tg.alert,
+	tg.alert_config,
+	tg.weight,
+	tg.learned
+FROM talkgroups tg WHERE tg.system_id = @system_id AND tg.tgid = @tgid;
+
+-- name: AddLearnedTalkgroup :one
+INSERT INTO talkgroups(
+	system_id,
+	tgid,
+	learned,
+	name,
+	alpha_tag,
 	tg_group
 ) VALUES (
 	@system_id,
 	@tgid,
+	TRUE,
 	sqlc.narg('name'),
 	sqlc.narg('alpha_tag'),
 	sqlc.narg('tg_group')
-) RETURNING id;
+) RETURNING *;
+
+-- name: RestoreTalkgroupVersion :one
+INSERT INTO talkgroups(
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned,
+	ignored
+)
+SELECT
+	system_id,
+	tgid,
+	name,
+	alpha_tag,
+	tg_group,
+	frequency,
+	metadata,
+	tags,
+	alert,
+	alert_config,
+	weight,
+	learned,
+	ignored
+FROM talkgroup_versions tgv ON CONFLICT (system_id, tgid) DO UPDATE SET
+	name = excluded.name,
+	alpha_tag = excluded.alpha_tag,
+	tg_group = excluded.tg_group,
+	metadata = excluded.metadata,
+	tags = excluded.tags,
+	alert = excluded.alert,
+	alert_config = excluded.alert_config,
+	weight = excluded.weight,
+	learned = excluded.learner,
+	ignored = excluded.ignored
+WHERE tgv.id = ANY(@version_ids)
+RETURNING *;