Merge pull request 'New tg ID schema and initial importer' (#35) from reid into trunk

Reviewed-on: #35

.gitignore

@@ -1,6 +1,6 @@
 config.yaml
 config.test.yaml
-mydb.sql
+/*.sql
 client/calls/
 !client/calls/.gitkeep
 /gordio
@@ -10,3 +10,4 @@ Session.vim
 *.log
 *.dlv
 cover.out
+backups/

.mockery.yaml

@@ -0,0 +1,10 @@
+dir: '{{ replaceAll .InterfaceDirRelative "internal" "internal_" }}/mocks'
+mockname: "{{.InterfaceName}}"
+outpkg: "mocks"
+filename: "{{.InterfaceName}}.go"
+with-expecter: true
+packages:
+  dynatron.me/x/stillbox/pkg/database:
+    config:
+    interfaces:
+      DB:

Makefile

@@ -2,6 +2,7 @@ VPKG=dynatron.me/x/stillbox/internal/version
 VER!=git describe --tags --always --dirty
 BUILDDATE!=date '+%Y%m%d'
 LDFLAGS=-ldflags="-X '${VPKG}.Version=${VER}' -X '${VPKG}.Built=${BUILDDATE}'"
+GOFLAGS=-v
 
 all: checkcalls
 	go build -o stillbox ${GOFLAGS} ${LDFLAGS} ./cmd/stillbox/
@@ -24,6 +25,7 @@ getcalls:
 generate:
 	sqlc generate -f sql/sqlc.yaml
 	protoc -I=pkg/pb/ --go_out=pkg/ pkg/pb/stillbox.proto
+	go generate ./...
 
 lint:
 	golangci-lint run
@@ -34,5 +36,15 @@ coverage-html:
 coverage:
 	go test -coverprofile cover.out
 
+# backup backs up the database without calls
+backup:
+	sh util/dumpdb.sh
+
+backupplain:
+	sh util/dumpdb.sh -p
+
 test:
 	go test -v ./...
+
+run:
+	go run -v ./cmd/stillbox/ serve

cmd/stillbox/main.go

@@ -24,7 +24,7 @@ func main() {
 	}
 	rootCmd.PersistentFlags().BoolP("version", "V", false, "show version")
 	cfg := config.New(rootCmd)
-	rootCmd.Run = func(cmd *cobra.Command, args []string) {
+	rootCmd.PreRun = func(cmd *cobra.Command, args []string) {
 		v, _ := rootCmd.PersistentFlags().GetBool("version")
 		if v {
 			fmt.Print(version.String())

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/exp/shiny v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/image v0.14.0 // indirect

go.sum

@@ -134,6 +134,8 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/common/template.go

@@ -7,7 +7,7 @@ import (
 	"text/template"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 )
 
 var (
@@ -27,7 +27,7 @@ var (
 			}
 			return dict, nil
 		},
-		"formTime": func(t jsontime.Time) string {
+		"formTime": func(t jsontypes.Time) string {
 			return time.Time(t).Format("2006-01-02T15:04")
 		},
 		"ago": func(s string) (string, error) {

internal/forms/forms.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"github.com/araddon/dateparse"
 )
@@ -262,13 +262,13 @@ func (o *options) iterFields(r *http.Request, destStruct reflect.Value) error {
 				return err
 			}
 			setVal(destFieldVal, set, val)
-		case time.Time, *time.Time, jsontime.Time, *jsontime.Time:
+		case time.Time, *time.Time, jsontypes.Time, *jsontypes.Time:
 			t, set, err := o.parseTime(ff)
 			if err != nil {
 				return err
 			}
 			setVal(destFieldVal, set, t)
-		case time.Duration, *time.Duration, jsontime.Duration, *jsontime.Duration:
+		case time.Duration, *time.Duration, jsontypes.Duration, *jsontypes.Duration:
 			d, set, err := o.parseDuration(ff)
 			if err != nil {
 				return err

internal/forms/forms_test.go

@@ -10,7 +10,7 @@ import (
 
 	"dynatron.me/x/stillbox/internal/common"
 	"dynatron.me/x/stillbox/internal/forms"
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"dynatron.me/x/stillbox/pkg/alerting"
 	"dynatron.me/x/stillbox/pkg/config"
@@ -48,19 +48,19 @@ type urlEncTest struct {
 }
 
 type urlEncTestJT struct {
-	LookbackDays uint              `json:"lookbackDays"`
+	LookbackDays uint               `json:"lookbackDays"`
-	HalfLife     jsontime.Duration `json:"halfLife"`
+	HalfLife     jsontypes.Duration `json:"halfLife"`
-	Recent       string            `json:"recent"`
+	Recent       string             `json:"recent"`
-	ScoreStart   jsontime.Time     `json:"scoreStart"`
+	ScoreStart   jsontypes.Time     `json:"scoreStart"`
-	ScoreEnd     jsontime.Time     `json:"scoreEnd"`
+	ScoreEnd     jsontypes.Time     `json:"scoreEnd"`
 }
 
 type ptrTestJT struct {
-	LookbackDays uint               `form:"lookbackDays"`
+	LookbackDays uint                `form:"lookbackDays"`
-	HalfLife     *jsontime.Duration `form:"halfLife"`
+	HalfLife     *jsontypes.Duration `form:"halfLife"`
-	Recent       *string            `form:"recent"`
+	Recent       *string             `form:"recent"`
-	ScoreStart   *jsontime.Time     `form:"scoreStart"`
+	ScoreStart   *jsontypes.Time     `form:"scoreStart"`
-	ScoreEnd     jsontime.Time      `form:"scoreEnd"`
+	ScoreEnd     jsontypes.Time      `form:"scoreEnd"`
 }
 
 var (
@@ -73,33 +73,33 @@ var (
 
 	UrlEncTestJT = urlEncTestJT{
 		LookbackDays: 7,
-		HalfLife:     jsontime.Duration(30 * time.Minute),
+		HalfLife:     jsontypes.Duration(30 * time.Minute),
 		Recent:       "2h0m0s",
-		ScoreStart:   jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
+		ScoreStart:   jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
 	}
 
 	PtrTestJT = ptrTestJT{
 		LookbackDays: 7,
-		HalfLife:     common.PtrTo(jsontime.Duration(30 * time.Minute)),
+		HalfLife:     common.PtrTo(jsontypes.Duration(30 * time.Minute)),
 		Recent:       common.PtrTo("2h0m0s"),
-		ScoreStart:   common.PtrTo(jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
+		ScoreStart:   common.PtrTo(jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
 	}
 
 	UrlEncTestJTLocal = urlEncTestJT{
 		LookbackDays: 7,
-		HalfLife:     jsontime.Duration(30 * time.Minute),
+		HalfLife:     jsontypes.Duration(30 * time.Minute),
 		Recent:       "2h0m0s",
-		ScoreStart:   jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
+		ScoreStart:   jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
 	}
 
 	realSim = &alerting.Simulation{
 		Alerting: config.Alerting{
 			LookbackDays: 7,
-			HalfLife:     jsontime.Duration(30 * time.Minute),
+			HalfLife:     jsontypes.Duration(30 * time.Minute),
-			Recent:       jsontime.Duration(2 * time.Hour),
+			Recent:       jsontypes.Duration(2 * time.Hour),
 		},
-		SimInterval: jsontime.Duration(5 * time.Minute),
+		SimInterval: jsontypes.Duration(5 * time.Minute),
-		ScoreStart:  jsontime.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
+		ScoreStart:  jsontypes.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
 	}
 
 	Call1 = callUploadRequest{

internal/jsontypes/jsontime.go

@@ -1,4 +1,4 @@
-package jsontime
+package jsontypes
 
 import (
 	"encoding/json"

internal/jsontypes/metadata.go

@@ -0,0 +1,3 @@
+package jsontypes
+
+type Metadata map[string]interface{}

pkg/alerting/alert/alert.go

@@ -37,7 +37,7 @@ func (a *Alert) ToAddAlertParams() database.AddAlertParams {
 		ID:        a.ID,
 		Time:      pgtype.Timestamptz{Time: a.Timestamp, Valid: true},
 		SystemID:  int(a.Score.ID.System),
-		Tgid:      int(a.Score.ID.Talkgroup),
+		TGID:      int(a.Score.ID.Talkgroup),
 		Weight:    &a.Weight,
 		Score:     &f32score,
 		OrigScore: origScore,

pkg/alerting/alerting.go

@@ -228,11 +228,11 @@ func (as *alerter) scoredTGs() []talkgroups.ID {
 	return tgs
 }
 
-// packedScoredTGs gets a list of packed TGIDs.
+// packedScoredTGs gets a list of TGID tuples.
-func (as *alerter) packedScoredTGs() []int64 {
+func (as *alerter) scoredTGsTuple() (tgs database.TGTuples) {
-	tgs := make([]int64, 0, len(as.scores))
+	tgs = database.MakeTGTuples(len(as.scores))
 	for _, s := range as.scores {
-		tgs = append(tgs, s.ID.Pack())
+		tgs.Append(s.ID.System, s.ID.Talkgroup)
 	}
 
 	return tgs
@@ -312,7 +312,7 @@ func (as *alerter) backfill(ctx context.Context, since time.Time, until time.Tim
 	db := database.FromCtx(ctx)
 	const backfillStatsQuery = `SELECT system, talkgroup, call_date FROM calls WHERE call_date > $1 AND call_date < $2 ORDER BY call_date ASC`
 
-	rows, err := db.Query(ctx, backfillStatsQuery, since, until)
+	rows, err := db.DB().Query(ctx, backfillStatsQuery, since, until)
 	if err != nil {
 		return count, err
 	}

pkg/alerting/simulate.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"dynatron.me/x/stillbox/internal/forms"
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/internal/trending"
 	"dynatron.me/x/stillbox/pkg/config"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
@@ -23,12 +23,12 @@ type Simulation struct {
 	config.Alerting
 
 	// ScoreStart is the time when scoring begins
-	ScoreStart jsontime.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
+	ScoreStart jsontypes.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
 	// ScoreEnd is the time when the score simulator ends. Left blank, it defaults to time.Now()
-	ScoreEnd jsontime.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
+	ScoreEnd jsontypes.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
 
 	// SimInterval is the interval at which the scorer will be called
-	SimInterval jsontime.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
+	SimInterval jsontypes.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
 
 	clock    offsetClock `json:"-"`
 	*alerter `json:"-"`
@@ -64,7 +64,7 @@ func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.I
 	s.Enable = true
 	s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter)
 	if time.Time(s.ScoreEnd).IsZero() {
-		s.ScoreEnd = jsontime.Time(now)
+		s.ScoreEnd = jsontypes.Time(now)
 	}
 	log.Debug().Time("scoreStart", s.ScoreStart.Time()).
 		Time("scoreEnd", s.ScoreEnd.Time()).

pkg/alerting/stats.go

@@ -40,20 +40,20 @@ func (as *alerter) tgStatsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	db := database.FromCtx(ctx)
 
-	tgs, err := db.GetTalkgroupsWithLearnedByPackedIDs(ctx, as.packedScoredTGs())
+	tgs, err := db.GetTalkgroupsWithLearnedBySysTGID(ctx, as.scoredTGsTuple())
 	if err != nil {
 		log.Error().Err(err).Msg("stats TG get failed")
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	tgMap := make(map[talkgroups.ID]database.GetTalkgroupsWithLearnedByPackedIDsRow, len(tgs))
+	tgMap := make(map[talkgroups.ID]database.GetTalkgroupsRow, len(tgs))
 	for _, t := range tgs {
-		tgMap[talkgroups.ID{System: uint32(t.System.ID), Talkgroup: uint32(t.Talkgroup.Tgid)}] = t
+		tgMap[talkgroups.ID{System: uint32(t.System.ID), Talkgroup: uint32(t.Talkgroup.TGID)}] = t
 	}
 
 	renderData := struct {
-		TGs        map[talkgroups.ID]database.GetTalkgroupsWithLearnedByPackedIDsRow
+		TGs        map[talkgroups.ID]database.GetTalkgroupsRow
 		Scores     trending.Scores[talkgroups.ID]
 		LastScore  time.Time
 		Simulation *Simulation

pkg/auth/jwt.go

@@ -70,7 +70,7 @@ func (a *Auth) initJWT() {
 }
 
 func (a *Auth) Login(ctx context.Context, username, password string) (token string, err error) {
-	q := database.New(database.FromCtx(ctx))
+	q := database.FromCtx(ctx)
 	users, err := q.GetUsers(ctx)
 	if err != nil {
 		log.Error().Err(err).Msg("getUsers failed")

pkg/calls/filter.go

@@ -71,7 +71,7 @@ func (f *TalkgroupFilter) compile(ctx context.Context) error {
 		}
 
 		for _, tg := range tagTGs {
-			f.talkgroups[tgs.ID{System: uint32(tg.SystemID), Talkgroup: uint32(tg.Tgid)}] = true
+			f.talkgroups[tgs.ID{System: uint32(tg.SystemID), Talkgroup: uint32(tg.TGID)}] = true
 		}
 	}
 

pkg/config/config.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -37,7 +37,8 @@ type CORS struct {
 }
 
 type DB struct {
-	Connect string `yaml:"connect"`
+	Connect    string `yaml:"connect"`
+	LogQueries bool   `yaml:"logQueries"`
 }
 
 type Logger struct {
@@ -54,12 +55,12 @@ type RateLimit struct {
 }
 
 type Alerting struct {
-	Enable         bool               `yaml:"enable" form:"enable"`
+	Enable         bool                `yaml:"enable" form:"enable"`
-	LookbackDays   uint               `yaml:"lookbackDays" form:"lookbackDays"`
+	LookbackDays   uint                `yaml:"lookbackDays" form:"lookbackDays"`
-	HalfLife       jsontime.Duration  `yaml:"halfLife" form:"halfLife"`
+	HalfLife       jsontypes.Duration  `yaml:"halfLife" form:"halfLife"`
-	Recent         jsontime.Duration  `yaml:"recent" form:"recent"`
+	Recent         jsontypes.Duration  `yaml:"recent" form:"recent"`
-	AlertThreshold float64            `yaml:"alertThreshold" form:"alertThreshold"`
+	AlertThreshold float64             `yaml:"alertThreshold" form:"alertThreshold"`
-	Renotify       *jsontime.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
+	Renotify       *jsontypes.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
 }
 
 type Notify []NotifyService

pkg/database/calls.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 // source: calls.sql
 
 package database
@@ -31,7 +31,7 @@ VALUES
 type AddAlertParams struct {
 	ID        uuid.UUID          `json:"id"`
 	Time      pgtype.Timestamptz `json:"time"`
-	Tgid      int                `json:"tgid"`
+	TGID      int                `json:"tgid"`
 	SystemID  int                `json:"system_id"`
 	Weight    *float32           `json:"weight"`
 	Score     *float32           `json:"score"`
@@ -44,7 +44,7 @@ func (q *Queries) AddAlert(ctx context.Context, arg AddAlertParams) error {
 	_, err := q.db.Exec(ctx, addAlert,
 		arg.ID,
 		arg.Time,
-		arg.Tgid,
+		arg.TGID,
 		arg.SystemID,
 		arg.Weight,
 		arg.Score,

pkg/database/database.go

@@ -11,16 +11,37 @@ import (
 	_ "github.com/golang-migrate/migrate/v4/database/pgx/v5"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/jackc/pgx/v5/tracelog"
+	"github.com/rs/zerolog/log"
 )
 
 // DB is a database handle.
-type DB struct {
+
+//go:generate mockery
+type DB interface {
+	Querier
+	talkgroupQuerier
+
+	DB() *Database
+}
+
+type Database struct {
 	*pgxpool.Pool
 	*Queries
 }
 
+func (db *Database) DB() *Database {
+	return db
+}
+
+type dbLogger struct{}
+
+func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
+	log.Debug().Fields(data).Msg(msg)
+}
+
 // NewClient creates a new DB using the provided config.
-func NewClient(ctx context.Context, conf config.DB) (*DB, error) {
+func NewClient(ctx context.Context, conf config.DB) (DB, error) {
 	dir, err := iofs.New(sqlembed.Migrations, "postgres/migrations")
 	if err != nil {
 		return nil, err
@@ -43,12 +64,19 @@ func NewClient(ctx context.Context, conf config.DB) (*DB, error) {
 		return nil, err
 	}
 
+	if conf.LogQueries {
+		pgConf.ConnConfig.Tracer = &tracelog.TraceLog{
+			Logger:   dbLogger{},
+			LogLevel: tracelog.LogLevelTrace,
+		}
+	}
+
 	pool, err := pgxpool.NewWithConfig(ctx, pgConf)
 	if err != nil {
 		return nil, err
 	}
 
-	db := &DB{
+	db := &Database{
 		Pool:    pool,
 		Queries: New(pool),
 	}
@@ -61,8 +89,8 @@ type dBCtxKey string
 const DBCtxKey dBCtxKey = "dbctx"
 
 // FromCtx returns the database handle from the provided Context.
-func FromCtx(ctx context.Context) *DB {
+func FromCtx(ctx context.Context) DB {
-	c, ok := ctx.Value(DBCtxKey).(*DB)
+	c, ok := ctx.Value(DBCtxKey).(DB)
 	if !ok {
 		panic("no DB in context")
 	}
@@ -71,7 +99,7 @@ func FromCtx(ctx context.Context) *DB {
 }
 
 // CtxWithDB returns a Context with the provided database handle.
-func CtxWithDB(ctx context.Context, conn *DB) context.Context {
+func CtxWithDB(ctx context.Context, conn DB) context.Context {
 	return context.WithValue(ctx, DBCtxKey, conn)
 }
 

pkg/database/db.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 
 package database
 

pkg/database/extend.go

@@ -1,14 +1,17 @@
 package database
 
-func (d GetTalkgroupsWithLearnedByPackedIDsRow) GetTalkgroup() Talkgroup { return d.Talkgroup }
+func (d GetTalkgroupsRow) GetTalkgroup() Talkgroup                    { return d.Talkgroup }
-func (d GetTalkgroupsWithLearnedByPackedIDsRow) GetSystem() System       { return d.System }
+func (d GetTalkgroupsRow) GetSystem() System                          { return d.System }
-func (d GetTalkgroupsWithLearnedByPackedIDsRow) GetLearned() bool        { return d.Learned }
+func (d GetTalkgroupsRow) GetLearned() bool                           { return d.Learned }
-func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup            { return g.Talkgroup }
+func (g GetTalkgroupWithLearnedRow) GetTalkgroup() Talkgroup          { return g.Talkgroup }
-func (g GetTalkgroupsWithLearnedRow) GetSystem() System                  { return g.System }
+func (g GetTalkgroupWithLearnedRow) GetSystem() System                { return g.System }
-func (g GetTalkgroupsWithLearnedRow) GetLearned() bool                   { return g.Learned }
+func (g GetTalkgroupWithLearnedRow) GetLearned() bool                 { return g.Learned }
-func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup    { return g.Talkgroup }
+func (g GetTalkgroupsWithLearnedRow) GetTalkgroup() Talkgroup         { return g.Talkgroup }
-func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System          { return g.System }
+func (g GetTalkgroupsWithLearnedRow) GetSystem() System               { return g.System }
-func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool           { return g.Learned }
+func (g GetTalkgroupsWithLearnedRow) GetLearned() bool                { return g.Learned }
-func (g Talkgroup) GetTalkgroup() Talkgroup                              { return g }
+func (g GetTalkgroupsWithLearnedBySystemRow) GetTalkgroup() Talkgroup { return g.Talkgroup }
-func (g Talkgroup) GetSystem() System                                    { return System{ID: int(g.SystemID)} }
+func (g GetTalkgroupsWithLearnedBySystemRow) GetSystem() System       { return g.System }
-func (g Talkgroup) GetLearned() bool                                     { return false }
+func (g GetTalkgroupsWithLearnedBySystemRow) GetLearned() bool        { return g.Learned }
+func (g Talkgroup) GetTalkgroup() Talkgroup                           { return g }
+func (g Talkgroup) GetSystem() System                                 { return System{ID: int(g.SystemID)} }
+func (g Talkgroup) GetLearned() bool                                  { return false }

pkg/database/models.go

@@ -1,12 +1,13 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 
 package database
 
 import (
 	"time"
 
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/alerting/rules"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
@@ -15,7 +16,7 @@ import (
 type Alert struct {
 	ID        uuid.UUID          `json:"id"`
 	Time      pgtype.Timestamptz `json:"time"`
-	Tgid      int                `json:"tgid"`
+	TGID      int                `json:"tgid"`
 	SystemID  int                `json:"system_id"`
 	Weight    *float32           `json:"weight"`
 	Score     *float32           `json:"score"`
@@ -82,24 +83,24 @@ type System struct {
 }
 
 type Talkgroup struct {
-	ID          int64            `json:"id"`
+	ID          uuid.UUID          `json:"id"`
-	SystemID    int32            `json:"system_id"`
+	SystemID    int32              `json:"system_id"`
-	Tgid        int32            `json:"tgid"`
+	TGID        int32              `json:"tgid"`
-	Name        *string          `json:"name"`
+	Name        *string            `json:"name"`
-	AlphaTag    *string          `json:"alpha_tag"`
+	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string          `json:"tg_group"`
+	TgGroup     *string            `json:"tg_group"`
-	Frequency   *int32           `json:"frequency"`
+	Frequency   *int32             `json:"frequency"`
-	Metadata    []byte           `json:"metadata"`
+	Metadata    jsontypes.Metadata `json:"metadata"`
-	Tags        []string         `json:"tags"`
+	Tags        []string           `json:"tags"`
-	Alert       bool             `json:"alert"`
+	Alert       bool               `json:"alert"`
-	AlertConfig rules.AlertRules `json:"alert_config"`
+	AlertConfig rules.AlertRules   `json:"alert_config"`
-	Weight      float32          `json:"weight"`
+	Weight      float32            `json:"weight"`
 }
 
 type TalkgroupsLearned struct {
 	ID       int32   `json:"id"`
 	SystemID int     `json:"system_id"`
-	Tgid     int     `json:"tgid"`
+	TGID     int     `json:"tgid"`
 	Name     string  `json:"name"`
 	AlphaTag *string `json:"alpha_tag"`
 	Ignored  *bool   `json:"ignored"`

pkg/database/querier.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 
 package database
 
@@ -14,7 +14,6 @@ import (
 type Querier interface {
 	AddAlert(ctx context.Context, arg AddAlertParams) error
 	AddCall(ctx context.Context, arg AddCallParams) error
-	BulkSetTalkgroupTags(ctx context.Context, iD int64, tags []string) error
 	CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error)
 	CreateUser(ctx context.Context, arg CreateUserParams) (User, error)
 	DeleteAPIKey(ctx context.Context, apiKey string) error
@@ -22,22 +21,20 @@ type Querier interface {
 	GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error)
 	GetDatabaseSize(ctx context.Context) (string, error)
 	GetSystemName(ctx context.Context, systemID int) (string, error)
-	GetTalkgroup(ctx context.Context, systemID int, tgid int) (GetTalkgroupRow, error)
+	GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error)
-	GetTalkgroupIDsByTags(ctx context.Context, anytags []string, alltags []string, nottags []string) ([]GetTalkgroupIDsByTagsRow, error)
+	GetTalkgroupIDsByTags(ctx context.Context, anyTags []string, allTags []string, notTags []string) ([]GetTalkgroupIDsByTagsRow, error)
-	GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error)
+	GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error)
-	GetTalkgroupWithLearned(ctx context.Context, systemID int, tgid int) (GetTalkgroupWithLearnedRow, error)
+	GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error)
-	GetTalkgroupsByPackedIDs(ctx context.Context, dollar_1 []int64) ([]GetTalkgroupsByPackedIDsRow, error)
 	GetTalkgroupsWithAllTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAllTagsRow, error)
 	GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) ([]GetTalkgroupsWithAnyTagsRow, error)
 	GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroupsWithLearnedRow, error)
-	GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, dollar_1 []int64) ([]GetTalkgroupsWithLearnedByPackedIDsRow, error)
 	GetTalkgroupsWithLearnedBySystem(ctx context.Context, system int32) ([]GetTalkgroupsWithLearnedBySystemRow, error)
 	GetUserByID(ctx context.Context, id int32) (User, error)
 	GetUserByUID(ctx context.Context, id int32) (User, error)
 	GetUserByUsername(ctx context.Context, username string) (User, error)
 	GetUsers(ctx context.Context) ([]User, error)
 	SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error
-	SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error
+	SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error
 	UpdatePassword(ctx context.Context, username string, password string) error
 	UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error)
 }

pkg/database/talkgroups.go

@@ -0,0 +1,132 @@
+package database
+
+import (
+	"context"
+)
+
+type talkgroupQuerier interface {
+	GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error)
+	GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error)
+	BulkSetTalkgroupTags(ctx context.Context, tgs TGTuples, tags []string) error
+}
+
+type TGTuples [2][]uint32
+
+func MakeTGTuples(cap int) TGTuples {
+	return [2][]uint32{
+		make([]uint32, 0, cap),
+		make([]uint32, 0, cap),
+	}
+}
+
+func (t *TGTuples) Append(sys, tg uint32) {
+	t[0] = append(t[0], sys)
+	t[1] = append(t[1], tg)
+}
+
+// Below queries are here because sqlc refuses to parse unnest(x, y)
+
+const getTalkgroupsWithLearnedBySysTGID = `SELECT
+tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
+FALSE learned
+FROM talkgroups tg
+JOIN systems sys ON tg.system_id = sys.id
+JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg)
+UNION
+SELECT
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
+CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
+TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
+TRUE learned
+FROM talkgroups_learned tgl
+JOIN systems sys ON tgl.system_id = sys.id
+JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tgl.system_id = tgt.sys AND tgl.tgid = tgt.tg);`
+
+type GetTalkgroupsRow struct {
+	Talkgroup Talkgroup `json:"talkgroup"`
+	System    System    `json:"system"`
+	Learned   bool      `json:"learned"`
+}
+
+func (q *Queries) GetTalkgroupsWithLearnedBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
+	rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedBySysTGID, ids[0], ids[1])
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetTalkgroupsRow
+	for rows.Next() {
+		var i GetTalkgroupsRow
+		if err := rows.Scan(
+			&i.Talkgroup.ID,
+			&i.Talkgroup.SystemID,
+			&i.Talkgroup.TGID,
+			&i.Talkgroup.Name,
+			&i.Talkgroup.AlphaTag,
+			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.Frequency,
+			&i.Talkgroup.Metadata,
+			&i.Talkgroup.Tags,
+			&i.Talkgroup.Alert,
+			&i.Talkgroup.AlertConfig,
+			&i.Talkgroup.Weight,
+			&i.System.ID,
+			&i.System.Name,
+			&i.Learned,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const getTalkgroupsBySysTGID = `SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
+JOIN systems sys ON tg.system_id = sys.id
+JOIN UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) ON (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
+
+func (q *Queries) GetTalkgroupsBySysTGID(ctx context.Context, ids TGTuples) ([]GetTalkgroupsRow, error) {
+	rows, err := q.db.Query(ctx, getTalkgroupsBySysTGID, ids[0], ids[1])
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetTalkgroupsRow
+	for rows.Next() {
+		var i GetTalkgroupsRow
+		if err := rows.Scan(
+			&i.Talkgroup.ID,
+			&i.Talkgroup.SystemID,
+			&i.Talkgroup.TGID,
+			&i.Talkgroup.Name,
+			&i.Talkgroup.AlphaTag,
+			&i.Talkgroup.TgGroup,
+			&i.Talkgroup.Frequency,
+			&i.Talkgroup.Metadata,
+			&i.Talkgroup.Tags,
+			&i.Talkgroup.Alert,
+			&i.Talkgroup.AlertConfig,
+			&i.Talkgroup.Weight,
+			&i.System.ID,
+			&i.System.Name,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const bulkSetTalkgroupTags = `UPDATE talkgroups tg SET tags = $3 FROM UNNEST($1::INT4[], $2::INT4[]) AS tgt(sys, tg) WHERE (tg.system_id = tgt.sys AND tg.tgid = tgt.tg);`
+
+func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, tgs TGTuples, tags []string) error {
+	_, err := q.db.Exec(ctx, bulkSetTalkgroupTags, tgs[0], tgs[1], tags)
+	return err
+}

pkg/database/talkgroups.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 // source: talkgroups.sql
 
 package database
@@ -8,19 +8,11 @@ package database
 import (
 	"context"
 
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/alerting/rules"
+	"github.com/jackc/pgx/v5/pgtype"
 )
 
-const bulkSetTalkgroupTags = `-- name: BulkSetTalkgroupTags :exec
-UPDATE talkgroups SET tags = $2
-WHERE id = ANY($1)
-`
-
-func (q *Queries) BulkSetTalkgroupTags(ctx context.Context, iD int64, tags []string) error {
-	_, err := q.db.Exec(ctx, bulkSetTalkgroupTags, iD, tags)
-	return err
-}
-
 const getSystemName = `-- name: GetSystemName :one
 SELECT name FROM systems WHERE id = $1
 `
@@ -34,20 +26,20 @@ func (q *Queries) GetSystemName(ctx context.Context, systemID int) (string, erro
 
 const getTalkgroup = `-- name: GetTalkgroup :one
 SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
-WHERE id = systg2id($1, $2)
+WHERE (system_id, tgid) = ($1, $2)
 `
 
 type GetTalkgroupRow struct {
 	Talkgroup Talkgroup `json:"talkgroup"`
 }
 
-func (q *Queries) GetTalkgroup(ctx context.Context, systemID int, tgid int) (GetTalkgroupRow, error) {
+func (q *Queries) GetTalkgroup(ctx context.Context, systemID int32, tgID int32) (GetTalkgroupRow, error) {
-	row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgid)
+	row := q.db.QueryRow(ctx, getTalkgroup, systemID, tgID)
 	var i GetTalkgroupRow
 	err := row.Scan(
 		&i.Talkgroup.ID,
 		&i.Talkgroup.SystemID,
-		&i.Talkgroup.Tgid,
+		&i.Talkgroup.TGID,
 		&i.Talkgroup.Name,
 		&i.Talkgroup.AlphaTag,
 		&i.Talkgroup.TgGroup,
@@ -70,11 +62,11 @@ AND NOT (tags @> ARRAY[$3])
 
 type GetTalkgroupIDsByTagsRow struct {
 	SystemID int32 `json:"system_id"`
-	Tgid     int32 `json:"tgid"`
+	TGID     int32 `json:"tgid"`
 }
 
-func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, alltags []string, nottags []string) ([]GetTalkgroupIDsByTagsRow, error) {
+func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anyTags []string, allTags []string, notTags []string) ([]GetTalkgroupIDsByTagsRow, error) {
-	rows, err := q.db.Query(ctx, getTalkgroupIDsByTags, anytags, alltags, nottags)
+	rows, err := q.db.Query(ctx, getTalkgroupIDsByTags, anyTags, allTags, notTags)
 	if err != nil {
 		return nil, err
 	}
@@ -82,7 +74,7 @@ func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, a
 	var items []GetTalkgroupIDsByTagsRow
 	for rows.Next() {
 		var i GetTalkgroupIDsByTagsRow
-		if err := rows.Scan(&i.SystemID, &i.Tgid); err != nil {
+		if err := rows.Scan(&i.SystemID, &i.TGID); err != nil {
 			return nil, err
 		}
 		items = append(items, i)
@@ -95,11 +87,11 @@ func (q *Queries) GetTalkgroupIDsByTags(ctx context.Context, anytags []string, a
 
 const getTalkgroupTags = `-- name: GetTalkgroupTags :one
 SELECT tags FROM talkgroups
-WHERE id = systg2id($1, $2)
+WHERE system_id = $1 AND tgid = $2
 `
 
-func (q *Queries) GetTalkgroupTags(ctx context.Context, sys int, tg int) ([]string, error) {
+func (q *Queries) GetTalkgroupTags(ctx context.Context, systemID int32, tgID int32) ([]string, error) {
-	row := q.db.QueryRow(ctx, getTalkgroupTags, sys, tg)
+	row := q.db.QueryRow(ctx, getTalkgroupTags, systemID, tgID)
 	var tags []string
 	err := row.Scan(&tags)
 	return tags, err
@@ -111,10 +103,10 @@ tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency,
 FALSE learned
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = systg2id($1, $2)
+WHERE (tg.system_id, tg.tgid) = ($1, $2)
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -130,13 +122,13 @@ type GetTalkgroupWithLearnedRow struct {
 	Learned   bool      `json:"learned"`
 }
 
-func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int, tgid int) (GetTalkgroupWithLearnedRow, error) {
+func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupWithLearnedRow, error) {
-	row := q.db.QueryRow(ctx, getTalkgroupWithLearned, systemID, tgid)
+	row := q.db.QueryRow(ctx, getTalkgroupWithLearned, systemID, tGID)
 	var i GetTalkgroupWithLearnedRow
 	err := row.Scan(
 		&i.Talkgroup.ID,
 		&i.Talkgroup.SystemID,
-		&i.Talkgroup.Tgid,
+		&i.Talkgroup.TGID,
 		&i.Talkgroup.Name,
 		&i.Talkgroup.AlphaTag,
 		&i.Talkgroup.TgGroup,
@@ -153,52 +145,6 @@ func (q *Queries) GetTalkgroupWithLearned(ctx context.Context, systemID int, tgi
 	return i, err
 }
 
-const getTalkgroupsByPackedIDs = `-- name: GetTalkgroupsByPackedIDs :many
-SELECT tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = ANY($1::INT8[])
-`
-
-type GetTalkgroupsByPackedIDsRow struct {
-	Talkgroup Talkgroup `json:"talkgroup"`
-	System    System    `json:"system"`
-}
-
-func (q *Queries) GetTalkgroupsByPackedIDs(ctx context.Context, dollar_1 []int64) ([]GetTalkgroupsByPackedIDsRow, error) {
-	rows, err := q.db.Query(ctx, getTalkgroupsByPackedIDs, dollar_1)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []GetTalkgroupsByPackedIDsRow
-	for rows.Next() {
-		var i GetTalkgroupsByPackedIDsRow
-		if err := rows.Scan(
-			&i.Talkgroup.ID,
-			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
-			&i.Talkgroup.Name,
-			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
-			&i.Talkgroup.Frequency,
-			&i.Talkgroup.Metadata,
-			&i.Talkgroup.Tags,
-			&i.Talkgroup.Alert,
-			&i.Talkgroup.AlertConfig,
-			&i.Talkgroup.Weight,
-			&i.System.ID,
-			&i.System.Name,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const getTalkgroupsWithAllTags = `-- name: GetTalkgroupsWithAllTags :many
 SELECT talkgroups.id, talkgroups.system_id, talkgroups.tgid, talkgroups.name, talkgroups.alpha_tag, talkgroups.tg_group, talkgroups.frequency, talkgroups.metadata, talkgroups.tags, talkgroups.alert, talkgroups.alert_config, talkgroups.weight FROM talkgroups
 WHERE tags && ARRAY[$1]
@@ -220,7 +166,7 @@ func (q *Queries) GetTalkgroupsWithAllTags(ctx context.Context, tags []string) (
 		if err := rows.Scan(
 			&i.Talkgroup.ID,
 			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
+			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
 			&i.Talkgroup.TgGroup,
@@ -262,7 +208,7 @@ func (q *Queries) GetTalkgroupsWithAnyTags(ctx context.Context, tags []string) (
 		if err := rows.Scan(
 			&i.Talkgroup.ID,
 			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
+			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
 			&i.Talkgroup.TgGroup,
@@ -291,7 +237,7 @@ FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -319,68 +265,7 @@ func (q *Queries) GetTalkgroupsWithLearned(ctx context.Context) ([]GetTalkgroups
 		if err := rows.Scan(
 			&i.Talkgroup.ID,
 			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
+			&i.Talkgroup.TGID,
-			&i.Talkgroup.Name,
-			&i.Talkgroup.AlphaTag,
-			&i.Talkgroup.TgGroup,
-			&i.Talkgroup.Frequency,
-			&i.Talkgroup.Metadata,
-			&i.Talkgroup.Tags,
-			&i.Talkgroup.Alert,
-			&i.Talkgroup.AlertConfig,
-			&i.Talkgroup.Weight,
-			&i.System.ID,
-			&i.System.Name,
-			&i.Learned,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
-const getTalkgroupsWithLearnedByPackedIDs = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many
-SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = ANY($1::INT8[])
-UNION
-SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE systg2id(tgl.system_id, tgl.tgid) = ANY($1::INT8[]) AND ignored IS NOT TRUE
-`
-
-type GetTalkgroupsWithLearnedByPackedIDsRow struct {
-	Talkgroup Talkgroup `json:"talkgroup"`
-	System    System    `json:"system"`
-	Learned   bool      `json:"learned"`
-}
-
-func (q *Queries) GetTalkgroupsWithLearnedByPackedIDs(ctx context.Context, dollar_1 []int64) ([]GetTalkgroupsWithLearnedByPackedIDsRow, error) {
-	rows, err := q.db.Query(ctx, getTalkgroupsWithLearnedByPackedIDs, dollar_1)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []GetTalkgroupsWithLearnedByPackedIDsRow
-	for rows.Next() {
-		var i GetTalkgroupsWithLearnedByPackedIDsRow
-		if err := rows.Scan(
-			&i.Talkgroup.ID,
-			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
 			&i.Talkgroup.TgGroup,
@@ -413,7 +298,7 @@ JOIN systems sys ON tg.system_id = sys.id
 WHERE tg.system_id = $1
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -441,7 +326,7 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
 		if err := rows.Scan(
 			&i.Talkgroup.ID,
 			&i.Talkgroup.SystemID,
-			&i.Talkgroup.Tgid,
+			&i.Talkgroup.TGID,
 			&i.Talkgroup.Name,
 			&i.Talkgroup.AlphaTag,
 			&i.Talkgroup.TgGroup,
@@ -466,12 +351,12 @@ func (q *Queries) GetTalkgroupsWithLearnedBySystem(ctx context.Context, system i
 }
 
 const setTalkgroupTags = `-- name: SetTalkgroupTags :exec
-UPDATE talkgroups SET tags = $3
+UPDATE talkgroups SET tags = $1
-WHERE id = systg2id($1, $2)
+WHERE system_id = $2 AND tgid = $3
 `
 
-func (q *Queries) SetTalkgroupTags(ctx context.Context, sys int, tg int, tags []string) error {
+func (q *Queries) SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tgID int32) error {
-	_, err := q.db.Exec(ctx, setTalkgroupTags, sys, tg, tags)
+	_, err := q.db.Exec(ctx, setTalkgroupTags, tags, systemID, tgID)
 	return err
 }
 
@@ -487,21 +372,23 @@ SET
 	alert = COALESCE($7, alert),
 	alert_config = COALESCE($8, alert_config),
 	weight = COALESCE($9, weight)
-WHERE id = $10
+WHERE id = $10 OR (system_id = $11 AND tgid = $12)
 RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, tags, alert, alert_config, weight
 `
 
 type UpdateTalkgroupParams struct {
-	Name        *string          `json:"name"`
+	Name        *string            `json:"name"`
-	AlphaTag    *string          `json:"alpha_tag"`
+	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string          `json:"tg_group"`
+	TgGroup     *string            `json:"tg_group"`
-	Frequency   *int32           `json:"frequency"`
+	Frequency   *int32             `json:"frequency"`
-	Metadata    []byte           `json:"metadata"`
+	Metadata    jsontypes.Metadata `json:"metadata"`
-	Tags        []string         `json:"tags"`
+	Tags        []string           `json:"tags"`
-	Alert       *bool            `json:"alert"`
+	Alert       *bool              `json:"alert"`
-	AlertConfig rules.AlertRules `json:"alert_config"`
+	AlertConfig rules.AlertRules   `json:"alert_config"`
-	Weight      *float32         `json:"weight"`
+	Weight      *float32           `json:"weight"`
-	ID          int64            `json:"id"`
+	ID          pgtype.UUID        `json:"id"`
+	SystemID    *int32             `json:"system_id"`
+	TGID        *int32             `json:"tgid"`
 }
 
 func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) {
@@ -516,12 +403,14 @@ func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams
 		arg.AlertConfig,
 		arg.Weight,
 		arg.ID,
+		arg.SystemID,
+		arg.TGID,
 	)
 	var i Talkgroup
 	err := row.Scan(
 		&i.ID,
 		&i.SystemID,
-		&i.Tgid,
+		&i.TGID,
 		&i.Name,
 		&i.AlphaTag,
 		&i.TgGroup,

pkg/database/talkgroups.sql_test.go

@@ -6,34 +6,16 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const getTalkgroupsWithLearnedByPackedIDsTest = `-- name: GetTalkgroupsWithLearnedByPackedIDs :many
-SELECT
-tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
-FALSE learned
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = ANY($1::INT8[])
-UNION
-SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE systg2id(tgl.system_id, tgl.tgid) = ANY($1::INT8[]) AND ignored IS NOT TRUE
-`
 const getTalkgroupWithLearnedTest = `-- name: GetTalkgroupWithLearned :one
 SELECT
 tg.id, tg.system_id, tg.tgid, tg.name, tg.alpha_tag, tg.tg_group, tg.frequency, tg.metadata, tg.tags, tg.alert, tg.alert_config, tg.weight, sys.id, sys.name,
 FALSE learned
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = systg2id($1, $2)
+WHERE (tg.system_id, tg.tgid) = ($1, $2)
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -52,7 +34,7 @@ JOIN systems sys ON tg.system_id = sys.id
 WHERE tg.system_id = $1
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -70,7 +52,7 @@ FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -81,7 +63,6 @@ WHERE ignored IS NOT TRUE
 `
 
 func TestQueryColumnsMatch(t *testing.T) {
-	require.Equal(t, getTalkgroupsWithLearnedByPackedIDsTest, getTalkgroupsWithLearnedByPackedIDs)
 	require.Equal(t, getTalkgroupWithLearnedTest, getTalkgroupWithLearned)
 	require.Equal(t, getTalkgroupsWithLearnedBySystemTest, getTalkgroupsWithLearnedBySystem)
 	require.Equal(t, getTalkgroupsWithLearnedTest, getTalkgroupsWithLearned)

pkg/database/users.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.27.0
 // source: users.sql
 
 package database

pkg/nexus/commands.go

@@ -2,7 +2,6 @@ package nexus
 
 import (
 	"context"
-	"encoding/json"
 
 	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/pb"
@@ -70,12 +69,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
 
 	var md *structpb.Struct
 	if len(tgi.Talkgroup.Metadata) > 0 {
-		m := make(map[string]interface{})
+		md, err = structpb.NewStruct(tgi.Talkgroup.Metadata)
-		err := json.Unmarshal(tgi.Talkgroup.Metadata, &m)
-		if err != nil {
-			log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("unmarshal tg metadata")
-		}
-		md, err = structpb.NewStruct(m)
 		if err != nil {
 			log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("new pb struct for tg metadata")
 		}

pkg/rest/api.go

@@ -79,8 +79,9 @@ func internalError(err error) render.Renderer {
 type errResponder func(error) render.Renderer
 
 var statusMapping = map[error]errResponder{
-	talkgroups.ErrNotFound: recordNotFound,
+	talkgroups.ErrNoSuchSystem: recordNotFound,
-	pgx.ErrNoRows:          recordNotFound,
+	talkgroups.ErrNotFound:     recordNotFound,
+	pgx.ErrNoRows:              recordNotFound,
 }
 
 func autoError(err error) render.Renderer {

pkg/rest/talkgroups.go

@@ -6,6 +6,7 @@ import (
 	"dynatron.me/x/stillbox/internal/forms"
 	"dynatron.me/x/stillbox/pkg/database"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/importer"
 
 	"github.com/go-chi/chi/v5"
 )
@@ -20,6 +21,7 @@ func (tga *talkgroupAPI) Subrouter() http.Handler {
 	r.Put("/{system:\\d+}/{id:\\d+}", tga.put)
 	r.Get("/{system:\\d+}/", tga.get)
 	r.Get("/", tga.get)
+	r.Post("/import", tga.tgImport)
 
 	return r
 }
@@ -96,7 +98,6 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 		wErr(w, r, badRequest(err))
 		return
 	}
-	input.ID = id.ToID().Pack()
 
 	record, err := tgs.UpdateTG(ctx, input)
 	if err != nil {
@@ -106,3 +107,19 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 
 	respond(w, r, record)
 }
+
+func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) {
+	var impJob importer.ImportJob
+	err := forms.Unmarshal(r, &impJob, forms.WithTag("json"), forms.WithAcceptBlank(), forms.WithOmitEmpty())
+	if err != nil {
+		wErr(w, r, badRequest(err))
+		return
+	}
+	recs, err := impJob.Import(r.Context())
+	if err != nil {
+		wErr(w, r, autoError(err))
+		return
+	}
+
+	respond(w, r, recs)
+}

pkg/server/server.go

@@ -27,7 +27,7 @@ const shutdownTimeout = 5 * time.Second
 type Server struct {
 	auth     *auth.Auth
 	conf     *config.Config
-	db       *database.DB
+	db       database.DB
 	r        *chi.Mux
 	sources  sources.Sources
 	sinks    sinks.Sinks
@@ -103,7 +103,7 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 }
 
 func (s *Server) Go(ctx context.Context) error {
-	defer s.db.Close()
+	defer s.db.DB().Close()
 
 	s.installHupHandler()
 

pkg/sinks/database.go

@@ -13,10 +13,10 @@ import (
 )
 
 type DatabaseSink struct {
-	db *database.DB
+	db database.DB
 }
 
-func NewDatabaseSink(db *database.DB) *DatabaseSink {
+func NewDatabaseSink(db database.DB) *DatabaseSink {
 	return &DatabaseSink{db: db}
 }
 

pkg/talkgroups/cache.go

@@ -39,8 +39,8 @@ type Store interface {
 	// Hint hints the Store that the provided talkgroups will be asked for.
 	Hint(ctx context.Context, tgs []ID) error
 
-	// Load loads the provided packed talkgroup IDs into the Store.
+	// Load loads the provided talkgroup ID tuples into the Store.
-	Load(ctx context.Context, tgs []int64) error
+	Load(ctx context.Context, tgs database.TGTuples) error
 
 	// Invalidate invalidates any caching in the Store.
 	Invalidate()
@@ -98,19 +98,20 @@ func NewCache() Store {
 
 func (t *cache) Hint(ctx context.Context, tgs []ID) error {
 	t.RLock()
-	var toLoad []int64
+	var toLoad database.TGTuples
 	if len(t.tgs) > len(tgs)/2 { // TODO: instrument this
 		for _, tg := range tgs {
 			_, ok := t.tgs[tg]
 			if !ok {
-				toLoad = append(toLoad, tg.Pack())
+				toLoad.Append(tg.System, tg.Talkgroup)
 			}
 		}
 
 	} else {
-		toLoad = make([]int64, 0, len(tgs))
+		toLoad[0] = make([]uint32, 0, len(tgs))
+		toLoad[1] = make([]uint32, 0, len(tgs))
 		for _, g := range tgs {
-			toLoad = append(toLoad, g.Pack())
+			toLoad.Append(g.System, g.Talkgroup)
 		}
 	}
 
@@ -127,7 +128,7 @@ func (t *cache) add(rec *Talkgroup) error {
 	t.Lock()
 	defer t.Unlock()
 
-	tg := TG(rec.System.ID, rec.Talkgroup.Tgid)
+	tg := TG(rec.System.ID, rec.Talkgroup.TGID)
 	t.tgs[tg] = rec
 	t.systems[int32(rec.System.ID)] = rec.System.Name
 
@@ -135,8 +136,8 @@ func (t *cache) add(rec *Talkgroup) error {
 }
 
 type row interface {
-	database.GetTalkgroupsWithLearnedByPackedIDsRow | database.GetTalkgroupsWithLearnedRow |
+	database.GetTalkgroupsRow | database.GetTalkgroupsWithLearnedRow |
-		database.GetTalkgroupsWithLearnedBySystemRow
+		database.GetTalkgroupsWithLearnedBySystemRow | database.GetTalkgroupWithLearnedRow
 	GetTalkgroup() database.Talkgroup
 	GetSystem() database.System
 	GetLearned() bool
@@ -180,7 +181,7 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
 		}
 		t.RUnlock()
 
-		tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, toGet.Packed())
+		tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, toGet.Tuples())
 		if err != nil {
 			return nil, err
 		}
@@ -196,8 +197,8 @@ func (t *cache) TGs(ctx context.Context, tgs IDs) ([]*Talkgroup, error) {
 	return addToRowList(t, r, tgRecords)
 }
 
-func (t *cache) Load(ctx context.Context, tgs []int64) error {
+func (t *cache) Load(ctx context.Context, tgs database.TGTuples) error {
-	tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, tgs)
+	tgRecords, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedBySysTGID(ctx, tgs)
 	if err != nil {
 		return err
 	}
@@ -245,7 +246,7 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
 		return rec, nil
 	}
 
-	recs, err := database.FromCtx(ctx).GetTalkgroupsWithLearnedByPackedIDs(ctx, []int64{tg.Pack()})
+	record, err := database.FromCtx(ctx).GetTalkgroupWithLearned(ctx, int32(tg.System), int32(tg.Talkgroup))
 	switch err {
 	case nil:
 	case pgx.ErrNoRows:
@@ -255,17 +256,13 @@ func (t *cache) TG(ctx context.Context, tg ID) (*Talkgroup, error) {
 		return nil, errors.Join(ErrNotFound, err)
 	}
 
-	if len(recs) < 1 {
+	err = t.add(rowToTalkgroup(record))
-		return nil, ErrNotFound
-	}
-
-	err = t.add(rowToTalkgroup(recs[0]))
 	if err != nil {
 		log.Error().Err(err).Msg("TG() cache add")
-		return rowToTalkgroup(recs[0]), errors.Join(ErrNotFound, err)
+		return rowToTalkgroup(record), errors.Join(ErrNotFound, err)
 	}
 
-	return rowToTalkgroup(recs[0]), nil
+	return rowToTalkgroup(record), nil
 }
 
 func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool) {
@@ -290,7 +287,7 @@ func (t *cache) SystemName(ctx context.Context, id int) (name string, has bool)
 }
 
 func (t *cache) UpdateTG(ctx context.Context, input database.UpdateTalkgroupParams) (*Talkgroup, error) {
-	sysName, has := t.SystemName(ctx, int(Unpack(input.ID).System))
+	sysName, has := t.SystemName(ctx, int(*input.SystemID))
 	if !has {
 		return nil, ErrNoSuchSystem
 	}

pkg/talkgroups/importer/import.go

@@ -0,0 +1,139 @@
+package importer
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"dynatron.me/x/stillbox/internal/jsontypes"
+	"dynatron.me/x/stillbox/pkg/database"
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+)
+
+type ImportSource string
+
+const (
+	ImportSrcRadioReference ImportSource = "radioreference"
+)
+
+var (
+	ErrBadImportType = errors.New("unknown import type")
+)
+
+type importer interface {
+	importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error)
+}
+
+type ImportJob struct {
+	Type     ImportSource `json:"type"`
+	SystemID int          `json:"systemID"`
+	Body     string       `json:"body"`
+
+	importer `json:"-"`
+}
+
+func (ij *ImportJob) Import(ctx context.Context) ([]talkgroups.Talkgroup, error) {
+	r := bytes.NewReader([]byte(ij.Body))
+
+	switch ij.Type {
+	case ImportSrcRadioReference:
+		ij.importer = new(radioReferenceImporter)
+	default:
+		return nil, ErrBadImportType
+	}
+
+	return ij.importTalkgroups(ctx, ij.SystemID, r)
+}
+
+type radioReferenceImporter struct {
+}
+
+type rrState int
+
+const (
+	rrsInitial rrState = iota
+	rrsGroupDesc
+	rrsTG
+)
+
+var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`)
+
+func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]talkgroups.Talkgroup, error) {
+	sc := bufio.NewScanner(r)
+	tgs := make([]talkgroups.Talkgroup, 0, 8)
+	sysn, has := talkgroups.StoreFrom(ctx).SystemName(ctx, sys)
+	if !has {
+		return nil, talkgroups.ErrNoSuchSystem
+	}
+
+	var groupName string
+	state := rrsInitial
+	for sc.Scan() {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+
+		ln := strings.Trim(sc.Text(), " \t\r\n")
+
+		switch state {
+		case rrsInitial:
+			groupName = ln
+			state++
+		case rrsGroupDesc:
+			if rrRE.MatchString(ln) {
+				state++
+			}
+		case rrsTG:
+			fields := strings.Split(ln, "\t")
+			if len(fields) != 6 {
+				state = rrsGroupDesc
+				groupName = ln
+				continue
+			}
+			tgid, err := strconv.Atoi(fields[0])
+			if err != nil {
+				continue
+			}
+			var metadata jsontypes.Metadata
+			tgt := talkgroups.TG(sys, tgid)
+			mode := fields[2]
+			if strings.Contains(mode, "E") {
+				metadata = make(jsontypes.Metadata)
+				metadata["encrypted"] = true
+			}
+			tags := []string{fields[5]}
+			gn := groupName // must take a copy
+			tgs = append(tgs, talkgroups.Talkgroup{
+				Talkgroup: database.Talkgroup{
+					ID:       uuid.New(),
+					TGID:     int32(tgt.Talkgroup),
+					SystemID: int32(tgt.System),
+					Name:     &fields[4],
+					AlphaTag: &fields[3],
+					TgGroup:  &gn,
+					Metadata: metadata,
+					Tags:     tags,
+					Weight:   1.0,
+				},
+				System: database.System{
+					ID:   sys,
+					Name: sysn,
+				},
+			})
+
+		}
+	}
+
+	if err := sc.Err(); err != nil {
+		return tgs, err
+	}
+
+	return tgs, nil
+}

pkg/talkgroups/importer/import_test.go

@@ -0,0 +1,90 @@
+package importer_test
+
+import (
+	"context"
+	"encoding/json"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"dynatron.me/x/stillbox/pkg/database"
+	"dynatron.me/x/stillbox/pkg/database/mocks"
+	"dynatron.me/x/stillbox/pkg/talkgroups"
+	"dynatron.me/x/stillbox/pkg/talkgroups/importer"
+)
+
+func getFixture(fixture string) []byte {
+	fixt, err := os.ReadFile("testdata/" + fixture)
+	if err != nil {
+		panic(err)
+	}
+
+	return fixt
+}
+
+func TestImport(t *testing.T) {
+	// this is for deterministic UUIDs
+	uuid.SetRand(rand.New(rand.NewSource(1)))
+
+	tests := []struct {
+		name      string
+		input     []byte
+		impType   string
+		sysID     int
+		sysName   string
+		jsExpect  []byte
+		expectErr error
+	}{
+		{
+			name:     "radioreference",
+			impType:  "radioreference",
+			input:    getFixture("riscon.txt"),
+			jsExpect: getFixture("riscon.json"),
+			sysID:    197,
+			sysName:  "RISCON",
+		},
+		{
+			name:      "unknown importer",
+			impType:   "nonexistent",
+			expectErr: importer.ErrBadImportType,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			dbMock := mocks.NewDB(t)
+			if tc.expectErr == nil {
+				dbMock.EXPECT().GetSystemName(mock.AnythingOfType("*context.valueCtx"), tc.sysID).Return(tc.sysName, nil)
+			}
+			ctx := database.CtxWithDB(context.Background(), dbMock)
+			ctx = talkgroups.CtxWithStore(ctx, talkgroups.NewCache())
+			ij := &importer.ImportJob{
+				Type:     importer.ImportSource(tc.impType),
+				SystemID: tc.sysID,
+				Body:     string(tc.input),
+			}
+
+			tgs, err := ij.Import(ctx)
+
+			if tc.expectErr != nil {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tc.expectErr.Error())
+			} else {
+				require.NoError(t, err)
+
+				var fixt []talkgroups.Talkgroup
+				err = json.Unmarshal(tc.jsExpect, &fixt)
+				// jse, _ := json.Marshal(tgs); os.WriteFile("testdata/riscon.json", jse, 0600)
+				require.NoError(t, err)
+
+				assert.Equal(t, fixt, tgs)
+			}
+
+		})
+	}
+}

pkg/talkgroups/importer/testdata/riscon.txt

@@ -0,0 +1,408 @@
+Statewide Mutual Aid/Intersystem
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+2	002	D	Intercity FD	Intercity Fire	Interop
+3	003	D	Intercity PD	Intercity Police	Interop
+State Police - District A (North)
+
+District A comprises barracks in Lincoln Woods and Scituate in the northern region of the state
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+21	015	D	RISP N Disp	North Dispatch 	Law Dispatch
+22	016	DE	RISP N Car	North Car-to-Car/Information	Law Talk
+24	018	DE	RISP N Tac 1	North Tactical Ops 1	Law Tac
+23	017	DE	RISP N Tac 2	North Tactical Ops 2	Law Tac
+State Police - District B (South)
+
+District B comprises barracks in Hope Valley, Wickford as well as detail assignments at TF Green Airport and Block Island in the southern region of the state
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+25	019	D	RISP S Disp	South Dispatch 	Law Dispatch
+27	01b	DE	RISP S Car	South Car-to-Car/Information	Law Talk
+Statewide Fire
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+16	010	D	State FMO	State Fire Marshall	Fire-Talk
+1038	40e	D	NRI Fire Chi	Northern Rhode Island Fire Chiefs	Fire-Talk
+1041	411	D	SRI Fire Chi	Southern Rhode Island Fire Chiefs	Fire-Talk
+1314	522	D	Tanker TF 1	Tanker Taskforce 1	Fire-Talk
+Statewide EMS and Hospitals
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+194	0c2	D	Lifepact Amb	Lifepact Ambulance (Statewide)	EMS Dispatch
+212	0d4	D	Fatima-St Joes	Fatima St Josephs	Business
+220	0dc	DE	Health 1	Health 1	EMS-Talk
+221	0dd	DE	Health 2	Health 2	EMS-Talk
+222	0de	D	Dept of HealthSW	Department of Health - Statewide	EMS-Talk
+228	0e4	DE	DMAT South	DMAT South	Emergency Ops
+232	0e8	D	Life Span 1	Life Span Net 1	EMS-Tac
+234	0ea	D	RI Hosp Ops	RI Hospital Operations	Business
+Department of Environmental Management
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+120	078	D	DEM PD Ops	Law Enforcement Operations	Law Dispatch
+122	07a	D	DEM Police	Law Enforcement Police	Law Talk
+Emergency Management Agency
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+10	00a	D	EMA-1	Emergency Management Agency 1	Emergency Ops
+20	014	D	EMA	Emergency Management Agency	Emergency Ops
+Statewide Area/Events
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+4	004	D	Wide Area 3	Wide Area 3	Interop
+5	005	D	Wide Area 4	Wide Area 4	Interop
+6	006	D	Wide Area 5	Wide Area 5	Interop
+7	007	D	Wide Area 6	Wide Area 6	Interop
+1018	3fa	D	SOUTHWIDE 1	Southwide CH-1	Interop
+1019	3fb	D	SOUTHWIDE 2	Southwide CH-2	Interop
+1022	3fe	D	WIDE AREA 7	Wide Area 7	Interop
+1023	3ff	DE	WIDE AREA 8	Wide Area 8	Interop
+1025	401	D	Inland Marine IO	Inland Marine Interop	Interop
+1037	40d	DE	SOUTHSIDE 5	Southside CH 5	Interop
+1173	495	D	NORTHWIDE1	North Wide 1	Interop
+1174	496	D	NORTHWIDE2	North Wide 2	Interop
+1177	499	DE	NORTHWIDE5	North Wide 5	Interop
+1185	4a1	D	METROWIDE1	Metro Wide 1	Interop
+1186	4a2	D	METROWIDE2	Metro Wide 2	Interop
+1187	4a3	DE	METROWIDE3	Metro Wide 3	Interop
+1335	537	D	EASTWIDE 1	East Wide 1	Interop
+1336	538	D	EASTWIDE 2	East Wide 2	Interop
+1337	539	DE	EASTWIDE 3	East Wide 3	Interop
+11186	2bb2	D	METROWIDE2	Metro Wide 2	Interop
+Statewide Emergency Response
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1033	409	D	TANK TF	Tanker Taskforce 	Fire-Tac
+1034	40a	D	HZT DC1	Hazmat 1	Fire-Tac
+1035	40b	D	HZT DC2	Hazmat 2	Fire-Tac
+Department of Transportation
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+176	0b0	D	RIDOT Primary	Department of Transportation - Primary	Public Works
+Tunnel and Bridge Authority
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+4421	1145	D	RITBA - Pell Bdg	Newport Pell Bridge Operations	Public Works
+Federal
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+274	112	D	VA Police	Providence VA Police	Law Dispatch
+RIPTA
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+186	0ba	DE	RIPTA	Rhode Island Public Transit Auth.	Transportation
+187	0bb	D	RIPTA	Rhode Island Public Transit Auth.	Transportation
+188	0bc	D	RIPTA	Rhode Island Public Transit Auth.	Transportation
+189	0bd	D	RIPTA	Rhode Island Public Transit Auth.	Transportation
+190	0be	D	RIPTA	Rhode Island Public Transit. Auth.	Transportation
+Quonset ANGB
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+304	130	D	Quonset ANGB FD	Fire Operations	Fire Dispatch
+Rhode Island Airport Commission
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+17	011	DE	TF Green PD	Airport Police Operations	Law Dispatch
+19	013	D	TF Green FD	Airport Fire Operations	Fire Dispatch
+College/Education Security
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1126	466	DE	URI PD	University of Rhode Island Police - Dispatch	Law Dispatch
+1131	46b	DE	URI EMS	University of Rhode Island - EMS	EMS Dispatch
+1348	544	D	St George Sec	St. George's School (Middletown) - Security	Security
+10228	27f4	DE	RISD Secuty	Rhode Island School of Design - Security	Security
+10229	27f5	DE	PROV COLL	Providence College Security - Dispatch	Security
+10230	27f6	D	RI COL SEC	Rhode Island College Security	Security
+11001	2af9	DE	BROWN UNIV	Brown University Police - Dispatch	Law Dispatch
+11002	2afa	DE	BROWN CAR	Brown University Police - Car-to-Car	Law Talk
+11003	2afb	DE	BROWN TAC	Brown University Police - Tactical	Law Tac
+Statewide Misc.
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+12	00c	DE	METROWIDE2	Metro Wide 2	Interop
+14	00e	DE	METROWIDE4	Metro Wide 4	Interop
+70	046	DE	TFC TRIBUNAL	RI Traffic Tribunal Security	Security
+168	0a8	D	Red Cross 1	Rhode Island Red Cross - Primary	Other
+169	0a9	D	Red Cross 2	Rhode Island Red Cross - Secondary	Other
+223	0df	D	NURSING HM	Statewide Nursing Homes Net	Other
+243	0f3	D	Slater Hosp Ops	Hospital Operations	Business
+244	0f4	D	Slater Hosp Sec	Slater Hospital Security	Security
+Washington County
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1042	412	D	WashCo FireG	County Fireground	Fire-Tac
+1479	5c7	D	WashCo FireS	County Fire Station/Station	Fire-Talk
+Barrington
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1712	6b0	D	BarringtnFD1	Fire 1 Dispatch	Fire Dispatch
+1713	6b1	D	BarringtnFD2	Fire 2	Fire-Tac
+1715	6b3	DE	BarringtonPD 1	Police Operations	Law Dispatch
+1716	6b4	D	BarringtonPD 2	Police Secondary	Law Tac
+Bristol
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1744	6d0	D	Bristol FD	Fire Operations (Patch from VHF)	Fire Dispatch
+1755	6db	D	Bristol Harbor	Harbormaster	Public Works
+Burrillville
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+2003	7d3	D	Burrville PD	Police	Law Dispatch
+2004	7d4	D	Burrvl PD2	Police 2	Law Talk
+2005	7d5	DE	Burrvl PD3	Police 3 Detectives	Law Tac
+2006	7d6	D	Burrvl PD4	Police 4	Law Tac
+2000	7d0	D	Burrvl FD	Fire Misc (Ops are VHF)	Fire-Tac
+2001	7d1	D	Burvl FDTAC1	Fire TAC-1	Fire-Tac
+2009	7d9	D	Burvl FDTAC2	Fire TAC-2	Fire-Tac
+2002	7d2	D	Burrvl EMS	EMS Misc (Ops are VHF)	EMS-Tac
+2007	7d7	D	Burrvl Town	Town-Wide	Multi-Tac
+2008	7d8	D	Burrvl EMA	Emergency Management	Emergency Ops
+Central Falls
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1838	72e	D	CentFallsPD1	Police 1 Dispatch	Law Dispatch
+1839	72f	D	CentFallsPD2	Police 2	Law Dispatch
+1835	72b	D	CentFalls FD 1	Fire Dispatch (Simulcast of UHF)	Fire Dispatch
+1836	72c	D	CentFalls FD 2	Fireground	Fire-Tac
+Charlestown
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1425	591	D	CharlestownPD	Police Operations - Simulcast of UHF	Law Dispatch
+1429	595	D	Chastown EMS	EMS - Linked to 151.3325	EMS Dispatch
+Coventry
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1483	5cb	D	Coventry PD	Police 1 - Dispatch	Law Dispatch
+1484	5cc	D	Coventry PD2	Police 2	Law Tac
+1480	5c8	D	Coventry FD	Fire	Fire Dispatch
+Cranston
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1500	5dc	D	Cranston FD Disp	Fire - Dispatch/Operations	Fire Dispatch
+1501	5dd	D	Cranston FD FG2	Fire - Fireground 2	Fire-Tac
+1502	5de	D	Cranston FD FG3	Fire - Fireground 3	Fire-Tac
+1503	5df	D	Cranston FD FG4	Fire - Fireground 4	Fire-Talk
+1504	5e0	D	Cranston FD Admi	Fire - Admin/Alt Fireground 5	Fire-Talk
+Cumberland
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1520	5f0	D	Cumberland FD	Fire	Fire Dispatch
+1523	5f3	D	Cumberland PD	Police Secondary	Law Dispatch
+East Greenwich
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1776	6f0	D	E Greenwich F-TA	Fire Talk Around	Fire-Talk
+1779	6f3	D	E Greenwich PD	Police Operations	Law Dispatch
+East Providence
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1869	74d	D	E Prov PD 1	Police 1 - Dispatch	Law Dispatch
+1872	750	DE	E Prov PD 2	Police 2	Law Talk
+1870	74e	DE	E Prov PD 3	Police 3	Law Talk
+1883	75b	DE	E Prov PD12	Detectives	Law Talk
+1866	74a	D	E Prov FD 1	Fire - Dispatch/Operations	Fire Dispatch
+1867	74b	D	E Prov FD 2	Fire "Channel 2"	Fire-Tac
+1878	756	D	E Prov FD 3	Fire "Channel 3"	Fire-Tac
+Exeter
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+2064	810	D	Exeter FD-G	Fire - Fireground	Fire-Tac
+Foster
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1904	770	D	Foster Fire	Fire	Fire Dispatch
+Glocester
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1939	793	D	Glocester PD	Police	Law Dispatch
+1940	794	D	Glocester PD 2	Police Secondary	Law Tac
+Hopkinton
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1410	582	DE	Hopkinton PD	Police	Law Dispatch
+Jamestown
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1100	44c	DE	Jamestown PD 1	Police 1 - Dispatch	Law Dispatch
+1101	44d	DE	Jamestown PD 2	Police 2	Law Dispatch
+1108	454	D	Jamestown FD	Fire	Fire Dispatch
+1120	460	D	Jamestown FG 1	Fireground 1	Fire-Tac
+1121	461	D	Jamestown FG 2	Fireground 2	Fire-Tac
+1114	45a	D	Jamestown DPW	Public Works	Public Works
+1107	453	D	Jamestown School	Town Schools	Schools
+Johnston
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1619	653	DE	Johnston PD	Police Operations	Law Dispatch
+1616	650	D	Johnston FD	Fire Operations	Fire Dispatch
+1617	651	D	Johnston FG	Fireground	Fire-Tac
+Lincoln
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1683	693	D	Lincoln Police	Police F1	Law Dispatch
+1684	694	D	Lincoln Police 2	Police F2	Law Tac
+1680	690	D	Lincoln Fire 1	Fire Dispatch	Fire Dispatch
+1681	691	D	Lincoln Fire 2	Fireground 2	Fire-Tac
+1691	69b	D	Lincoln Fire 3	Fireground 3	Fire-Tac
+1682	692	D	Lincoln EMS	EMS	EMS Dispatch
+1688	698	D	Lincoln EMA	Emergency Management	Emergency Ops
+1687	697	D	Lincoln Townwide	Townwide	Interop
+1692	69c	D	Lincoln DPW	Public Works	Public Works
+Little Compton
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1264	4f0	D	LittleCompPD	Police	Law Dispatch
+1266	4f2	D	LittleCompFD	Fire	Fire Dispatch
+Middletown
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1338	53a	D	MiddletownPD	Police Operations	Law Dispatch
+1343	53f	D	Middletown FD	Fire Operations	Fire Dispatch
+1345	541	D	MiddletownTW	Townwide	Multi-Dispatch
+Narragansett
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1001	3e9	DE	Narrag PD 1	Police - Dispatch	Law Dispatch
+1002	3ea	DE	Narrag PD 2	Police - Car/Car	Law Talk
+1003	3eb	DE	Narrag PD 3	Police - Special Details 1/Town Beaches	Law Tac
+1004	3ec	DE	Narrag PD 4	Police - Special Details 2	Law Tac
+1005	3ed	DE	Narrag PD 5	Police - Harbormaster	Law Talk
+1007	3ef	DE	Narrag PD 7	Police - Detectives	Law Talk
+1008	3f0	DE	Narrag PD 8	Police - Detectives	Law Talk
+1006	3ee	D	Narrag FD	Fire - Dispatch	Fire Dispatch
+1012	3f4	D	Narrag FDFG1	Fire - Fireground 1	Fire-Tac
+1013	3f5	D	Narrag FDFG2	Fire - Fireground 2	Fire-Tac
+1016	3f8	D	Narrag FD AD	Fire - Administration	Fire-Talk
+1014	3f6	D	Narrag EMS	Fire - EMS Ops	EMS Dispatch
+1017	3f9	D	Narrag DPW	Public Works	Public Works
+1010	3f2	D	Narrag TownA	Town Administration	Other
+1011	3f3	D	Narrag IOP	Townwide Interop	Interop
+New Shoreham
+
+New Shoreham is on Block Island. New Shoreham operates primarily on their own Capacity Plus trunk.
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1376	560	D	New Shore PD	Police	Law Dispatch
+Newport
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1300	514	DE	Newport PD 1	Police 1 - Dispatch	Law Dispatch
+1302	516	DE	Newport PD 2	Police 2 - Records	Law Talk
+1304	518	DE	Newport PD 4	Police 4 - Tactical 1	Law Talk
+1307	51b	DE	Newport PD 7	Police 7 - Tactical 4	Law Talk
+1308	51c	DE	Newport PD 8	Police 8 - Tactical 5	Law Talk
+1303	517	D	Newport FD1	Fire Dispatch/Operations	Fire Dispatch
+1305	519	D	Newport FG1	Fireground Ops 1	Fire-Tac
+1306	51a	D	Newport FG2	Fireground Ops 2	Fire-Tac
+1301	515	D	Newport FDT	Fire - Training	Fire-Talk
+1291	50b	D	Newport Water	Water Department	Public Works
+1293	50d	D	Newport DPW	Public Works	Public Works
+1297	511	D	Newport Evnt	Citywide Events	Public Works
+1312	520	D	Newport CW	Newport Citywide	Interop
+North Kingstown
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1285	505	D	NKing PD 1	Police 1 - Dispatch	Law Dispatch
+1286	506	DE	NKing PD 2	Police 2 - Admin	Law Talk
+1287	507	De	NKing PD 3	Police 3 - Car/Car	Law Tac
+1280	500	D	NKing Fire D	Fire - Dispatch	Fire Dispatch
+1281	501	D	NKing Fire G	Fire - Fireground	Fire-Tac
+North Providence
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1536	600	DE	NorthPrv PD1	Police 1 - Dispatch	Law Dispatch
+1537	601	DE	NorthPrv PD2	Police 2 - Car/Car	Law Talk
+1538	602	DE	NorthPrv PD3	Police 3 - Tactical	Law Tac
+1547	60b	D	NorthPrv FDD	Fire Dispatch 	Fire Dispatch
+1548	60c	D	NorthPrv Fire 2	Fire 2	Fire-Tac
+1549	60d	D	NorthPrv Fire 3	Fire 3	Fire-Tac
+1550	60e	D	NorthPrv Fire 4	Fire 4	Fire-Tac
+1551	60f	D	NorthPrv Fire 5	Fire 5	Fire Dispatch
+1552	610	DE	NorthPrv Fire 6	Fire 6	Fire-Tac
+1544	608	D	NorthPrv TownW 1	Townwide 1	Interop
+1545	609	D	NorthPrv TownW 2	Townwide 2	Interop
+1554	612	D	NorthPrv DPW	Public Works	Public Works
+North Smithfield
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1971	7b3	DE	N Smithfd PD	Police	Law Dispatch
+1968	7b0	D	N Smithfield FD	Fire Dispatch/Operations	Fire Dispatch
+1969	7b1	D	N Smithfield FD2	Fire Secondary	Fire-Tac
+1981	7bd	D	N Smithfield FD3	Fireground	Fire-Tac
+Pawtucket
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1440	5a0	D	Pawtucket FD 1	Fire - Operations	Fire Dispatch
+1441	5a1	D	Pawtucket FG	Fireground	Fire-Tac
+1442	5a2	D	Pawtucket EMSTac	EMS Tac	EMS-Tac
+Portsmouth
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1248	4e0	DE	PortsmouthPD	Police	Law Dispatch
+1253	4e5	D	Portsmouth FD	Fire Dispatch (Patch to VHF Primary)	Fire Dispatch
+1255	4e7	D	Portsmouth FG	Fireground	Fire-Tac
+1262	4ee	D	Prudence Isl FD	Island Fire Dispatch	Fire Dispatch
+Providence (City)
+
+Providence fireground channels may be patched.
+As of this writing,
+FG 05 (10102) and 02 (10107) are patched
+FG 06 (10103) and 03 (10108) are patched 
+FG 07 (10104) and 04 (10109) are patched.
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+10000	2710	D	PPD ATG	Police - All Call - Emergency Broadcasts	Emergency Ops
+10001	2711	D	PPD CH 1	Police 1 - Dispatch	Law Dispatch
+10002	2712	DE	PPD CH 2	Police 2	Law Talk
+10003	2713	DE	PPD CH 3	Police 3	Law Talk
+10004	2714	DE	PPD CH-4	Police 4	Law Talk
+10005	2715	DE	PPD DETEC 1	Police 5 -Detectives 1	Law Talk
+10006	2716	DE	PPD T/A	Police 6 - Car-to-Car	Law Talk
+10007	2717	DE	PPD NARC 1	Police 7 - Narcotics 1	Law Talk
+10008	2718	DE	PPD NARC 2	Police 8 - Narcotics 2	Law Tac
+10009	2719	DE	PPD DETEC 2	Police 9 - Detectives 2	Law Talk
+10010	271a	DE	PPD DETAIL 1	Police 10 - Special Details 1	Law Tac
+10011	271b	DE	PPD DETAIL 2	Police 11 - Special Details 2	Law Tac
+10012	271c	DE	PPD CORR SEC	Police 12 - Corrections Security	Law Talk
+10013	271d	DE	PPD SRU	Police 13 - Special Response Unit	Law Tac
+10014	271e	DE	PPD ADMIN	Police 14 - Administration	Law Talk
+10100	2774	D	PROV FD ATG	Fire All Call - Emergency Broadcasts	Emergency Ops
+10101	2775	D	PFD DISPATCH	Fire Dispatch	Fire Dispatch
+10107	277b	D	PFD CH-2 FG	Fireground 2	Fire-Tac
+10108	277c	D	PFD CH-3 FG	Fireground 3	Fire-Tac
+10109	277d	D	PFD CH-4 FG	Fireground 4	Fire-Tac
+10102	2776	D	PFD CH-5	Fire 5	Fire-Tac
+10103	2777	D	PFD CH-6	Fire 6	Fire-Tac
+10104	2778	D	PFD CH-7	Fire 7	Fire-Tac
+10110	277e	D	PFD M/A 1	Fire - Mutual Aid 1	Fire-Tac
+10111	277f	D	PFD M/A 2	Fire - Mutual Aid 2	Fire-Tac
+10112	2780	D	PFD M/A 3	Fire - Mutual Aid 3	Fire-Tac
+10113	2781	D	PFD Fireground 8	Fireground 8	Fire-Talk
+10105	2779	D	PFD ADMIN	Fire - Administration	Fire-Talk
+10106	277a	D	PFD COMM	Fire - Communications	Fire-Talk
+10207	27df	D	PROV DPW	Public Works	Public Works
+Richmond
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+2035	7f3	D	Richmond PD	Police	Law Dispatch
+2042	7fa	D	Chariho Reg HS	Chariho Regional High School	Schools
+Scituate
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1460	5b4	D	Scituate PD	Police	Law Dispatch
+1463	5b7	D	Scituate FD	Fire Operations	Fire Dispatch
+Smithfield
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1651	673	D	SmithfieldPD	Police Operations	Law Dispatch
+1652	674	D	Smfld PD 2	Police Secondary	Law Tac
+1653	675	DE	Smfld PD Det	Police Detectives	Law Tac
+1654	676	DE	Smfld PD Adm	Police Admin	Law Talk
+1661	67d	D	Smfld PD Dtl	Police Details	Law Talk
+1648	670	D	SmithfieldFD	Fire - Fireground	Fire-Tac
+1655	677	D	Smfld Town	Town-Wide	Multi-Tac
+1657	679	D	Smfld EMA	Emergency Management	Emergency Ops
+1660	67c	D	Smfld DPW	Public Works	Public Works
+South Kingstown
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1225	4c9	DE	SKing PD 1	Police 1 - Dispatch	Law Dispatch
+1226	4ca	DE	SKing PD 2	Police 2 - Car/Car	Law Talk
+1235	4d3	DE	SKing PD 3	Police 3 - Tactical	Law Tac
+1236	4d4	DE	SKing PD 5	Police 5 - Tactical	Law Tac
+1232	4d0	D	SKing FD Lnk	Fire - UHF Simulcast	Fire Dispatch
+1240	4d8	D	SKing Fire D	Fire - Detail	Fire-Talk
+1227	4cb	D	UnionFD FG 1	Union Fire District - Fireground 1	Fire-Tac
+1237	4d5	D	UnionFD FG 2	Union Fire District - Fireground 2	Fire-Tac
+1026	402	D	UnionFD Evnt	Union Fire District - Special Events	Fire-Talk
+1015	3f7	DE	SKing EMS	EMS	EMS Dispatch
+Tiverton
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1316	524	D	Tiverton PD	Police (Simulcast 482.9625)	Law Dispatch
+1315	523	D	Tiverton FD	Fire (Simulcast 471.7875)	Fire Dispatch
+Warwick
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1162	48a	D	Warwick FD	Fire	Fire Dispatch
+1170	492	D	Warwick FG	Fireground	Fire-Tac
+West Greenwich
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1805	70d	D	W Greenwh PD	Police	Law Dispatch
+1806	70e	D	W GreenwichPD2	Police Secondary	Law Tac
+West Warwick
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1208	4b8	D	W Warwick FD	Fire Operations	Fire Dispatch
+Westerly
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1050	41a	DE	Westerly PD1	Police 1 - Dispatch	Law Dispatch
+1051	41b	DE	Westerly PD2	Police 2	Law Talk
+1052	41c	DE	Westerly PD3	Police 3	Law Talk
+1053	41d	DE	Westerly PD4	Police 4	Law Talk
+1054	41e	D	Westerly PD5	Police 5 - Reserve Officers	Law Talk
+1064	428	D	Westerly PD6	Police 6 - Traffic Division	Law Talk
+1063	427	D	Westerly FD	Fire Operations	Fire Dispatch
+1072	430	D	Westerly PFE	Police/Fire/EMS Ops	Multi-Talk
+1082	43a	D	Westerly EMS 	EMS Operations	EMS Dispatch
+Woonsocket
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1363	553	D	Woonskt PD 1	Police 1 - Dispatch	Law Dispatch
+1364	554	DE	Woonskt PD 2	Police 2	Law Talk
+1360	550	D	Woonsocket FD D	Fire Dispatch - Operations	Fire-Talk
+1361	551	D	Woonsocket FD 2	Fire Secondary	Fire Dispatch
+1354	54a	D	Woonskt FD 3	Fire - Fireground 3	Fire-Tac
+1367	557	D	Woonskt City	Citywide	Multi-Talk
+1368	558	D	Woonsocket PW	Public Works - Streets	Public Works
+Radio Technicians
+DEC 	HEX 	Mode 	Alpha Tag	Description	Tag
+1	001	D	Radio Techs	RISCON Radio Technicians	Public Works
+10125	278d	D	Radio Techs	RISCON Radio Technicians	Public Works

pkg/talkgroups/talkgroup.go

@@ -12,6 +12,8 @@ type Talkgroup struct {
 	Learned bool            `json:"learned"`
 }
 
+type Metadata map[string]interface{}
+
 type Names struct {
 	System    string
 	Talkgroup string
@@ -24,13 +26,16 @@ type ID struct {
 
 type IDs []ID
 
-func (ids *IDs) Packed() []int64 {
+func (t IDs) Tuples() database.TGTuples {
-	r := make([]int64, len(*ids))
+	sys := make([]uint32, len(t))
-	for i := range *ids {
+	tg := make([]uint32, len(t))
-		r[i] = (*ids)[i].Pack()
+
+	for i := range t {
+		sys[i] = t[i].System
+		tg[i] = t[i].Talkgroup
 	}
 
-	return r
+	return database.TGTuples{sys, tg}
 }
 
 type intId interface {
@@ -44,18 +49,6 @@ func TG[T intId, U intId](sys T, tgid U) ID {
 	}
 }
 
-func (t ID) Pack() int64 {
-	// P25 system IDs are 12 bits, so we can fit them in a signed 8 byte int (int64, pg INT8)
-	return int64((int64(t.System) << 32) | int64(t.Talkgroup))
-}
-
-func Unpack(id int64) ID {
-	return ID{
-		System:    uint32(id >> 32),
-		Talkgroup: uint32(id & 0xffffffff),
-	}
-}
-
 func (t ID) String() string {
 	return fmt.Sprintf("%d:%d", t.System, t.Talkgroup)
 

sql/postgres/migrations/001_initial.up.sql

@@ -23,31 +23,10 @@ CREATE TABLE IF NOT EXISTS systems(
 	name TEXT NOT NULL
 );
 
-CREATE OR REPLACE FUNCTION systg2id(_sys INTEGER, _tg INTEGER) RETURNS INT8 LANGUAGE plpgsql AS
-$$
-BEGIN
-	RETURN ((_sys::BIGINT << 32) | _tg);
-END
-$$;
-
-CREATE OR REPLACE FUNCTION tgfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
-$$
-BEGIN
-	RETURN (_id & x'ffffffff'::BIGINT);
-END
-$$;
-
-CREATE OR REPLACE FUNCTION sysfromid(_id INT8) RETURNS INTEGER LANGUAGE plpgsql AS
-$$
-BEGIN
-	RETURN (_id >> 32);
-END
-$$;
-
 CREATE TABLE IF NOT EXISTS talkgroups(
-	id INT8 PRIMARY KEY,
+	id UUID PRIMARY KEY,
-	system_id INT4 REFERENCES systems(id) NOT NULL GENERATED ALWAYS AS (id >> 32) STORED,
+	system_id INT4 REFERENCES systems(id) NOT NULL,
-	tgid INT4 NOT NULL GENERATED ALWAYS AS (id & x'ffffffff'::BIGINT) STORED,
+	tgid INT4 NOT NULL,
 	name TEXT,
 	alpha_tag TEXT,
 	tg_group TEXT,
@@ -56,9 +35,12 @@ CREATE TABLE IF NOT EXISTS talkgroups(
 	tags TEXT[] NOT NULL DEFAULT '{}',
 	alert BOOLEAN NOT NULL DEFAULT 'true',
 	alert_config JSONB,
-	weight REAL NOT NULL DEFAULT 1.0
+	weight REAL NOT NULL DEFAULT 1.0,
+	UNIQUE (system_id, tgid)
 );
 
+CREATE INDEX talkgroups_system_tgid_idx ON talkgroups (system_id, tgid);
+
 CREATE INDEX IF NOT EXISTS talkgroup_id_tags ON talkgroups USING GIN (tags);
 
 CREATE TABLE IF NOT EXISTS talkgroups_learned(

sql/postgres/queries/talkgroups.sql

@@ -8,30 +8,21 @@ WHERE tags && ARRAY[$1];
 
 -- name: GetTalkgroupIDsByTags :many
 SELECT system_id, tgid FROM talkgroups
-WHERE (tags @> ARRAY[sqlc.arg(anyTags)])
+WHERE (tags @> ARRAY[@any_tags])
-AND (tags && ARRAY[sqlc.arg(allTags)])
+AND (tags && ARRAY[@all_tags])
-AND NOT (tags @> ARRAY[sqlc.arg(notTags)]);
+AND NOT (tags @> ARRAY[@not_tags]);
 
 -- name: GetTalkgroupTags :one
 SELECT tags FROM talkgroups
-WHERE id = systg2id($1, $2);
+WHERE system_id = @system_id AND tgid = @tg_id;
 
 -- name: SetTalkgroupTags :exec
-UPDATE talkgroups SET tags = $3
+UPDATE talkgroups SET tags = @tags
-WHERE id = systg2id($1, $2);
+WHERE system_id = @system_id AND tgid = @tg_id;
-
--- name: BulkSetTalkgroupTags :exec
-UPDATE talkgroups SET tags = $2
-WHERE id = ANY($1);
 
 -- name: GetTalkgroup :one
 SELECT sqlc.embed(talkgroups) FROM talkgroups
-WHERE id = systg2id(sqlc.arg(system_id), sqlc.arg(tgid));
+WHERE (system_id, tgid) = (@system_id, @tg_id);
-
--- name: GetTalkgroupsByPackedIDs :many
-SELECT sqlc.embed(tg), sqlc.embed(sys) FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = ANY($1::INT8[]);
 
 -- name: GetTalkgroupWithLearned :one
 SELECT
@@ -39,35 +30,17 @@ sqlc.embed(tg), sqlc.embed(sys),
 FALSE learned
 FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = systg2id(sqlc.arg(system_id), sqlc.arg(tgid))
+WHERE (tg.system_id, tg.tgid) = (@system_id, @tgid)
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
 TRUE learned
 FROM talkgroups_learned tgl
 JOIN systems sys ON tgl.system_id = sys.id
-WHERE tgl.system_id = sqlc.arg(system_id) AND tgl.tgid = sqlc.arg(tgid) AND ignored IS NOT TRUE;
+WHERE tgl.system_id = @system_id AND tgl.tgid = @tgid AND ignored IS NOT TRUE;
-
--- name: GetTalkgroupsWithLearnedByPackedIDs :many
-SELECT
-sqlc.embed(tg), sqlc.embed(sys),
-FALSE learned
-FROM talkgroups tg
-JOIN systems sys ON tg.system_id = sys.id
-WHERE tg.id = ANY($1::INT8[])
-UNION
-SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
-tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
-CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
-TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
-TRUE learned
-FROM talkgroups_learned tgl
-JOIN systems sys ON tgl.system_id = sys.id
-WHERE systg2id(tgl.system_id, tgl.tgid) = ANY($1::INT8[]) AND ignored IS NOT TRUE;
 
 -- name: GetTalkgroupsWithLearnedBySystem :many
 SELECT
@@ -78,7 +51,7 @@ JOIN systems sys ON tg.system_id = sys.id
 WHERE tg.system_id = @system
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -95,7 +68,7 @@ FROM talkgroups tg
 JOIN systems sys ON tg.system_id = sys.id
 UNION
 SELECT
-tgl.id::INT8, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
+NULL::UUID, tgl.system_id::INT4, tgl.tgid::INT4, tgl.name,
 tgl.alpha_tag, tgl.alpha_tag, NULL::INTEGER, NULL::JSONB,
 CASE WHEN tgl.alpha_tag IS NULL THEN NULL ELSE ARRAY[tgl.alpha_tag] END,
 TRUE, NULL::JSONB, 1.0, sys.id, sys.name,
@@ -105,7 +78,7 @@ JOIN systems sys ON tgl.system_id = sys.id
 WHERE ignored IS NOT TRUE;
 
 -- name: GetSystemName :one
-SELECT name FROM systems WHERE id = sqlc.arg(system_id);
+SELECT name FROM systems WHERE id = @system_id;
 
 -- name: UpdateTalkgroup :one
 UPDATE talkgroups
@@ -119,5 +92,5 @@ SET
 	alert = COALESCE(sqlc.narg('alert'), alert),
 	alert_config = COALESCE(sqlc.narg('alert_config'), alert_config),
 	weight = COALESCE(sqlc.narg('weight'), weight)
-WHERE id = @id
+WHERE id = sqlc.narg('id') OR (system_id = sqlc.narg('system_id') AND tgid = sqlc.narg('tgid'))
 RETURNING *;

sql/sqlc.yaml

@@ -11,6 +11,9 @@ sql:
         query_parameter_limit: 3
         emit_json_tags: true
         emit_interface: true
+        initialisms:
+          - id
+          - tgid
         emit_pointers_for_null_types: true
         overrides:
           - db_type: "uuid"
@@ -32,3 +35,8 @@ sql:
               import: "dynatron.me/x/stillbox/pkg/alerting/rules"
               type: "AlertRules"
             nullable: true
+          - column: "talkgroups.metadata"
+            go_type:
+              import: "dynatron.me/x/stillbox/internal/jsontypes"
+              type: "Metadata"
+            nullable: true

util/dumpdb.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+config=config.yaml
+pgformat="-Fc"
+ext=pgdump
+
+while getopts ":p" arg; do
+  case $arg in
+  c)
+    config=$OPTARG
+    ;;
+  p)
+    pgformat="-Fp"
+    ext=sql
+    ;;
+  esac
+done
+
+filen=`date "+backups/%Y%m%d_%H%M%S.${ext}"`
+
+mkdir -p backups/
+dbstring=`yq -r .db.connect "${config}"`
+pg_dump "${pgformat}" -f "${filen}" -T calls "${dbstring}"
+echo "backed up to ${filen}"