Importer, rename jsontime to jsontypes

internal/common/template.go

@@ -7,7 +7,7 @@ import (
 	"text/template"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 )
 
 var (
@@ -27,7 +27,7 @@ var (
 			}
 			return dict, nil
 		},
-		"formTime": func(t jsontime.Time) string {
+		"formTime": func(t jsontypes.Time) string {
 			return time.Time(t).Format("2006-01-02T15:04")
 		},
 		"ago": func(s string) (string, error) {

internal/forms/forms.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"github.com/araddon/dateparse"
 )
@@ -262,13 +262,13 @@ func (o *options) iterFields(r *http.Request, destStruct reflect.Value) error {
 				return err
 			}
 			setVal(destFieldVal, set, val)
-		case time.Time, *time.Time, jsontime.Time, *jsontime.Time:
+		case time.Time, *time.Time, jsontypes.Time, *jsontypes.Time:
 			t, set, err := o.parseTime(ff)
 			if err != nil {
 				return err
 			}
 			setVal(destFieldVal, set, t)
-		case time.Duration, *time.Duration, jsontime.Duration, *jsontime.Duration:
+		case time.Duration, *time.Duration, jsontypes.Duration, *jsontypes.Duration:
 			d, set, err := o.parseDuration(ff)
 			if err != nil {
 				return err

internal/forms/forms_test.go

@@ -10,7 +10,7 @@ import (
 
 	"dynatron.me/x/stillbox/internal/common"
 	"dynatron.me/x/stillbox/internal/forms"
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"dynatron.me/x/stillbox/pkg/alerting"
 	"dynatron.me/x/stillbox/pkg/config"
@@ -48,19 +48,19 @@ type urlEncTest struct {
 }
 
 type urlEncTestJT struct {
-	LookbackDays uint              `json:"lookbackDays"`
+	LookbackDays uint               `json:"lookbackDays"`
-	HalfLife     jsontime.Duration `json:"halfLife"`
+	HalfLife     jsontypes.Duration `json:"halfLife"`
-	Recent       string            `json:"recent"`
+	Recent       string             `json:"recent"`
-	ScoreStart   jsontime.Time     `json:"scoreStart"`
+	ScoreStart   jsontypes.Time     `json:"scoreStart"`
-	ScoreEnd     jsontime.Time     `json:"scoreEnd"`
+	ScoreEnd     jsontypes.Time     `json:"scoreEnd"`
 }
 
 type ptrTestJT struct {
-	LookbackDays uint               `form:"lookbackDays"`
+	LookbackDays uint                `form:"lookbackDays"`
-	HalfLife     *jsontime.Duration `form:"halfLife"`
+	HalfLife     *jsontypes.Duration `form:"halfLife"`
-	Recent       *string            `form:"recent"`
+	Recent       *string             `form:"recent"`
-	ScoreStart   *jsontime.Time     `form:"scoreStart"`
+	ScoreStart   *jsontypes.Time     `form:"scoreStart"`
-	ScoreEnd     jsontime.Time      `form:"scoreEnd"`
+	ScoreEnd     jsontypes.Time      `form:"scoreEnd"`
 }
 
 var (
@@ -73,33 +73,33 @@ var (
 
 	UrlEncTestJT = urlEncTestJT{
 		LookbackDays: 7,
-		HalfLife:     jsontime.Duration(30 * time.Minute),
+		HalfLife:     jsontypes.Duration(30 * time.Minute),
 		Recent:       "2h0m0s",
-		ScoreStart:   jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
+		ScoreStart:   jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
 	}
 
 	PtrTestJT = ptrTestJT{
 		LookbackDays: 7,
-		HalfLife:     common.PtrTo(jsontime.Duration(30 * time.Minute)),
+		HalfLife:     common.PtrTo(jsontypes.Duration(30 * time.Minute)),
 		Recent:       common.PtrTo("2h0m0s"),
-		ScoreStart:   common.PtrTo(jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
+		ScoreStart:   common.PtrTo(jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
 	}
 
 	UrlEncTestJTLocal = urlEncTestJT{
 		LookbackDays: 7,
-		HalfLife:     jsontime.Duration(30 * time.Minute),
+		HalfLife:     jsontypes.Duration(30 * time.Minute),
 		Recent:       "2h0m0s",
-		ScoreStart:   jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
+		ScoreStart:   jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
 	}
 
 	realSim = &alerting.Simulation{
 		Alerting: config.Alerting{
 			LookbackDays: 7,
-			HalfLife:     jsontime.Duration(30 * time.Minute),
+			HalfLife:     jsontypes.Duration(30 * time.Minute),
-			Recent:       jsontime.Duration(2 * time.Hour),
+			Recent:       jsontypes.Duration(2 * time.Hour),
 		},
-		SimInterval: jsontime.Duration(5 * time.Minute),
+		SimInterval: jsontypes.Duration(5 * time.Minute),
-		ScoreStart:  jsontime.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
+		ScoreStart:  jsontypes.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
 	}
 
 	Call1 = callUploadRequest{
@@ -194,7 +194,7 @@ func TestUnmarshal(t *testing.T) {
 			opts:   []forms.Option{forms.WithAcceptBlank()},
 		},
 		{
-			name:      "url encoded jsontime",
+			name:      "url encoded jsontypes",
 			r:         makeRequest("urlenc.http"),
 			dest:      &urlEncTestJT{},
 			expect:    &UrlEncTestJT,
@@ -202,14 +202,14 @@ func TestUnmarshal(t *testing.T) {
 			opts:      []forms.Option{forms.WithTag("json")},
 		},
 		{
-			name:   "url encoded jsontime with tz",
+			name:   "url encoded jsontypes with tz",
 			r:      makeRequest("urlenc.http"),
 			dest:   &urlEncTestJT{},
 			expect: &UrlEncTestJT,
 			opts:   []forms.Option{forms.WithAcceptBlank(), forms.WithParseTimeInTZ(time.UTC), forms.WithTag("json")},
 		},
 		{
-			name:   "url encoded jsontime with local",
+			name:   "url encoded jsontypes with local",
 			r:      makeRequest("urlenc.http"),
 			dest:   &urlEncTestJT{},
 			expect: &UrlEncTestJTLocal,

internal/jsontypes/jsontime.go

@@ -1,4 +1,4 @@
-package jsontime
+package jsontypes
 
 import (
 	"encoding/json"

internal/jsontypes/metadata.go

@@ -0,0 +1,3 @@
+package jsontypes
+
+type Metadata map[string]interface{}

pkg/alerting/simulate.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"dynatron.me/x/stillbox/internal/forms"
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/internal/trending"
 	"dynatron.me/x/stillbox/pkg/config"
 	"dynatron.me/x/stillbox/pkg/talkgroups"
@@ -23,12 +23,12 @@ type Simulation struct {
 	config.Alerting
 
 	// ScoreStart is the time when scoring begins
-	ScoreStart jsontime.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
+	ScoreStart jsontypes.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
 	// ScoreEnd is the time when the score simulator ends. Left blank, it defaults to time.Now()
-	ScoreEnd jsontime.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
+	ScoreEnd jsontypes.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
 
 	// SimInterval is the interval at which the scorer will be called
-	SimInterval jsontime.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
+	SimInterval jsontypes.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
 
 	clock    offsetClock `json:"-"`
 	*alerter `json:"-"`
@@ -64,7 +64,7 @@ func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.I
 	s.Enable = true
 	s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter)
 	if time.Time(s.ScoreEnd).IsZero() {
-		s.ScoreEnd = jsontime.Time(now)
+		s.ScoreEnd = jsontypes.Time(now)
 	}
 	log.Debug().Time("scoreStart", s.ScoreStart.Time()).
 		Time("scoreEnd", s.ScoreEnd.Time()).

pkg/config/config.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"dynatron.me/x/stillbox/internal/jsontime"
+	"dynatron.me/x/stillbox/internal/jsontypes"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -54,12 +54,12 @@ type RateLimit struct {
 }
 
 type Alerting struct {
-	Enable         bool               `yaml:"enable" form:"enable"`
+	Enable         bool                `yaml:"enable" form:"enable"`
-	LookbackDays   uint               `yaml:"lookbackDays" form:"lookbackDays"`
+	LookbackDays   uint                `yaml:"lookbackDays" form:"lookbackDays"`
-	HalfLife       jsontime.Duration  `yaml:"halfLife" form:"halfLife"`
+	HalfLife       jsontypes.Duration  `yaml:"halfLife" form:"halfLife"`
-	Recent         jsontime.Duration  `yaml:"recent" form:"recent"`
+	Recent         jsontypes.Duration  `yaml:"recent" form:"recent"`
-	AlertThreshold float64            `yaml:"alertThreshold" form:"alertThreshold"`
+	AlertThreshold float64             `yaml:"alertThreshold" form:"alertThreshold"`
-	Renotify       *jsontime.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
+	Renotify       *jsontypes.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
 }
 
 type Notify []NotifyService

pkg/database/models.go

@@ -7,6 +7,7 @@ package database
 import (
 	"time"
 
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/alerting/rules"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
@@ -82,18 +83,18 @@ type System struct {
 }
 
 type Talkgroup struct {
-	ID          int64            `json:"id"`
+	ID          int64              `json:"id"`
-	SystemID    int32            `json:"system_id"`
+	SystemID    int32              `json:"system_id"`
-	Tgid        int32            `json:"tgid"`
+	Tgid        int32              `json:"tgid"`
-	Name        *string          `json:"name"`
+	Name        *string            `json:"name"`
-	AlphaTag    *string          `json:"alpha_tag"`
+	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string          `json:"tg_group"`
+	TgGroup     *string            `json:"tg_group"`
-	Frequency   *int32           `json:"frequency"`
+	Frequency   *int32             `json:"frequency"`
-	Metadata    []byte           `json:"metadata"`
+	Metadata    jsontypes.Metadata `json:"metadata"`
-	Tags        []string         `json:"tags"`
+	Tags        []string           `json:"tags"`
-	Alert       bool             `json:"alert"`
+	Alert       bool               `json:"alert"`
-	AlertConfig rules.AlertRules `json:"alert_config"`
+	AlertConfig rules.AlertRules   `json:"alert_config"`
-	Weight      float32          `json:"weight"`
+	Weight      float32            `json:"weight"`
 }
 
 type TalkgroupsLearned struct {

pkg/database/talkgroups.sql.go

@@ -8,6 +8,7 @@ package database
 import (
 	"context"
 
+	"dynatron.me/x/stillbox/internal/jsontypes"
 	"dynatron.me/x/stillbox/pkg/alerting/rules"
 )
 
@@ -492,16 +493,16 @@ RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, t
 `
 
 type UpdateTalkgroupParams struct {
-	Name        *string          `json:"name"`
+	Name        *string            `json:"name"`
-	AlphaTag    *string          `json:"alpha_tag"`
+	AlphaTag    *string            `json:"alpha_tag"`
-	TgGroup     *string          `json:"tg_group"`
+	TgGroup     *string            `json:"tg_group"`
-	Frequency   *int32           `json:"frequency"`
+	Frequency   *int32             `json:"frequency"`
-	Metadata    []byte           `json:"metadata"`
+	Metadata    jsontypes.Metadata `json:"metadata"`
-	Tags        []string         `json:"tags"`
+	Tags        []string           `json:"tags"`
-	Alert       *bool            `json:"alert"`
+	Alert       *bool              `json:"alert"`
-	AlertConfig rules.AlertRules `json:"alert_config"`
+	AlertConfig rules.AlertRules   `json:"alert_config"`
-	Weight      *float32         `json:"weight"`
+	Weight      *float32           `json:"weight"`
-	ID          int64            `json:"id"`
+	ID          int64              `json:"id"`
 }
 
 func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) {

pkg/nexus/commands.go

@@ -2,7 +2,6 @@ package nexus
 
 import (
 	"context"
-	"encoding/json"
 
 	"dynatron.me/x/stillbox/pkg/calls"
 	"dynatron.me/x/stillbox/pkg/pb"
@@ -70,12 +69,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
 
 	var md *structpb.Struct
 	if len(tgi.Talkgroup.Metadata) > 0 {
-		m := make(map[string]interface{})
+		md, err = structpb.NewStruct(tgi.Talkgroup.Metadata)
-		err := json.Unmarshal(tgi.Talkgroup.Metadata, &m)
-		if err != nil {
-			log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("unmarshal tg metadata")
-		}
-		md, err = structpb.NewStruct(m)
 		if err != nil {
 			log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("new pb struct for tg metadata")
 		}

pkg/rest/api.go

@@ -42,7 +42,6 @@ type errResponse struct {
 
 func (e *errResponse) Render(w http.ResponseWriter, r *http.Request) error {
 	switch e.Code {
-	case http.StatusNotFound:
 	default:
 		log.Error().Str("path", r.URL.Path).Err(e.Err).Int("code", e.Code).Str("msg", e.Error).Msg("request failed")
 	}
@@ -79,8 +78,9 @@ func internalError(err error) render.Renderer {
 type errResponder func(error) render.Renderer
 
 var statusMapping = map[error]errResponder{
-	talkgroups.ErrNotFound: recordNotFound,
+	talkgroups.ErrNoSuchSystem: recordNotFound,
-	pgx.ErrNoRows:          recordNotFound,
+	talkgroups.ErrNotFound:     recordNotFound,
+	pgx.ErrNoRows:              recordNotFound,
 }
 
 func autoError(err error) render.Renderer {

pkg/rest/talkgroups.go

@@ -20,6 +20,7 @@ func (tga *talkgroupAPI) Subrouter() http.Handler {
 	r.Put("/{system:\\d+}/{id:\\d+}", tga.put)
 	r.Get("/{system:\\d+}/", tga.get)
 	r.Get("/", tga.get)
+	r.Post("/import", tga.tgImport)
 
 	return r
 }
@@ -106,3 +107,19 @@ func (tga *talkgroupAPI) put(w http.ResponseWriter, r *http.Request) {
 
 	respond(w, r, record)
 }
+
+func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) {
+	var impJob talkgroups.ImportJob
+	err := forms.Unmarshal(r, &impJob, forms.WithTag("json"), forms.WithAcceptBlank(), forms.WithOmitEmpty())
+	if err != nil {
+		wErr(w, r, badRequest(err))
+		return
+	}
+	recs, err := impJob.Import(r.Context())
+	if err != nil {
+		wErr(w, r, autoError(err))
+		return
+	}
+
+	respond(w, r, recs)
+}

pkg/talkgroups/import.go

@@ -0,0 +1,136 @@
+package talkgroups
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"dynatron.me/x/stillbox/internal/jsontypes"
+	"dynatron.me/x/stillbox/pkg/database"
+)
+
+type ImportSource string
+
+const (
+	ImportSrcRadioReference ImportSource = "radioreference"
+)
+
+var (
+	ErrBadImportType = errors.New("unknown import type")
+)
+
+type importer interface {
+	importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error)
+}
+
+type ImportJob struct {
+	Type     ImportSource `json:"type"`
+	SystemID int          `json:"systemID"`
+	Body     string       `json:"body"`
+
+	importer `json:"-"`
+}
+
+func (ij *ImportJob) Import(ctx context.Context) ([]Talkgroup, error) {
+	r := bytes.NewReader([]byte(ij.Body))
+
+	switch ij.Type {
+	case ImportSrcRadioReference:
+		ij.importer = &radioReferenceImporter{}
+	default:
+		return nil, ErrBadImportType
+	}
+	return ij.importTalkgroups(ctx, ij.SystemID, r)
+}
+
+type radioReferenceImporter struct {
+}
+
+type rrState int
+
+const (
+	rrsInitial rrState = iota
+	rrsGroupDesc
+	rrsTG
+)
+
+var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`)
+
+func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error) {
+	sc := bufio.NewScanner(r)
+	tgs := make([]Talkgroup, 0, 8)
+	sysn, has := StoreFrom(ctx).SystemName(ctx, sys)
+	if !has {
+		return nil, ErrNoSuchSystem
+	}
+
+	var groupName string
+	state := rrsInitial
+	for sc.Scan() {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+
+		ln := strings.Trim(sc.Text(), " \t\r\n")
+
+		switch state {
+		case rrsInitial:
+			groupName = ln
+			state++
+		case rrsGroupDesc:
+			if rrRE.MatchString(ln) {
+				state++
+			}
+		case rrsTG:
+			fields := strings.Split(ln, "\t")
+			if len(fields) < 6 {
+				state = rrsGroupDesc
+				groupName = ln
+				continue
+			}
+			tgid, err := strconv.Atoi(fields[0])
+			if err != nil {
+				continue
+			}
+			var metadata jsontypes.Metadata
+			tgt := TG(sys, tgid)
+			mode := fields[2]
+			if strings.Contains(mode, "E") {
+				metadata = jsontypes.Metadata{
+					"encrypted": true,
+				}
+			}
+			tags := []string{fields[5]}
+			gn := groupName // must take a copy
+			tgs = append(tgs, Talkgroup{
+				Talkgroup: database.Talkgroup{
+					ID:       tgt.Pack(),
+					Tgid:     int32(tgt.Talkgroup),
+					SystemID: int32(tgt.System),
+					Name:     &fields[4],
+					AlphaTag: &fields[3],
+					TgGroup:  &gn,
+					Metadata: metadata,
+					Tags:     tags,
+					Weight:   1.0,
+				},
+				System: database.System{
+					ID:   sys,
+					Name: sysn,
+				},
+			})
+
+		}
+	}
+
+	if err := sc.Err(); err != nil {
+		return tgs, err
+	}
+
+	return tgs, nil
+}

pkg/talkgroups/talkgroup.go

@@ -12,6 +12,8 @@ type Talkgroup struct {
 	Learned bool            `json:"learned"`
 }
 
+type Metadata map[string]interface{}
+
 type Names struct {
 	System    string
 	Talkgroup string

sql/sqlc.yaml

@@ -32,3 +32,8 @@ sql:
               import: "dynatron.me/x/stillbox/pkg/alerting/rules"
               type: "AlertRules"
             nullable: true
+          - column: "talkgroups.metadata"
+            go_type:
+              import: "dynatron.me/x/stillbox/internal/jsontypes"
+              type: "Metadata"
+            nullable: true