From fb1b6a475c7442113a754fb7f087abe680430582 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Wed, 13 Nov 2024 09:24:11 -0500 Subject: [PATCH] Importer, rename jsontime to jsontypes --- internal/common/template.go | 4 +- internal/forms/forms.go | 6 +-- internal/forms/forms_test.go | 48 +++++++++--------- internal/{jsontime => jsontypes}/jsontime.go | 2 +- internal/jsontypes/metadata.go | 3 ++ pkg/alerting/simulate.go | 10 ++-- pkg/config/config.go | 14 +++--- pkg/database/models.go | 25 +++++----- pkg/database/talkgroups.sql.go | 21 ++++---- pkg/nexus/commands.go | 8 +-- pkg/rest/api.go | 6 +-- pkg/rest/talkgroups.go | 4 +- pkg/talkgroups/import.go | 51 ++++++++++++-------- pkg/talkgroups/talkgroup.go | 2 + sql/sqlc.yaml | 5 ++ 15 files changed, 113 insertions(+), 96 deletions(-) rename internal/{jsontime => jsontypes}/jsontime.go (99%) create mode 100644 internal/jsontypes/metadata.go diff --git a/internal/common/template.go b/internal/common/template.go index f4724c1..8357260 100644 --- a/internal/common/template.go +++ b/internal/common/template.go @@ -7,7 +7,7 @@ import ( "text/template" "time" - "dynatron.me/x/stillbox/internal/jsontime" + "dynatron.me/x/stillbox/internal/jsontypes" ) var ( @@ -27,7 +27,7 @@ var ( } return dict, nil }, - "formTime": func(t jsontime.Time) string { + "formTime": func(t jsontypes.Time) string { return time.Time(t).Format("2006-01-02T15:04") }, "ago": func(s string) (string, error) { diff --git a/internal/forms/forms.go b/internal/forms/forms.go index bc28cc4..f8a7f7a 100644 --- a/internal/forms/forms.go +++ b/internal/forms/forms.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "dynatron.me/x/stillbox/internal/jsontime" + "dynatron.me/x/stillbox/internal/jsontypes" "github.com/araddon/dateparse" ) @@ -262,13 +262,13 @@ func (o *options) iterFields(r *http.Request, destStruct reflect.Value) error { return err } setVal(destFieldVal, set, val) - case time.Time, *time.Time, jsontime.Time, *jsontime.Time: + case time.Time, *time.Time, jsontypes.Time, *jsontypes.Time: t, set, err := o.parseTime(ff) if err != nil { return err } setVal(destFieldVal, set, t) - case time.Duration, *time.Duration, jsontime.Duration, *jsontime.Duration: + case time.Duration, *time.Duration, jsontypes.Duration, *jsontypes.Duration: d, set, err := o.parseDuration(ff) if err != nil { return err diff --git a/internal/forms/forms_test.go b/internal/forms/forms_test.go index 5f73dca..c7108e8 100644 --- a/internal/forms/forms_test.go +++ b/internal/forms/forms_test.go @@ -10,7 +10,7 @@ import ( "dynatron.me/x/stillbox/internal/common" "dynatron.me/x/stillbox/internal/forms" - "dynatron.me/x/stillbox/internal/jsontime" + "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/alerting" "dynatron.me/x/stillbox/pkg/config" @@ -48,19 +48,19 @@ type urlEncTest struct { } type urlEncTestJT struct { - LookbackDays uint `json:"lookbackDays"` - HalfLife jsontime.Duration `json:"halfLife"` - Recent string `json:"recent"` - ScoreStart jsontime.Time `json:"scoreStart"` - ScoreEnd jsontime.Time `json:"scoreEnd"` + LookbackDays uint `json:"lookbackDays"` + HalfLife jsontypes.Duration `json:"halfLife"` + Recent string `json:"recent"` + ScoreStart jsontypes.Time `json:"scoreStart"` + ScoreEnd jsontypes.Time `json:"scoreEnd"` } type ptrTestJT struct { - LookbackDays uint `form:"lookbackDays"` - HalfLife *jsontime.Duration `form:"halfLife"` - Recent *string `form:"recent"` - ScoreStart *jsontime.Time `form:"scoreStart"` - ScoreEnd jsontime.Time `form:"scoreEnd"` + LookbackDays uint `form:"lookbackDays"` + HalfLife *jsontypes.Duration `form:"halfLife"` + Recent *string `form:"recent"` + ScoreStart *jsontypes.Time `form:"scoreStart"` + ScoreEnd jsontypes.Time `form:"scoreEnd"` } var ( @@ -73,33 +73,33 @@ var ( UrlEncTestJT = urlEncTestJT{ LookbackDays: 7, - HalfLife: jsontime.Duration(30 * time.Minute), + HalfLife: jsontypes.Duration(30 * time.Minute), Recent: "2h0m0s", - ScoreStart: jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)), + ScoreStart: jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)), } PtrTestJT = ptrTestJT{ LookbackDays: 7, - HalfLife: common.PtrTo(jsontime.Duration(30 * time.Minute)), + HalfLife: common.PtrTo(jsontypes.Duration(30 * time.Minute)), Recent: common.PtrTo("2h0m0s"), - ScoreStart: common.PtrTo(jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))), + ScoreStart: common.PtrTo(jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))), } UrlEncTestJTLocal = urlEncTestJT{ LookbackDays: 7, - HalfLife: jsontime.Duration(30 * time.Minute), + HalfLife: jsontypes.Duration(30 * time.Minute), Recent: "2h0m0s", - ScoreStart: jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)), + ScoreStart: jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)), } realSim = &alerting.Simulation{ Alerting: config.Alerting{ LookbackDays: 7, - HalfLife: jsontime.Duration(30 * time.Minute), - Recent: jsontime.Duration(2 * time.Hour), + HalfLife: jsontypes.Duration(30 * time.Minute), + Recent: jsontypes.Duration(2 * time.Hour), }, - SimInterval: jsontime.Duration(5 * time.Minute), - ScoreStart: jsontime.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)), + SimInterval: jsontypes.Duration(5 * time.Minute), + ScoreStart: jsontypes.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)), } Call1 = callUploadRequest{ @@ -194,7 +194,7 @@ func TestUnmarshal(t *testing.T) { opts: []forms.Option{forms.WithAcceptBlank()}, }, { - name: "url encoded jsontime", + name: "url encoded jsontypes", r: makeRequest("urlenc.http"), dest: &urlEncTestJT{}, expect: &UrlEncTestJT, @@ -202,14 +202,14 @@ func TestUnmarshal(t *testing.T) { opts: []forms.Option{forms.WithTag("json")}, }, { - name: "url encoded jsontime with tz", + name: "url encoded jsontypes with tz", r: makeRequest("urlenc.http"), dest: &urlEncTestJT{}, expect: &UrlEncTestJT, opts: []forms.Option{forms.WithAcceptBlank(), forms.WithParseTimeInTZ(time.UTC), forms.WithTag("json")}, }, { - name: "url encoded jsontime with local", + name: "url encoded jsontypes with local", r: makeRequest("urlenc.http"), dest: &urlEncTestJT{}, expect: &UrlEncTestJTLocal, diff --git a/internal/jsontime/jsontime.go b/internal/jsontypes/jsontime.go similarity index 99% rename from internal/jsontime/jsontime.go rename to internal/jsontypes/jsontime.go index 9eb9425..03880e0 100644 --- a/internal/jsontime/jsontime.go +++ b/internal/jsontypes/jsontime.go @@ -1,4 +1,4 @@ -package jsontime +package jsontypes import ( "encoding/json" diff --git a/internal/jsontypes/metadata.go b/internal/jsontypes/metadata.go new file mode 100644 index 0000000..9bda0f6 --- /dev/null +++ b/internal/jsontypes/metadata.go @@ -0,0 +1,3 @@ +package jsontypes + +type Metadata map[string]interface{} diff --git a/pkg/alerting/simulate.go b/pkg/alerting/simulate.go index 0ebefb2..bfe69b7 100644 --- a/pkg/alerting/simulate.go +++ b/pkg/alerting/simulate.go @@ -9,7 +9,7 @@ import ( "time" "dynatron.me/x/stillbox/internal/forms" - "dynatron.me/x/stillbox/internal/jsontime" + "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/internal/trending" "dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/talkgroups" @@ -23,12 +23,12 @@ type Simulation struct { config.Alerting // ScoreStart is the time when scoring begins - ScoreStart jsontime.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"` + ScoreStart jsontypes.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"` // ScoreEnd is the time when the score simulator ends. Left blank, it defaults to time.Now() - ScoreEnd jsontime.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"` + ScoreEnd jsontypes.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"` // SimInterval is the interval at which the scorer will be called - SimInterval jsontime.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"` + SimInterval jsontypes.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"` clock offsetClock `json:"-"` *alerter `json:"-"` @@ -64,7 +64,7 @@ func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.I s.Enable = true s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter) if time.Time(s.ScoreEnd).IsZero() { - s.ScoreEnd = jsontime.Time(now) + s.ScoreEnd = jsontypes.Time(now) } log.Debug().Time("scoreStart", s.ScoreStart.Time()). Time("scoreEnd", s.ScoreEnd.Time()). diff --git a/pkg/config/config.go b/pkg/config/config.go index 6a848f9..47e288a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "dynatron.me/x/stillbox/internal/jsontime" + "dynatron.me/x/stillbox/internal/jsontypes" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -54,12 +54,12 @@ type RateLimit struct { } type Alerting struct { - Enable bool `yaml:"enable" form:"enable"` - LookbackDays uint `yaml:"lookbackDays" form:"lookbackDays"` - HalfLife jsontime.Duration `yaml:"halfLife" form:"halfLife"` - Recent jsontime.Duration `yaml:"recent" form:"recent"` - AlertThreshold float64 `yaml:"alertThreshold" form:"alertThreshold"` - Renotify *jsontime.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"` + Enable bool `yaml:"enable" form:"enable"` + LookbackDays uint `yaml:"lookbackDays" form:"lookbackDays"` + HalfLife jsontypes.Duration `yaml:"halfLife" form:"halfLife"` + Recent jsontypes.Duration `yaml:"recent" form:"recent"` + AlertThreshold float64 `yaml:"alertThreshold" form:"alertThreshold"` + Renotify *jsontypes.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"` } type Notify []NotifyService diff --git a/pkg/database/models.go b/pkg/database/models.go index 4d74489..9da62d6 100644 --- a/pkg/database/models.go +++ b/pkg/database/models.go @@ -7,6 +7,7 @@ package database import ( "time" + "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/alerting/rules" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -82,18 +83,18 @@ type System struct { } type Talkgroup struct { - ID int64 `json:"id"` - SystemID int32 `json:"system_id"` - Tgid int32 `json:"tgid"` - Name *string `json:"name"` - AlphaTag *string `json:"alpha_tag"` - TgGroup *string `json:"tg_group"` - Frequency *int32 `json:"frequency"` - Metadata []byte `json:"metadata"` - Tags []string `json:"tags"` - Alert bool `json:"alert"` - AlertConfig rules.AlertRules `json:"alert_config"` - Weight float32 `json:"weight"` + ID int64 `json:"id"` + SystemID int32 `json:"system_id"` + Tgid int32 `json:"tgid"` + Name *string `json:"name"` + AlphaTag *string `json:"alpha_tag"` + TgGroup *string `json:"tg_group"` + Frequency *int32 `json:"frequency"` + Metadata jsontypes.Metadata `json:"metadata"` + Tags []string `json:"tags"` + Alert bool `json:"alert"` + AlertConfig rules.AlertRules `json:"alert_config"` + Weight float32 `json:"weight"` } type TalkgroupsLearned struct { diff --git a/pkg/database/talkgroups.sql.go b/pkg/database/talkgroups.sql.go index d511a2b..732b26e 100644 --- a/pkg/database/talkgroups.sql.go +++ b/pkg/database/talkgroups.sql.go @@ -8,6 +8,7 @@ package database import ( "context" + "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/alerting/rules" ) @@ -492,16 +493,16 @@ RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, t ` type UpdateTalkgroupParams struct { - Name *string `json:"name"` - AlphaTag *string `json:"alpha_tag"` - TgGroup *string `json:"tg_group"` - Frequency *int32 `json:"frequency"` - Metadata []byte `json:"metadata"` - Tags []string `json:"tags"` - Alert *bool `json:"alert"` - AlertConfig rules.AlertRules `json:"alert_config"` - Weight *float32 `json:"weight"` - ID int64 `json:"id"` + Name *string `json:"name"` + AlphaTag *string `json:"alpha_tag"` + TgGroup *string `json:"tg_group"` + Frequency *int32 `json:"frequency"` + Metadata jsontypes.Metadata `json:"metadata"` + Tags []string `json:"tags"` + Alert *bool `json:"alert"` + AlertConfig rules.AlertRules `json:"alert_config"` + Weight *float32 `json:"weight"` + ID int64 `json:"id"` } func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) { diff --git a/pkg/nexus/commands.go b/pkg/nexus/commands.go index a14a9ba..502859a 100644 --- a/pkg/nexus/commands.go +++ b/pkg/nexus/commands.go @@ -2,7 +2,6 @@ package nexus import ( "context" - "encoding/json" "dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/pb" @@ -70,12 +69,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error { var md *structpb.Struct if len(tgi.Talkgroup.Metadata) > 0 { - m := make(map[string]interface{}) - err := json.Unmarshal(tgi.Talkgroup.Metadata, &m) - if err != nil { - log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("unmarshal tg metadata") - } - md, err = structpb.NewStruct(m) + md, err = structpb.NewStruct(tgi.Talkgroup.Metadata) if err != nil { log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("new pb struct for tg metadata") } diff --git a/pkg/rest/api.go b/pkg/rest/api.go index 86eab07..339ed36 100644 --- a/pkg/rest/api.go +++ b/pkg/rest/api.go @@ -42,7 +42,6 @@ type errResponse struct { func (e *errResponse) Render(w http.ResponseWriter, r *http.Request) error { switch e.Code { - case http.StatusNotFound: default: log.Error().Str("path", r.URL.Path).Err(e.Err).Int("code", e.Code).Str("msg", e.Error).Msg("request failed") } @@ -79,8 +78,9 @@ func internalError(err error) render.Renderer { type errResponder func(error) render.Renderer var statusMapping = map[error]errResponder{ - talkgroups.ErrNotFound: recordNotFound, - pgx.ErrNoRows: recordNotFound, + talkgroups.ErrNoSuchSystem: recordNotFound, + talkgroups.ErrNotFound: recordNotFound, + pgx.ErrNoRows: recordNotFound, } func autoError(err error) render.Renderer { diff --git a/pkg/rest/talkgroups.go b/pkg/rest/talkgroups.go index c8f7786..7185148 100644 --- a/pkg/rest/talkgroups.go +++ b/pkg/rest/talkgroups.go @@ -20,7 +20,7 @@ func (tga *talkgroupAPI) Subrouter() http.Handler { r.Put("/{system:\\d+}/{id:\\d+}", tga.put) r.Get("/{system:\\d+}/", tga.get) r.Get("/", tga.get) - r.Put("/import", tga.tgImport) + r.Post("/import", tga.tgImport) return r } @@ -115,7 +115,7 @@ func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) { wErr(w, r, badRequest(err)) return } - recs, err := impJob.Import() + recs, err := impJob.Import(r.Context()) if err != nil { wErr(w, r, autoError(err)) return diff --git a/pkg/talkgroups/import.go b/pkg/talkgroups/import.go index 6fdf041..9a59984 100644 --- a/pkg/talkgroups/import.go +++ b/pkg/talkgroups/import.go @@ -3,13 +3,14 @@ package talkgroups import ( "bufio" "bytes" + "context" "errors" "io" - "encoding/json" "regexp" "strconv" "strings" + "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/database" ) @@ -24,18 +25,18 @@ var ( ) type importer interface { - importTalkgroups(sys int, r io.Reader) ([]Talkgroup, error) + importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error) } type ImportJob struct { - Type ImportSource `json:"type"` - SystemID int `json:"systemID"` - Body string `json:"body"` + Type ImportSource `json:"type"` + SystemID int `json:"systemID"` + Body string `json:"body"` importer `json:"-"` } -func (ij *ImportJob) Import() ([]Talkgroup, error) { +func (ij *ImportJob) Import(ctx context.Context) ([]Talkgroup, error) { r := bytes.NewReader([]byte(ij.Body)) switch ij.Type { @@ -44,13 +45,14 @@ func (ij *ImportJob) Import() ([]Talkgroup, error) { default: return nil, ErrBadImportType } - return ij.importTalkgroups(ij.SystemID, r) + return ij.importTalkgroups(ctx, ij.SystemID, r) } type radioReferenceImporter struct { } type rrState int + const ( rrsInitial rrState = iota rrsGroupDesc @@ -59,13 +61,21 @@ const ( var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`) -func (rr *radioReferenceImporter) importTalkgroups(sys int, r io.Reader) ([]Talkgroup, error) { +func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error) { sc := bufio.NewScanner(r) tgs := make([]Talkgroup, 0, 8) + sysn, has := StoreFrom(ctx).SystemName(ctx, sys) + if !has { + return nil, ErrNoSuchSystem + } var groupName string state := rrsInitial for sc.Scan() { + if err := ctx.Err(); err != nil { + return nil, err + } + ln := strings.Trim(sc.Text(), " \t\r\n") switch state { @@ -87,30 +97,31 @@ func (rr *radioReferenceImporter) importTalkgroups(sys int, r io.Reader) ([]Talk if err != nil { continue } - var metadata []byte + var metadata jsontypes.Metadata tgt := TG(sys, tgid) mode := fields[2] if strings.Contains(mode, "E") { - metadata, _ = json.Marshal(&struct{ - Encrypted bool `json:"encrypted"` - }{true}) + metadata = jsontypes.Metadata{ + "encrypted": true, + } } tags := []string{fields[5]} + gn := groupName // must take a copy tgs = append(tgs, Talkgroup{ Talkgroup: database.Talkgroup{ - ID: tgt.Pack(), - Tgid: int32(tgt.Talkgroup), + ID: tgt.Pack(), + Tgid: int32(tgt.Talkgroup), SystemID: int32(tgt.System), - Name: &fields[4], + Name: &fields[4], AlphaTag: &fields[3], - TgGroup: &groupName, + TgGroup: &gn, Metadata: metadata, - Tags: tags, - Weight: 1.0, + Tags: tags, + Weight: 1.0, }, System: database.System{ - ID: sys, - Name: "", + ID: sys, + Name: sysn, }, }) diff --git a/pkg/talkgroups/talkgroup.go b/pkg/talkgroups/talkgroup.go index e3cca71..b9b746b 100644 --- a/pkg/talkgroups/talkgroup.go +++ b/pkg/talkgroups/talkgroup.go @@ -12,6 +12,8 @@ type Talkgroup struct { Learned bool `json:"learned"` } +type Metadata map[string]interface{} + type Names struct { System string Talkgroup string diff --git a/sql/sqlc.yaml b/sql/sqlc.yaml index 6b617e4..342e2fb 100644 --- a/sql/sqlc.yaml +++ b/sql/sqlc.yaml @@ -32,3 +32,8 @@ sql: import: "dynatron.me/x/stillbox/pkg/alerting/rules" type: "AlertRules" nullable: true + - column: "talkgroups.metadata" + go_type: + import: "dynatron.me/x/stillbox/internal/jsontypes" + type: "Metadata" + nullable: true