amigan/stillbox
1
0
Fork 0
Code Issues 17 Pull requests Projects Releases Packages Wiki Activity Actions

New tg ID schema and initial importer #35

Merged
amigan merged 17 commits from reid into trunk 2024-11-15 13:28:05 -05:00
Conversation 0 Commits 17 Files changed 44 +113 -96
15 changed files with 113 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit fb1b6a475c - Show all commits

4
internal/common/template.go
View file

@ -7,7 +7,7 @@ import (
"text/template"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/jsontypes"
)
var (
@ -27,7 +27,7 @@ var (
}
return dict, nil
},
"formTime": func(t jsontime.Time) string {
"formTime": func(t jsontypes.Time) string {
return time.Time(t).Format("2006-01-02T15:04")
},
"ago": func(s string) (string, error) {

6
internal/forms/forms.go
View file

@ -11,7 +11,7 @@ import (
"strings"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/jsontypes"
"github.com/araddon/dateparse"
)
@ -262,13 +262,13 @@ func (o *options) iterFields(r *http.Request, destStruct reflect.Value) error {
return err
}
setVal(destFieldVal, set, val)
case time.Time, *time.Time, jsontime.Time, *jsontime.Time:
case time.Time, *time.Time, jsontypes.Time, *jsontypes.Time:
t, set, err := o.parseTime(ff)
if err != nil {
return err
}
setVal(destFieldVal, set, t)
case time.Duration, *time.Duration, jsontime.Duration, *jsontime.Duration:
case time.Duration, *time.Duration, jsontypes.Duration, *jsontypes.Duration:
d, set, err := o.parseDuration(ff)
if err != nil {
return err

48
internal/forms/forms_test.go
View file

@ -10,7 +10,7 @@ import (
"dynatron.me/x/stillbox/internal/common"
"dynatron.me/x/stillbox/internal/forms"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/alerting"
"dynatron.me/x/stillbox/pkg/config"
@ -48,19 +48,19 @@ type urlEncTest struct {
}
type urlEncTestJT struct {
LookbackDays uint `json:"lookbackDays"`
HalfLife jsontime.Duration `json:"halfLife"`
Recent string `json:"recent"`
ScoreStart jsontime.Time `json:"scoreStart"`
ScoreEnd jsontime.Time `json:"scoreEnd"`
LookbackDays uint `json:"lookbackDays"`
HalfLife jsontypes.Duration `json:"halfLife"`
Recent string `json:"recent"`
ScoreStart jsontypes.Time `json:"scoreStart"`
ScoreEnd jsontypes.Time `json:"scoreEnd"`
}
type ptrTestJT struct {
LookbackDays uint `form:"lookbackDays"`
HalfLife *jsontime.Duration `form:"halfLife"`
Recent *string `form:"recent"`
ScoreStart *jsontime.Time `form:"scoreStart"`
ScoreEnd jsontime.Time `form:"scoreEnd"`
LookbackDays uint `form:"lookbackDays"`
HalfLife *jsontypes.Duration `form:"halfLife"`
Recent *string `form:"recent"`
ScoreStart *jsontypes.Time `form:"scoreStart"`
ScoreEnd jsontypes.Time `form:"scoreEnd"`
}
var (
@ -73,33 +73,33 @@ var (
UrlEncTestJT = urlEncTestJT{
LookbackDays: 7,
HalfLife: jsontime.Duration(30 * time.Minute),
HalfLife: jsontypes.Duration(30 * time.Minute),
Recent: "2h0m0s",
ScoreStart: jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
ScoreStart: jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC)),
}
PtrTestJT = ptrTestJT{
LookbackDays: 7,
HalfLife: common.PtrTo(jsontime.Duration(30 * time.Minute)),
HalfLife: common.PtrTo(jsontypes.Duration(30 * time.Minute)),
Recent: common.PtrTo("2h0m0s"),
ScoreStart: common.PtrTo(jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
ScoreStart: common.PtrTo(jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.UTC))),
}
UrlEncTestJTLocal = urlEncTestJT{
LookbackDays: 7,
HalfLife: jsontime.Duration(30 * time.Minute),
HalfLife: jsontypes.Duration(30 * time.Minute),
Recent: "2h0m0s",
ScoreStart: jsontime.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
ScoreStart: jsontypes.Time(time.Date(2024, time.October, 28, 9, 25, 0, 0, time.Local)),
}
realSim = &alerting.Simulation{
Alerting: config.Alerting{
LookbackDays: 7,
HalfLife: jsontime.Duration(30 * time.Minute),
Recent: jsontime.Duration(2 * time.Hour),
HalfLife: jsontypes.Duration(30 * time.Minute),
Recent: jsontypes.Duration(2 * time.Hour),
},
SimInterval: jsontime.Duration(5 * time.Minute),
ScoreStart: jsontime.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
SimInterval: jsontypes.Duration(5 * time.Minute),
ScoreStart: jsontypes.Time(time.Date(2024, time.October, 22, 17, 49, 0, 0, time.Local)),
}
Call1 = callUploadRequest{
@ -194,7 +194,7 @@ func TestUnmarshal(t *testing.T) {
opts: []forms.Option{forms.WithAcceptBlank()},
},
{
name: "url encoded jsontime",
name: "url encoded jsontypes",
r: makeRequest("urlenc.http"),
dest: &urlEncTestJT{},
expect: &UrlEncTestJT,
@ -202,14 +202,14 @@ func TestUnmarshal(t *testing.T) {
opts: []forms.Option{forms.WithTag("json")},
},
{
name: "url encoded jsontime with tz",
name: "url encoded jsontypes with tz",
r: makeRequest("urlenc.http"),
dest: &urlEncTestJT{},
expect: &UrlEncTestJT,
opts: []forms.Option{forms.WithAcceptBlank(), forms.WithParseTimeInTZ(time.UTC), forms.WithTag("json")},
},
{
name: "url encoded jsontime with local",
name: "url encoded jsontypes with local",
r: makeRequest("urlenc.http"),
dest: &urlEncTestJT{},
expect: &UrlEncTestJTLocal,

2
internal/jsontime/jsontime.go → internal/jsontypes/jsontime.go
View file

@ -1,4 +1,4 @@
package jsontime
package jsontypes
import (
"encoding/json"

3
internal/jsontypes/metadata.go Normal file
View file

@ -0,0 +1,3 @@
package jsontypes
type Metadata map[string]interface{}

10
pkg/alerting/simulate.go
View file

@ -9,7 +9,7 @@ import (
"time"
"dynatron.me/x/stillbox/internal/forms"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/internal/trending"
"dynatron.me/x/stillbox/pkg/config"
"dynatron.me/x/stillbox/pkg/talkgroups"
@ -23,12 +23,12 @@ type Simulation struct {
config.Alerting
// ScoreStart is the time when scoring begins
ScoreStart jsontime.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
ScoreStart jsontypes.Time `json:"scoreStart" yaml:"scoreStart" form:"scoreStart"`
// ScoreEnd is the time when the score simulator ends. Left blank, it defaults to time.Now()
ScoreEnd jsontime.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
ScoreEnd jsontypes.Time `json:"scoreEnd" yaml:"scoreEnd" form:"scoreEnd"`
// SimInterval is the interval at which the scorer will be called
SimInterval jsontime.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
SimInterval jsontypes.Duration `json:"simInterval" yaml:"simInterval" form:"simInterval"`
clock offsetClock `json:"-"`
*alerter `json:"-"`
@ -64,7 +64,7 @@ func (s *Simulation) Simulate(ctx context.Context) (trending.Scores[talkgroups.I
s.Enable = true
s.alerter = New(s.Alerting, tgc, WithClock(&s.clock)).(*alerter)
if time.Time(s.ScoreEnd).IsZero() {
s.ScoreEnd = jsontime.Time(now)
s.ScoreEnd = jsontypes.Time(now)
}
log.Debug().Time("scoreStart", s.ScoreStart.Time()).
Time("scoreEnd", s.ScoreEnd.Time()).

14
pkg/config/config.go
View file

@ -5,7 +5,7 @@ import (
"sync"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/jsontypes"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
@ -54,12 +54,12 @@ type RateLimit struct {
}
type Alerting struct {
Enable bool `yaml:"enable" form:"enable"`
LookbackDays uint `yaml:"lookbackDays" form:"lookbackDays"`
HalfLife jsontime.Duration `yaml:"halfLife" form:"halfLife"`
Recent jsontime.Duration `yaml:"recent" form:"recent"`
AlertThreshold float64 `yaml:"alertThreshold" form:"alertThreshold"`
Renotify *jsontime.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
Enable bool `yaml:"enable" form:"enable"`
LookbackDays uint `yaml:"lookbackDays" form:"lookbackDays"`
HalfLife jsontypes.Duration `yaml:"halfLife" form:"halfLife"`
Recent jsontypes.Duration `yaml:"recent" form:"recent"`
AlertThreshold float64 `yaml:"alertThreshold" form:"alertThreshold"`
Renotify *jsontypes.Duration `yaml:"renotify,omitempty" form:"renotify,omitempty"`
}
type Notify []NotifyService

25
pkg/database/models.go
View file

@ -7,6 +7,7 @@ package database
import (
"time"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/alerting/rules"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
@ -82,18 +83,18 @@ type System struct {
}
type Talkgroup struct {
ID int64 `json:"id"`
SystemID int32 `json:"system_id"`
Tgid int32 `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata []byte `json:"metadata"`
Tags []string `json:"tags"`
Alert bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight float32 `json:"weight"`
ID int64 `json:"id"`
SystemID int32 `json:"system_id"`
Tgid int32 `json:"tgid"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata jsontypes.Metadata `json:"metadata"`
Tags []string `json:"tags"`
Alert bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight float32 `json:"weight"`
}
type TalkgroupsLearned struct {

21
pkg/database/talkgroups.sql.go
View file

@ -8,6 +8,7 @@ package database
import (
"context"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/alerting/rules"
)
@ -492,16 +493,16 @@ RETURNING id, system_id, tgid, name, alpha_tag, tg_group, frequency, metadata, t
`
type UpdateTalkgroupParams struct {
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata []byte `json:"metadata"`
Tags []string `json:"tags"`
Alert *bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight *float32 `json:"weight"`
ID int64 `json:"id"`
Name *string `json:"name"`
AlphaTag *string `json:"alpha_tag"`
TgGroup *string `json:"tg_group"`
Frequency *int32 `json:"frequency"`
Metadata jsontypes.Metadata `json:"metadata"`
Tags []string `json:"tags"`
Alert *bool `json:"alert"`
AlertConfig rules.AlertRules `json:"alert_config"`
Weight *float32 `json:"weight"`
ID int64 `json:"id"`
}
func (q *Queries) UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) {

8
pkg/nexus/commands.go
View file

@ -2,7 +2,6 @@ package nexus
import (
"context"
"encoding/json"
"dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/pb"
@ -70,12 +69,7 @@ func (c *client) Talkgroup(ctx context.Context, tg *pb.Talkgroup) error {
var md *structpb.Struct
if len(tgi.Talkgroup.Metadata) > 0 {
m := make(map[string]interface{})
err := json.Unmarshal(tgi.Talkgroup.Metadata, &m)
if err != nil {
log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("unmarshal tg metadata")
}
md, err = structpb.NewStruct(m)
md, err = structpb.NewStruct(tgi.Talkgroup.Metadata)
if err != nil {
log.Error().Err(err).Int32("sys", tg.System).Int32("tg", tg.Talkgroup).Msg("new pb struct for tg metadata")
}

6
pkg/rest/api.go
View file

@ -42,7 +42,6 @@ type errResponse struct {
func (e *errResponse) Render(w http.ResponseWriter, r *http.Request) error {
switch e.Code {
case http.StatusNotFound:
default:
log.Error().Str("path", r.URL.Path).Err(e.Err).Int("code", e.Code).Str("msg", e.Error).Msg("request failed")
}
@ -79,8 +78,9 @@ func internalError(err error) render.Renderer {
type errResponder func(error) render.Renderer
var statusMapping = map[error]errResponder{
talkgroups.ErrNotFound: recordNotFound,
pgx.ErrNoRows: recordNotFound,
talkgroups.ErrNoSuchSystem: recordNotFound,
talkgroups.ErrNotFound: recordNotFound,
pgx.ErrNoRows: recordNotFound,
}
func autoError(err error) render.Renderer {

4
pkg/rest/talkgroups.go
View file

@ -20,7 +20,7 @@ func (tga *talkgroupAPI) Subrouter() http.Handler {
r.Put("/{system:\\d+}/{id:\\d+}", tga.put)
r.Get("/{system:\\d+}/", tga.get)
r.Get("/", tga.get)
r.Put("/import", tga.tgImport)
r.Post("/import", tga.tgImport)
return r
}
@ -115,7 +115,7 @@ func (tga *talkgroupAPI) tgImport(w http.ResponseWriter, r *http.Request) {
wErr(w, r, badRequest(err))
return
}
recs, err := impJob.Import()
recs, err := impJob.Import(r.Context())
if err != nil {
wErr(w, r, autoError(err))
return

51
pkg/talkgroups/import.go
View file

@ -3,13 +3,14 @@ package talkgroups
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"encoding/json"
"regexp"
"strconv"
"strings"
"dynatron.me/x/stillbox/internal/jsontypes"
"dynatron.me/x/stillbox/pkg/database"
)
@ -24,18 +25,18 @@ var (
)
type importer interface {
importTalkgroups(sys int, r io.Reader) ([]Talkgroup, error)
importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error)
}
type ImportJob struct {
Type ImportSource `json:"type"`
SystemID int `json:"systemID"`
Body string `json:"body"`
Type ImportSource `json:"type"`
SystemID int `json:"systemID"`
Body string `json:"body"`
importer `json:"-"`
}
func (ij *ImportJob) Import() ([]Talkgroup, error) {
func (ij *ImportJob) Import(ctx context.Context) ([]Talkgroup, error) {
r := bytes.NewReader([]byte(ij.Body))
switch ij.Type {
@ -44,13 +45,14 @@ func (ij *ImportJob) Import() ([]Talkgroup, error) {
default:
return nil, ErrBadImportType
}
return ij.importTalkgroups(ij.SystemID, r)
return ij.importTalkgroups(ctx, ij.SystemID, r)
}
type radioReferenceImporter struct {
}
type rrState int
const (
rrsInitial rrState = iota
rrsGroupDesc
@ -59,13 +61,21 @@ const (
var rrRE = regexp.MustCompile(`DEC\s+HEX\s+Mode\s+Alpha Tag\s+Description\s+Tag`)
func (rr *radioReferenceImporter) importTalkgroups(sys int, r io.Reader) ([]Talkgroup, error) {
func (rr *radioReferenceImporter) importTalkgroups(ctx context.Context, sys int, r io.Reader) ([]Talkgroup, error) {
sc := bufio.NewScanner(r)
tgs := make([]Talkgroup, 0, 8)
sysn, has := StoreFrom(ctx).SystemName(ctx, sys)
if !has {
return nil, ErrNoSuchSystem
}
var groupName string
state := rrsInitial
for sc.Scan() {
if err := ctx.Err(); err != nil {
return nil, err
}
ln := strings.Trim(sc.Text(), " \t\r\n")
switch state {
@ -87,30 +97,31 @@ func (rr *radioReferenceImporter) importTalkgroups(sys int, r io.Reader) ([]Talk
if err != nil {
continue
}
var metadata []byte
var metadata jsontypes.Metadata
tgt := TG(sys, tgid)
mode := fields[2]
if strings.Contains(mode, "E") {
metadata, _ = json.Marshal(&struct{
Encrypted bool `json:"encrypted"`
}{true})
metadata = jsontypes.Metadata{
"encrypted": true,
}
}
tags := []string{fields[5]}
gn := groupName // must take a copy
tgs = append(tgs, Talkgroup{
Talkgroup: database.Talkgroup{
ID: tgt.Pack(),
Tgid: int32(tgt.Talkgroup),
ID: tgt.Pack(),
Tgid: int32(tgt.Talkgroup),
SystemID: int32(tgt.System),
Name: &fields[4],
Name: &fields[4],
AlphaTag: &fields[3],
TgGroup: &groupName,
TgGroup: &gn,
Metadata: metadata,
Tags: tags,
Weight: 1.0,
Tags: tags,
Weight: 1.0,
},
System: database.System{
ID: sys,
Name: "<imported>",
ID: sys,
Name: sysn,
},
})

2
pkg/talkgroups/talkgroup.go
View file

@ -12,6 +12,8 @@ type Talkgroup struct {
Learned bool `json:"learned"`
}
type Metadata map[string]interface{}
type Names struct {
System string
Talkgroup string

5
sql/sqlc.yaml
View file

@ -32,3 +32,8 @@ sql:
import: "dynatron.me/x/stillbox/pkg/alerting/rules"
type: "AlertRules"
nullable: true
- column: "talkgroups.metadata"
go_type:
import: "dynatron.me/x/stillbox/internal/jsontypes"
type: "Metadata"
nullable: true