From a25aaccbf3490ade2041611adddae3e3aacc4a97 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 27 Jul 2024 19:25:16 -0400 Subject: [PATCH] calls trigger --- internal/common/common.go | 13 ++++++ pkg/gordio/admin/admin.go | 3 +- pkg/gordio/database/calls.sql.go | 20 ++++++--- pkg/gordio/database/models.go | 22 +++++++--- pkg/gordio/database/users.sql.go | 4 +- pkg/gordio/server/calls.go | 47 +++++++++++---------- sql/postgres/migrations/001_initial.up.sql | 49 ++++++++++++++++++---- sql/postgres/queries/calls.sql | 4 +- sql/sqlc.yaml | 5 ++- 9 files changed, 120 insertions(+), 47 deletions(-) diff --git a/internal/common/common.go b/internal/common/common.go index abd926d..fffb96d 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -25,3 +25,16 @@ func RunE(c cmdOptions) func(cmd *cobra.Command, args []string) error { return err } } + +func PtrTo[T any](t T) *T { + return &t +} + +func PtrOrNull[T comparable](val T) *T { + var zero T + if val == zero { + return nil + } + + return &val +} diff --git a/pkg/gordio/admin/admin.go b/pkg/gordio/admin/admin.go index 2eabafc..25d65ba 100644 --- a/pkg/gordio/admin/admin.go +++ b/pkg/gordio/admin/admin.go @@ -8,7 +8,6 @@ import ( "dynatron.me/x/stillbox/pkg/gordio/config" "dynatron.me/x/stillbox/pkg/gordio/database" - "github.com/jackc/pgx/v5/pgtype" "github.com/spf13/cobra" "golang.org/x/crypto/bcrypt" "golang.org/x/term" @@ -55,7 +54,7 @@ func AddUser(ctx context.Context, username, email string, isAdmin bool) error { Username: username, Password: string(hashpw), Email: email, - IsAdmin: pgtype.Bool{Bool: isAdmin, Valid: true}, + IsAdmin: isAdmin, }) return err diff --git a/pkg/gordio/database/calls.sql.go b/pkg/gordio/database/calls.sql.go index ad14efe..af4a68b 100644 --- a/pkg/gordio/database/calls.sql.go +++ b/pkg/gordio/database/calls.sql.go @@ -27,9 +27,11 @@ INSERT INTO calls ( frequencies, patches, tg_label, + tg_tag, + tg_group, source - ) VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) -RETURNING id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_type, audio_url, frequency, frequencies, patches, tg_label, source, transcript + ) VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) +RETURNING id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_tag, tg_group, source, transcript ` type AddCallParams struct { @@ -41,11 +43,13 @@ type AddCallParams struct { AudioBlob []byte `json:"audio_blob"` AudioType *string `json:"audio_type"` AudioUrl *string `json:"audio_url"` - Frequency *int32 `json:"frequency"` - Frequencies []byte `json:"frequencies"` - Patches []byte `json:"patches"` + Frequency int `json:"frequency"` + Frequencies []int `json:"frequencies"` + Patches []int `json:"patches"` TgLabel *string `json:"tg_label"` - Source *string `json:"source"` + TgTag *string `json:"tg_tag"` + TgGroup *string `json:"tg_group"` + Source int `json:"source"` } func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) (Call, error) { @@ -62,6 +66,8 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) (Call, error) arg.Frequencies, arg.Patches, arg.TgLabel, + arg.TgTag, + arg.TgGroup, arg.Source, ) var i Call @@ -79,6 +85,8 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) (Call, error) &i.Frequencies, &i.Patches, &i.TgLabel, + &i.TgTag, + &i.TgGroup, &i.Source, &i.Transcript, ) diff --git a/pkg/gordio/database/models.go b/pkg/gordio/database/models.go index c943d21..6f7ea93 100644 --- a/pkg/gordio/database/models.go +++ b/pkg/gordio/database/models.go @@ -13,7 +13,7 @@ import ( type ApiKey struct { ID int32 `json:"id"` - Owner *int32 `json:"owner"` + Owner int `json:"owner"` CreatedAt time.Time `json:"created_at"` Expires pgtype.Timestamp `json:"expires"` Disabled *bool `json:"disabled"` @@ -30,11 +30,13 @@ type Call struct { AudioBlob []byte `json:"audio_blob"` AudioType *string `json:"audio_type"` AudioUrl *string `json:"audio_url"` - Frequency *int32 `json:"frequency"` - Frequencies []byte `json:"frequencies"` - Patches []byte `json:"patches"` + Frequency int `json:"frequency"` + Frequencies []int `json:"frequencies"` + Patches []int `json:"patches"` TgLabel *string `json:"tg_label"` - Source *string `json:"source"` + TgTag *string `json:"tg_tag"` + TgGroup *string `json:"tg_group"` + Source int `json:"source"` Transcript *string `json:"transcript"` } @@ -74,6 +76,14 @@ type Talkgroup struct { Metadata []byte `json:"metadata"` } +type TalkgroupsLearned struct { + ID int32 `json:"id"` + SystemID int `json:"system_id"` + Tgid int `json:"tgid"` + GroupName string `json:"group_name"` + GroupTag *string `json:"group_tag"` +} + type TalkgroupsTag struct { SystemID int `json:"system_id"` TalkgroupID int `json:"talkgroup_id"` @@ -85,6 +95,6 @@ type User struct { Username string `json:"username"` Password string `json:"password"` Email string `json:"email"` - IsAdmin *bool `json:"is_admin"` + IsAdmin bool `json:"is_admin"` Prefs []byte `json:"prefs"` } diff --git a/pkg/gordio/database/users.sql.go b/pkg/gordio/database/users.sql.go index b135a3d..7cd45a6 100644 --- a/pkg/gordio/database/users.sql.go +++ b/pkg/gordio/database/users.sql.go @@ -23,7 +23,7 @@ INSERT INTO api_keys( RETURNING id, owner, created_at, expires, disabled, api_key ` -func (q *Queries) CreateAPIKey(ctx context.Context, owner *int32, expires pgtype.Timestamp, disabled *bool) (ApiKey, error) { +func (q *Queries) CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error) { row := q.db.QueryRow(ctx, createAPIKey, owner, expires, disabled) var i ApiKey err := row.Scan( @@ -51,7 +51,7 @@ type CreateUserParams struct { Username string `json:"username"` Password string `json:"password"` Email string `json:"email"` - IsAdmin *bool `json:"is_admin"` + IsAdmin bool `json:"is_admin"` } func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) { diff --git a/pkg/gordio/server/calls.go b/pkg/gordio/server/calls.go index 4cc6065..7743904 100644 --- a/pkg/gordio/server/calls.go +++ b/pkg/gordio/server/calls.go @@ -9,9 +9,9 @@ import ( "strings" "time" + "dynatron.me/x/stillbox/internal/common" "dynatron.me/x/stillbox/pkg/gordio/database" "github.com/google/uuid" - "github.com/jackc/pgtype" "github.com/rs/zerolog/log" ) @@ -34,26 +34,22 @@ type callUploadRequest struct { TalkgroupTag string `form:"talkgroupTag"` } -type AddCallParams struct { - Submitter pgtype.Int4 `json:"submitter"` - System int32 `json:"system"` - Talkgroup int32 `json:"talkgroup"` - CallDate pgtype.Timestamp `json:"call_date"` - AudioName pgtype.Text `json:"audio_name"` - AudioBlob []byte `json:"audio_blob"` - AudioType pgtype.Text `json:"audio_type"` - AudioUrl pgtype.Text `json:"audio_url"` - Frequency pgtype.Int4 `json:"frequency"` - Frequencies []byte `json:"frequencies"` - Patches []byte `json:"patches"` - TgLabel pgtype.Text `json:"tg_label"` - Source pgtype.Text `json:"source"` -} - func (car *callUploadRequest) ToAddCallParams(submitter int) database.AddCallParams { return database.AddCallParams{ - Submitter: submitter, - System: car.System, + Submitter: common.PtrTo(int32(submitter)), + System: car.System, + Talkgroup: car.Talkgroup, + CallDate: car.DateTime, + AudioName: common.PtrOrNull(car.AudioName), + AudioBlob: car.Audio, + AudioType: common.PtrOrNull(car.AudioType), + Frequency: car.Frequency, + Frequencies: car.Frequencies, + Patches: car.Patches, + TgLabel: common.PtrOrNull(car.TalkgroupLabel), + TgTag: common.PtrOrNull(car.TalkgroupTag), + TgGroup: common.PtrOrNull(car.TalkgroupGroup), + Source: car.Source, } } @@ -81,7 +77,7 @@ func (s *Server) routeCallUpload(w http.ResponseWriter, r *http.Request) { return } - if apik.Disabled.Bool || (apik.Expires.Valid && time.Now().After(apik.Expires.Time)) { + if (apik.Disabled != nil && *apik.Disabled) || (apik.Expires.Valid && time.Now().After(apik.Expires.Time)) { http.Error(w, "disabled", http.StatusUnauthorized) log.Error().Str("key", apik.ApiKey.String()).Msg("key disabled") return @@ -94,12 +90,14 @@ func (s *Server) routeCallUpload(w http.ResponseWriter, r *http.Request) { return } - dbCall, err := db.AddCall(r.Context(), call.ToAddCallParams()) + dbCall, err := db.AddCall(r.Context(), call.ToAddCallParams(apik.Owner)) if err != nil { http.Error(w, "internal error", http.StatusInternalServerError) log.Error().Err(err).Msg("add call") return } + + _ = dbCall } func (car *callUploadRequest) fill(r *http.Request) error { @@ -124,7 +122,12 @@ func (car *callUploadRequest) fill(r *http.Request) error { f.SetBytes(audioBytes) case time.Time: - t, err := time.Parse(time.RFC3339, r.Form.Get(formField)) + tval := r.Form.Get(formField) + if iv, err := strconv.Atoi(tval); err == nil { + f.Set(reflect.ValueOf(time.Unix(int64(iv), 0))) + break + } + t, err := time.Parse(time.RFC3339, tval) if err != nil { return fmt.Errorf("parse time: %w", err) } diff --git a/sql/postgres/migrations/001_initial.up.sql b/sql/postgres/migrations/001_initial.up.sql index c2d9fb1..0d38e0d 100644 --- a/sql/postgres/migrations/001_initial.up.sql +++ b/sql/postgres/migrations/001_initial.up.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS users( username VARCHAR (255) UNIQUE NOT NULL, password TEXT NOT NULL, email TEXT NOT NULL, - is_admin BOOLEAN, + is_admin BOOLEAN NOT NULL, prefs JSONB ); @@ -11,7 +11,7 @@ CREATE INDEX IF NOT EXISTS users_username_idx ON users(username); CREATE TABLE IF NOT EXISTS api_keys( id SERIAL PRIMARY KEY, - owner INTEGER REFERENCES users(id), + owner INTEGER REFERENCES users(id) NOT NULL, created_at TIMESTAMP NOT NULL, expires TIMESTAMP, disabled BOOLEAN, @@ -33,6 +33,37 @@ CREATE TABLE IF NOT EXISTS talkgroups( PRIMARY KEY (system_id, tgid) ); +CREATE TABLE IF NOT EXISTS talkgroups_learned( + id SERIAL PRIMARY KEY, + system_id INTEGER REFERENCES systems(id) NOT NULL, + tgid INTEGER NOT NULL, + group_name TEXT NOT NULL, + group_tag TEXT, + UNIQUE (system_id, tgid, group_name, group_tag) +); + +CREATE OR REPLACE FUNCTION learn_talkgroup() +RETURNS TRIGGER AS $$ +BEGIN + IF NOT EXISTS (SELECT * + FROM talkgroups + JOIN talkgroups_tags + ON talkgroups_tags.system_id = talkgroups.system_id AND talkgroups_tags.talkgroup_id = talkgroups.tgid + WHERE + talkgroups.system_id = NEW.system AND talkgroups.tgid = NEW.talkgroup AND + ( + talkgroups.name != NEW.tg_label + OR NOT (talkgroups_tags.tags @> ARRAY[NEW.tg_tag]) + ) + ) THEN + INSERT INTO talkgroups_learned(system_id, tgid, group_name, group_tag) VALUES( + NEW.system, NEW.talkgroup, NEW.tg_label, NEW.tg_tag + ) ON CONFLICT DO NOTHING; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; + CREATE TABLE IF NOT EXISTS talkgroups_tags( system_id INTEGER NOT NULL, talkgroup_id INTEGER NOT NULL, @@ -52,15 +83,20 @@ CREATE TABLE IF NOT EXISTS calls( audio_blob BYTEA, audio_type TEXT, audio_url TEXT, - frequency INTEGER, - frequencies JSONB, - patches JSONB, + frequency INTEGER NOT NULL, + frequencies INTEGER[], + patches INTEGER[], tg_label TEXT, - source TEXT, + tg_tag TEXT, + tg_group TEXT, + source INTEGER NOT NULL, transcript TEXT ); +CREATE OR REPLACE TRIGGER learn_tg AFTER INSERT ON calls +FOR EACH ROW EXECUTE FUNCTION learn_talkgroup(); + CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript)); CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date); @@ -89,6 +125,5 @@ CREATE TABLE IF NOT EXISTS incidents_calls( incident_id UUID REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, call_id UUID REFERENCES calls(id) ON UPDATE CASCADE, notes JSONB, - -- CONSTRAINT incident_call_pkey PRIMARY KEY (incident_id, call_id) PRIMARY KEY (incident_id, call_id) ); diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index f05f543..42a1f77 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -13,8 +13,10 @@ INSERT INTO calls ( frequencies, patches, tg_label, + tg_tag, + tg_group, source - ) VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ) VALUES (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *; -- name: SetCallTranscript :exec diff --git a/sql/sqlc.yaml b/sql/sqlc.yaml index ed0bfb0..a65204f 100644 --- a/sql/sqlc.yaml +++ b/sql/sqlc.yaml @@ -10,6 +10,7 @@ sql: sql_package: "pgx/v5" query_parameter_limit: 3 emit_json_tags: true + emit_interface: true emit_pointers_for_null_types: true overrides: - db_type: "uuid" @@ -20,7 +21,9 @@ sql: go_type: "int" - db_type: "pg_catalog.serial4" go_type: "int" - - db_type: "int" + - db_type: "integer" go_type: "int" - db_type: "pg_catalog.timestamp" go_type: "time.Time" + - db_type: "pg_catalog.text" + go_type: "string"