From 19aa6bcffdf2b0a855560780646936d70d2545fe Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Thu, 31 Oct 2024 09:09:58 -0400 Subject: [PATCH] Break out form unmarshal --- config.sample.yaml | 2 +- go.mod | 8 +- go.sum | 2 + internal/forms/forms.go | 113 +++++++++++++++++++++++++++++ internal/forms/forms_test.go | 94 ++++++++++++++++++++++++ internal/forms/testdata/call1.http | 54 ++++++++++++++ pkg/gordio/sources/http.go | 86 +--------------------- 7 files changed, 271 insertions(+), 88 deletions(-) create mode 100644 internal/forms/forms.go create mode 100644 internal/forms/forms_test.go create mode 100644 internal/forms/testdata/call1.http diff --git a/config.sample.yaml b/config.sample.yaml index 14a251b..cee3a9f 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -29,7 +29,7 @@ alerting: lookbackDays: 7 halfLife: 30m recent: 2h - alertThreshold: 0.5 + alertThreshold: 0.3 renotify: 30m notify: - provider: slackwebhook diff --git a/go.mod b/go.mod index 36e4b52..5cb2a64 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,10 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/hajimehoshi/oto v1.0.1 github.com/jackc/pgx/v5 v5.6.0 + github.com/nikoksr/notify v1.0.1 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 @@ -29,11 +31,11 @@ require ( require ( github.com/ajg/form v1.5.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/go-audio/audio v1.0.0 // indirect github.com/go-audio/riff v1.0.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -50,16 +52,14 @@ require ( github.com/lestrrat-go/option v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/nikoksr/notify v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.9.0 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/exp/shiny v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/image v0.14.0 // indirect golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a // indirect - golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index da2c282..d73a798 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= +github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/internal/forms/forms.go b/internal/forms/forms.go new file mode 100644 index 0000000..87253bc --- /dev/null +++ b/internal/forms/forms.go @@ -0,0 +1,113 @@ +package forms + +import ( + "errors" + "fmt" + "io" + "net/http" + "reflect" + "strconv" + "strings" + "time" +) + +var ( + ErrNotStruct = errors.New("destination is not a struct") + ErrNotPointer = errors.New("destination is not a pointer") +) + +func Unmarshal(r *http.Request, dest any) error { + rv := reflect.ValueOf(dest) + if k := rv.Kind(); k == reflect.Ptr { + rv = rv.Elem() + } else { + return ErrNotPointer + } + + if rv.Kind() != reflect.Struct { + return ErrNotStruct + } + + rt := rv.Type() + + for i := 0; i < rv.NumField(); i++ { + f := rv.Field(i) + fi := f.Interface() + formField, has := rt.Field(i).Tag.Lookup("form") + if !has || formField == "-" { + continue + } + + switch v := fi.(type) { + case string: + f.SetString(r.Form.Get(formField)) + case int: + ff := r.Form.Get(formField) + val, err := strconv.Atoi(ff) + if err != nil { + return fmt.Errorf("atoi('%s'): %w", ff, err) + } + f.SetInt(int64(val)) + case bool: + ff := r.Form.Get(formField) + if ff == "" { + continue + } + val, err := strconv.ParseBool(ff) + if err != nil { + return fmt.Errorf("parsebool('%s'): %w", ff, err) + } + f.SetBool(val) + case []byte: + file, hdr, err := r.FormFile(formField) + if err != nil { + return fmt.Errorf("get form file: %w", err) + } + + nameField, hasFilename := rt.Field(i).Tag.Lookup("filenameField") + if hasFilename { + fnf := rv.FieldByName(nameField) + if fnf == (reflect.Value{}) { + panic(fmt.Errorf("filenameField '%s' does not exist", nameField)) + } + + fnf.SetString(hdr.Filename) + } + audioBytes, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("file read: %w", err) + } + + f.SetBytes(audioBytes) + case time.Time: + tval := r.Form.Get(formField) + if iv, err := strconv.Atoi(tval); err == nil { + f.Set(reflect.ValueOf(time.Unix(int64(iv), 0))) + break + } + t, err := time.Parse(time.RFC3339, tval) + if err != nil { + return fmt.Errorf("parse time: %w", err) + } + f.Set(reflect.ValueOf(t)) + case []int: + val := strings.Trim(r.Form.Get(formField), "[]") + if val == "" { + continue + } + vals := strings.Split(val, ",") + ar := make([]int, 0, len(vals)) + for _, v := range vals { + i, err := strconv.Atoi(v) + if err == nil { + ar = append(ar, i) + } + } + f.Set(reflect.ValueOf(ar)) + default: + panic(fmt.Errorf("unsupported type %T", v)) + } + } + + return nil +} diff --git a/internal/forms/forms_test.go b/internal/forms/forms_test.go new file mode 100644 index 0000000..a6af909 --- /dev/null +++ b/internal/forms/forms_test.go @@ -0,0 +1,94 @@ +package forms_test + +import ( + "bufio" + "net/http" + "os" + "testing" + "time" + + "dynatron.me/x/stillbox/internal/forms" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type callUploadRequest struct { + Audio []byte `form:"audio" filenameField:"AudioName"` + AudioName string + AudioType string `form:"audioType"` + DateTime time.Time `form:"dateTime"` + Frequencies []int `form:"frequencies"` + Frequency int `form:"frequency"` + Key string `form:"key"` + Patches []int `form:"patches"` + Source int `form:"source"` + Sources []int `form:"sources"` + System int `form:"system"` + SystemLabel string `form:"systemLabel"` + Talkgroup int `form:"talkgroup"` + TalkgroupGroup string `form:"talkgroupGroup"` + TalkgroupLabel string `form:"talkgroupLabel"` + TalkgroupTag string `form:"talkgroupTag"` + DontStore bool `form:"dontStore"` +} + +var Call1 = callUploadRequest{ + AudioName: "20241031_081939.285.mp3", + Audio: []byte("ID3dotdotdotLAME3.98.4a"), + DateTime: time.Unix(1730377177, 0), + Frequency: 852637500, + Key: "2b7c871b-abcd-defa-0123-456789abcdef", + Source: 3237, + System: 197, + SystemLabel: "RISCON", + Talkgroup: 2, + TalkgroupGroup: "Wide Area", + TalkgroupLabel: "Wide Area 1 FD/EMS Intercity", +} + +func makeRequest(fixture string) *http.Request { + fixt, err := os.Open("testdata/"+fixture) + if err != nil { + panic(err) + } + + r, err := http.ReadRequest(bufio.NewReader(fixt)) + if err != nil { + panic(err) + } + + return r +} + +func TestUnmarshal(t *testing.T) { + tests := []struct{ + name string + r *http.Request + dest any + compare any + expectErr error + }{ + { + name: "base case", + r: makeRequest("call1.http"), + dest: &callUploadRequest{}, + compare: &Call1, + }, + { + name: "not a pointer", + r: makeRequest("call1.http"), + dest: callUploadRequest{}, + compare: callUploadRequest{}, + expectErr: forms.ErrNotPointer, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := forms.Unmarshal(tc.r, tc.dest) + require.Equal(t, tc.expectErr, err) + assert.Equal(t, tc.compare, tc.dest) + }) + } +} diff --git a/internal/forms/testdata/call1.http b/internal/forms/testdata/call1.http new file mode 100644 index 0000000..c819d22 --- /dev/null +++ b/internal/forms/testdata/call1.http @@ -0,0 +1,54 @@ +POST /api/call-upload HTTP/1.1 +Connection: Upgrade, HTTP2-Settings +Content-Length: 5372 +Host: xenon:3050 +HTTP2-Settings: AAEAAEAAAAIAAAAAAAMAAAAAAAQBAAAAAAUAAEAAAAYABgAA +Upgrade: h2c +Content-Type: multipart/form-data; boundary=--sdrtrunk-sdrtrunk-sdrtrunk +User-Agent: sdrtrunk + +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="key" + +2b7c871b-abcd-defa-0123-456789abcdef +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="system" + +197 +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="dateTime" + +1730377177 +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="talkgroup" + +2 +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="source" + +3237 +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="frequency" + +852637500 +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="talkgroupLabel" + +Wide Area 1 FD/EMS Intercity +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="talkgroupGroup" + +Wide Area +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="systemLabel" + +RISCON +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; name="patches" + +[] +----sdrtrunk-sdrtrunk-sdrtrunk +Content-Disposition: form-data; filename="20241031_081939.285.mp3"; name="audio" + +ID3dotdotdotLAME3.98.4a +----sdrtrunk-sdrtrunk-sdrtrunk-- diff --git a/pkg/gordio/sources/http.go b/pkg/gordio/sources/http.go index 2a5f13f..40048e5 100644 --- a/pkg/gordio/sources/http.go +++ b/pkg/gordio/sources/http.go @@ -1,15 +1,12 @@ package sources import ( - "fmt" - "io" "net/http" - "reflect" - "strconv" "strings" "time" "dynatron.me/x/stillbox/internal/common" + "dynatron.me/x/stillbox/internal/forms" "dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/gordio/auth" "github.com/go-chi/chi/v5" @@ -40,7 +37,7 @@ func (h *RdioHTTP) InstallPublicRoutes(r chi.Router) { } type callUploadRequest struct { - Audio []byte `form:"audio"` + Audio []byte `form:"audio" filenameField:"AudioName"` AudioName string AudioType string `form:"audioType"` DateTime time.Time `form:"dateTime"` @@ -115,7 +112,7 @@ func (h *RdioHTTP) routeCallUpload(w http.ResponseWriter, r *http.Request) { } cur := new(callUploadRequest) - err = cur.fill(r) + err = forms.Unmarshal(r, cur) if err != nil { http.Error(w, "cannot bind upload "+err.Error(), http.StatusExpectationFailed) return @@ -141,80 +138,3 @@ func (h *RdioHTTP) routeCallUpload(w http.ResponseWriter, r *http.Request) { log.Error().Err(err).Int("written", written).Msg("upload response failed") } } - -func (car *callUploadRequest) fill(r *http.Request) error { - rv := reflect.ValueOf(car).Elem() - rt := rv.Type() - - for i := 0; i < rv.NumField(); i++ { - f := rv.Field(i) - fi := f.Interface() - formField, has := rt.Field(i).Tag.Lookup("form") - if !has { - continue - } - switch v := fi.(type) { - case []byte: - file, hdr, err := r.FormFile(formField) - if err != nil { - return fmt.Errorf("get form file: %w", err) - } - - car.AudioName = hdr.Filename - audioBytes, err := io.ReadAll(file) - if err != nil { - return fmt.Errorf("file read: %w", err) - } - - f.SetBytes(audioBytes) - case time.Time: - tval := r.Form.Get(formField) - if iv, err := strconv.Atoi(tval); err == nil { - f.Set(reflect.ValueOf(time.Unix(int64(iv), 0))) - break - } - t, err := time.Parse(time.RFC3339, tval) - if err != nil { - return fmt.Errorf("parse time: %w", err) - } - f.Set(reflect.ValueOf(t)) - case []int: - val := strings.Trim(r.Form.Get(formField), "[]") - if val == "" { - continue - } - vals := strings.Split(val, ",") - ar := make([]int, 0, len(vals)) - for _, v := range vals { - i, err := strconv.Atoi(v) - if err == nil { - ar = append(ar, i) - } - } - f.Set(reflect.ValueOf(ar)) - case int: - ff := r.Form.Get(formField) - val, err := strconv.Atoi(ff) - if err != nil { - return fmt.Errorf("atoi('%s'): %w", ff, err) - } - f.SetInt(int64(val)) - case string: - f.SetString(r.Form.Get(formField)) - case bool: - ff := r.Form.Get(formField) - if ff == "" { - continue - } - val, err := strconv.ParseBool(ff) - if err != nil { - return fmt.Errorf("parsebool('%s'): %w", ff, err) - } - f.SetBool(val) - default: - panic(fmt.Errorf("unsupported type %T", v)) - } - } - - return nil -}