amigan/stillbox
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

Sim form

Browse source
This commit is contained in:
Daniel 2024-10-28 18:45:05 -04:00
parent 56f77bb509
commit 5233bc9623
10 changed files with 199 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -28,11 +28,13 @@ require (
require (
github.com/ajg/form v1.5.1 // indirect
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/go-audio/audio v1.0.0 // indirect
github.com/go-audio/riff v1.0.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gorilla/schema v1.4.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect

7
go.sum
View file

@ -6,6 +6,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@ -55,6 +57,8 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=
github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hajimehoshi/oto v1.0.1 h1:8AMnq0Yr2YmzaiqTg/k1Yzd6IygUGk2we9nmjgbgPn4=
@ -99,6 +103,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
@ -111,12 +116,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=

94
internal/jsontime/jsontime.go Normal file
View file

@ -0,0 +1,94 @@
package jsontime
import (
"encoding/json"
"strings"
"time"
"github.com/araddon/dateparse"
"gopkg.in/yaml.v3"
)
type Time time.Time
func (t *Time) UnmarshalYAML(n *yaml.Node) error {
var s string
err := n.Decode(&s)
if err != nil {
return err
}
tm, err := dateparse.ParseAny(s)
if err != nil {
return err
}
*t = Time(tm)
return nil
}
func (t *Time) UnmarshalJSON(b []byte) error {
s := strings.Trim(string(b), `"`)
tm, err := dateparse.ParseAny(s)
if err != nil {
return err
}
*t = Time(tm)
return nil
}
func (t Time) MarshalJSON() ([]byte, error) {
return json.Marshal(time.Time(t))
}
func (t Time) String() string {
return time.Time(t).String()
}
type Duration time.Duration
func (d *Duration) UnmarshalYAML(n *yaml.Node) error {
var s string
err := n.Decode(&s)
if err != nil {
return err
}
dur, err := time.ParseDuration(s)
if err != nil {
return err
}
*d = Duration(dur)
return nil
}
func (d *Duration) UnmarshalJSON(b []byte) error {
s := strings.Trim(string(b), `"`)
dur, err := time.ParseDuration(s)
if err != nil {
return err
}
*d = Duration(dur)
return nil
}
func (d Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(time.Duration(d))
}
func (d Duration) String() string {
return time.Duration(d).String()
}
func ParseDuration(s string) (Duration, error) {
d, err := time.ParseDuration(s)
return Duration(d), err
}
func ParseAny(s string, opt ...dateparse.ParserOption) (Time, error) {
t, err := dateparse.ParseAny(s, opt...)
return Time(t), err
}

4
pkg/gordio/alerting/alerting.go
View file

@ -83,8 +83,8 @@ func New(cfg config.Alerting, opts ...AlertOption) Alerter {
as.scorer = trending.NewScorer[cl.Talkgroup](
trending.WithTimeSeries(as.newTimeSeries),
trending.WithStorageDuration[cl.Talkgroup](time.Hour*24*time.Duration(cfg.LookbackDays)),
trending.WithRecentDuration[cl.Talkgroup](cfg.Recent),
trending.WithHalfLife[cl.Talkgroup](cfg.HalfLife),
trending.WithRecentDuration[cl.Talkgroup](time.Duration(cfg.Recent)),
trending.WithHalfLife[cl.Talkgroup](time.Duration(cfg.HalfLife)),
trending.WithScoreThreshold[cl.Talkgroup](ScoreThreshold),
trending.WithCountThreshold[cl.Talkgroup](CountThreshold),
trending.WithClock[cl.Talkgroup](as.clock),

50
pkg/gordio/alerting/simulate.go
View file

@ -5,8 +5,10 @@ import (
"encoding/json"
"net/http"
"sort"
"strconv"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
"dynatron.me/x/stillbox/internal/trending"
cl "dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/gordio/config"
@ -16,23 +18,24 @@ import (
type Simulation struct {
config.Alerting
ScoreStart time.Time `json:"scoreStart"`
ScoreStart jsontime.Time `json:"scoreStart"`
clock offsetClock
clock offsetClock `json:"-"`
*alerter `json:"-"`
}
func (s *Simulation) Simulate(ctx context.Context) trending.Scores[cl.Talkgroup] {
const step = 5 * time.Minute
s.Enable = true
s.alerter = New(s.Alerting, WithClock(&s.clock)).(*alerter)
s.alerter.sim = s
now := time.Now()
sinceLookback := now.Add(-24 * time.Hour * time.Duration(s.LookbackDays))
s.backfill(ctx, sinceLookback, s.ScoreStart)
s.backfill(ctx, sinceLookback, time.Time(s.ScoreStart))
for s.clock = offsetClock(now.Sub(s.ScoreStart)); now.After(now.Add(s.clock.Duration())); s.clock += offsetClock(time.Minute) {
s.backfill(ctx, s.clock.Now().Add(-1*time.Minute), s.clock.Now())
for s.clock = offsetClock(now.Sub(time.Time(s.ScoreStart))); now.After(now.Add(-s.clock.Duration())); s.clock -= offsetClock(step) {
s.backfill(ctx, s.clock.Now().Add(-step), s.clock.Now())
s.scores = s.scorer.Score()
}
@ -42,16 +45,51 @@ func (s *Simulation) Simulate(ctx context.Context) trending.Scores[cl.Talkgroup]
return s.scores
}
func simulateHandler(w http.ResponseWriter, r *http.Request) {
func (as *alerter) simulateHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
s := new(Simulation)
switch r.Header.Get("Content-Type") {
case "application/json":
err := json.NewDecoder(r.Body).Decode(s)
if err != nil {
log.Error().Err(err).Msg("simulate decode")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
default:
err := r.ParseForm()
if err != nil {
log.Error().Err(err).Msg("simulate form parse")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
lbd, err := strconv.Atoi(r.Form["lookbackDays"][0])
if err != nil {
log.Error().Err(err).Msg("simulate form parse")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.LookbackDays = uint(lbd)
s.HalfLife, err = jsontime.ParseDuration(r.Form["halfLife"][0])
if err != nil {
log.Error().Err(err).Msg("simulate form parse")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.Recent, err = jsontime.ParseDuration(r.Form["recent"][0])
if err != nil {
log.Error().Err(err).Msg("simulate form parse")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.ScoreStart, err = jsontime.ParseAny(r.Form["scoreStart"][0])
if err != nil {
log.Error().Err(err).Msg("simulate form parse")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// TODO: sanity check that ScoreStart is not before lookback and lookback is sane
s.Simulate(ctx)

19
pkg/gordio/alerting/stats.go
View file

@ -2,12 +2,14 @@ package alerting
import (
_ "embed"
"errors"
"fmt"
"html/template"
"net/http"
"time"
"dynatron.me/x/stillbox/pkg/calls"
"dynatron.me/x/stillbox/pkg/gordio/config"
"dynatron.me/x/stillbox/pkg/gordio/database"
"dynatron.me/x/stillbox/internal/trending"
@ -28,12 +30,27 @@ var (
"f": func(v float64) string {
return fmt.Sprintf("%.4f", v)
},
"dict": func(values ...interface{}) (map[string]interface{}, error) {
if len(values)%2 != 0 {
return nil, errors.New("invalid dict call")
}
dict := make(map[string]interface{}, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, ok := values[i].(string)
if !ok {
return nil, errors.New("dict keys must be strings")
}
dict[key] = values[i+1]
}
return dict, nil
},
}
statTmpl = template.Must(template.New("stats").Funcs(funcMap).Parse(statsTemplateFile))
)
func (as *alerter) PrivateRoutes(r chi.Router) {
r.Get("/tgstats", as.tgStats)
r.Post("/simulate", as.simulateHandler)
}
func (as *noopAlerter) PrivateRoutes(r chi.Router) {}
@ -64,10 +81,12 @@ func (as *alerter) tgStats(w http.ResponseWriter, r *http.Request) {
Scores trending.Scores[calls.Talkgroup]
LastScore time.Time
Simulation *Simulation
Config config.Alerting
}{
TGs: tgMap,
Scores: as.scores,
LastScore: as.lastScore,
Config: as.cfg,
Simulation: as.sim,
}

20
pkg/gordio/alerting/stats.html
View file

@ -12,7 +12,7 @@
background: #105469;
font-family: sans-serif;
}
table {
table, #simform {
background: #012B39;
border-radius: 0.25em;
border-collapse: collapse;
@ -29,7 +29,7 @@
padding: 0.5em 1em;
text-align: left;
}
td {
td, #simform {
color: #fff;
font-weight: 400;
padding: 0.65em 1em;
@ -47,9 +47,23 @@
</head>
<body>
<div id="simform">
<form action="/simulate" method="POST">
{{ define "simform" }}
<label for="lookbackDays">Lookback Days</label> <input id="lookbackDays" name="lookbackDays" type="number" min="1" max="14" value="{{ .Lookback }}" />
<label for="halfLife">Half life</label> <input id="halfLife" name="halfLife" type="text" value="{{ .HalfLife }}" />
<label for="recent">Recent</label> <input id="recent" name="recent" type="text" value="{{ .Recent }}" />
<label for="scoreStart">Score Start</label> <input id="scoreStart" name="scoreStart" type="datetime-local" value="{{ .ScoreStart }}" />
<input type="submit" value="Simulate" />
{{end}}
{{ if .Simulation }}
{{ template "simform" dict "Lookback" .Simulation.LookbackDays "HalfLife" .Simulation.HalfLife "Recent" .Simulation.Recent "ScoreStart" .Simulation.ScoreStart }}
{{ else }}
{{ template "simform" dict "Lookback" .Config.LookbackDays "HalfLife" .Config.HalfLife "Recent" .Config.Recent "ScoreStart" "" }}
{{ end }}
</form>
</div>
<table>
{{ if .Simluation }}
{{ if .Simulation }}
<tr>
<td colspan="10">Simulating from {{ .Simulation.ScoreStart }} until now</td>
</tr>

6
pkg/gordio/config/config.go
View file

@ -5,6 +5,8 @@ import (
"sync"
"time"
"dynatron.me/x/stillbox/internal/jsontime"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
@ -53,8 +55,8 @@ type RateLimit struct {
type Alerting struct {
Enable bool `yaml:"enable"`
LookbackDays uint `yaml:"lookbackDays"`
HalfLife time.Duration `yaml:"halfLife"`
Recent time.Duration `yaml:"recent"`
HalfLife jsontime.Duration `yaml:"halfLife"`
Recent jsontime.Duration `yaml:"recent"`
}
func (rl *RateLimit) Verify() bool {

1
pkg/gordio/server/noprofile.go
View file

@ -1,3 +1,4 @@
//go:build !pprof
// +build !pprof
package server

1
pkg/gordio/server/profile.go
View file

@ -1,3 +1,4 @@
//go:build pprof
// +build pprof
package server