From 4bec0b5a78fee5ba49fe6a16ca91d01ad50c8279 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 22 Feb 2025 16:03:59 -0500 Subject: [PATCH 1/5] Fix getincidentcalls --- pkg/database/incidents.sql.go | 2 +- sql/postgres/queries/incidents.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/database/incidents.sql.go b/pkg/database/incidents.sql.go index 518f8b5..9ae6feb 100644 --- a/pkg/database/incidents.sql.go +++ b/pkg/database/incidents.sql.go @@ -203,7 +203,7 @@ FROM incidents_calls ic, LATERAL ( sc.audio_url, sc.frequency, sc.frequencies, - c.talker_alias, + sc.talker_alias, sc.patches, sc.source, sc.transcript diff --git a/sql/postgres/queries/incidents.sql b/sql/postgres/queries/incidents.sql index dc3591d..d0ee575 100644 --- a/sql/postgres/queries/incidents.sql +++ b/sql/postgres/queries/incidents.sql @@ -151,7 +151,7 @@ FROM incidents_calls ic, LATERAL ( sc.audio_url, sc.frequency, sc.frequencies, - c.talker_alias, + sc.talker_alias, sc.patches, sc.source, sc.transcript From 6c3dd5e636f32dfe599e511d884087b30b5d3b97 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Tue, 18 Feb 2025 08:58:12 -0500 Subject: [PATCH 2/5] break intervals out --- internal/common/intervals.go | 84 +++++++++++++++++++++++++++++++ pkg/database/partman/intervals.go | 73 +++++++++------------------ pkg/database/partman/partman.go | 20 +------- pkg/database/stats.sql.go | 4 +- pkg/stats/stats.go | 13 ++++- sql/postgres/queries/stats.sql | 4 +- 6 files changed, 125 insertions(+), 73 deletions(-) create mode 100644 internal/common/intervals.go diff --git a/internal/common/intervals.go b/internal/common/intervals.go new file mode 100644 index 0000000..0458f3c --- /dev/null +++ b/internal/common/intervals.go @@ -0,0 +1,84 @@ +package common + +import ( + "time" +) + +const ( + DaysInWeek = 7 + MonthsInQuarter = 3 +) + +type TimeBounder interface { + GetDailyBounds(date time.Time) (lowerBound, upperBound time.Time) + GetWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) + GetMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) + GetQuarterlyBounds(date time.Time) (lowerBound, upperBound time.Time) + GetYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) +} + +type tbOpt func(*timeBounder) + +func WithLocation(l *time.Location) tbOpt { + return func(tb *timeBounder) { + tb.loc = l + } +} + +func NewTimeBounder(opts ...tbOpt) timeBounder { + tb := timeBounder{} + + for _, opt := range opts { + opt(&tb) + } + + if tb.loc == nil { + tb.loc = time.UTC + } + + return tb +} + +type timeBounder struct { + loc *time.Location +} + +func (tb timeBounder) GetDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, tb.loc) + upperBound = lowerBound.AddDate(0, 0, 1) + + return +} + +func (tb timeBounder) GetWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, tb.loc).AddDate(0, 0, -int(date.Weekday()-time.Monday)) + upperBound = lowerBound.AddDate(0, 0, DaysInWeek) + + return +} + +func (tb timeBounder) GetMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, tb.loc) + upperBound = lowerBound.AddDate(0, 1, 0) + + return +} + +func (tb *timeBounder) GetQuarterlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + year, _, _ := date.Date() + + quarter := (int(date.Month()) - 1) / MonthsInQuarter + firstMonthOfTheQuarter := time.Month(quarter*MonthsInQuarter + 1) + + lowerBound = time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, tb.loc) + upperBound = lowerBound.AddDate(0, MonthsInQuarter, 0) + + return +} + +func (tb timeBounder) GetYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, tb.loc) + upperBound = lowerBound.AddDate(1, 0, 0) + + return +} diff --git a/pkg/database/partman/intervals.go b/pkg/database/partman/intervals.go index 0252302..e040b02 100644 --- a/pkg/database/partman/intervals.go +++ b/pkg/database/partman/intervals.go @@ -3,66 +3,23 @@ package partman import ( "fmt" "time" + + "dynatron.me/x/stillbox/internal/common" ) -const ( - daysInWeek = 7 - monthsInQuarter = 3 -) - -func getDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC) - upperBound = lowerBound.AddDate(0, 0, 1) - - return -} - -func getWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, -int(date.Weekday()-time.Monday)) - upperBound = lowerBound.AddDate(0, 0, daysInWeek) - - return -} - -func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC) - upperBound = lowerBound.AddDate(0, 1, 0) - - return -} - -func getQuarterlyBounds(date time.Time) (lowerBound, upperBound time.Time) { - year, _, _ := date.Date() - - quarter := (int(date.Month()) - 1) / monthsInQuarter - firstMonthOfTheQuarter := time.Month(quarter*monthsInQuarter + 1) - - lowerBound = time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) - upperBound = lowerBound.AddDate(0, monthsInQuarter, 0) - - return -} - -func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, time.UTC) - upperBound = lowerBound.AddDate(1, 0, 0) - - return -} - func (p Partition) Next(i int) Partition { var t time.Time switch p.Interval { case Daily: t = p.Time.AddDate(0, 0, i) case Weekly: - t = p.Time.AddDate(0, 0, i*daysInWeek) + t = p.Time.AddDate(0, 0, i*common.DaysInWeek) case Monthly: year, month, _ := p.Time.Date() t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, p.Time.Location()) case Quarterly: - t = p.Time.AddDate(0, i*monthsInQuarter, 0) + t = p.Time.AddDate(0, i*common.MonthsInQuarter, 0) case Yearly: year, _, _ := p.Time.Date() @@ -125,13 +82,13 @@ func (p Partition) Prev(i int) Partition { case Daily: t = p.Time.AddDate(0, 0, -i) case Weekly: - t = p.Time.AddDate(0, 0, -i*daysInWeek) + t = p.Time.AddDate(0, 0, -i*common.DaysInWeek) case Monthly: year, month, _ := p.Time.Date() t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, p.Time.Location()) case Quarterly: - t = p.Time.AddDate(0, -i*monthsInQuarter, 0) + t = p.Time.AddDate(0, -i*common.MonthsInQuarter, 0) case Yearly: year, _, _ := p.Time.Date() @@ -150,3 +107,21 @@ func (p Partition) Prev(i int) Partition { return pp } + +func (p Partition) Range() (time.Time, time.Time) { + b := common.NewTimeBounder() + switch p.Interval { + case Daily: + return b.GetDailyBounds(p.Time) + case Weekly: + return b.GetWeeklyBounds(p.Time) + case Monthly: + return b.GetMonthlyBounds(p.Time) + case Quarterly: + return b.GetQuarterlyBounds(p.Time) + case Yearly: + return b.GetYearlyBounds(p.Time) + } + + panic("unknown interval!") +} diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index 9b7afcd..461d67b 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "dynatron.me/x/stillbox/internal/common" "dynatron.me/x/stillbox/internal/isoweek" "dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/database" @@ -325,23 +326,6 @@ func (pm *partman) Check(ctx context.Context, now time.Time) error { }, pgx.TxOptions{}) } -func (p Partition) Range() (time.Time, time.Time) { - switch p.Interval { - case Daily: - return getDailyBounds(p.Time) - case Weekly: - return getWeeklyBounds(p.Time) - case Monthly: - return getMonthlyBounds(p.Time) - case Quarterly: - return getQuarterlyBounds(p.Time) - case Yearly: - return getYearlyBounds(p.Time) - } - - panic("unknown interval!") -} - func (p Partition) PartitionName() string { return p.Name } @@ -423,7 +407,7 @@ func (pm *partman) verifyPartName(pr database.PartitionResult) (p Partition, err if quarterNum > 4 { return p, PartitionError(pn, errors.New("invalid quarter")) } - firstMonthOfTheQuarter := time.Month((quarterNum-1)*monthsInQuarter + 1) + firstMonthOfTheQuarter := time.Month((quarterNum-1)*common.MonthsInQuarter + 1) parsed := time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) if parsed != p.Time { return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) diff --git a/pkg/database/stats.sql.go b/pkg/database/stats.sql.go index 3db6ae8..6a54683 100644 --- a/pkg/database/stats.sql.go +++ b/pkg/database/stats.sql.go @@ -20,7 +20,7 @@ WHERE CASE WHEN $2::TIMESTAMPTZ IS NOT NULL THEN c.call_date >= $2 ELSE TRUE END AND CASE WHEN $3::TIMESTAMPTZ IS NOT NULL THEN - c.call_date <= $3 ELSE TRUE END + c.call_date < $3 ELSE TRUE END GROUP BY date ORDER BY date DESC ` @@ -61,7 +61,7 @@ WHERE CASE WHEN $2::TIMESTAMPTZ IS NOT NULL THEN c.call_date >= $2 ELSE TRUE END AND CASE WHEN $3::TIMESTAMPTZ IS NOT NULL THEN - c.call_date <= $3 ELSE TRUE END + c.call_date < $3 ELSE TRUE END GROUP BY 2, 3, 4 ORDER BY 4 DESC ` diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go index b1e03bd..e8c4d31 100644 --- a/pkg/stats/stats.go +++ b/pkg/stats/stats.go @@ -5,10 +5,12 @@ import ( "time" "dynatron.me/x/stillbox/internal/cache" + "dynatron.me/x/stillbox/internal/common" "dynatron.me/x/stillbox/internal/jsontypes" "dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/calls/callstore" "dynatron.me/x/stillbox/pkg/services" + "github.com/rs/zerolog/log" ) const DefaultExpiration = 5 * time.Minute @@ -56,20 +58,27 @@ func (s *stats) GetCallStats(ctx context.Context, interval calls.StatsInterval) var start time.Time now := time.Now() + end := now + bnd := common.NewTimeBounder(common.WithLocation(now.Location())) + switch interval { case calls.IntervalHour: start = now.Add(-24 * time.Hour) // one day case calls.IntervalDay: start = now.Add(-7 * 24 * time.Hour) // one week case calls.IntervalWeek: - start = now.Add(-30 * 24 * time.Hour) // one month + start, end = bnd.GetMonthlyBounds(now) + start, _ = bnd.GetWeeklyBounds(start) + _, end = bnd.GetWeeklyBounds(end) case calls.IntervalMonth: start = now.Add(-365 * 24 * time.Hour) // one year default: return nil, calls.ErrInvalidInterval } - st, err := s.cs.CallStats(ctx, interval, jsontypes.Time(start), jsontypes.Time(now)) + log.Debug().Str("start", start.String()).Str("end", end.String()).Msg("bound") + + st, err := s.cs.CallStats(ctx, interval, jsontypes.Time(start), jsontypes.Time(end)) if err != nil { return nil, err } diff --git a/sql/postgres/queries/stats.sql b/sql/postgres/queries/stats.sql index e201da8..32538c7 100644 --- a/sql/postgres/queries/stats.sql +++ b/sql/postgres/queries/stats.sql @@ -9,7 +9,7 @@ WHERE CASE WHEN sqlc.narg('start')::TIMESTAMPTZ IS NOT NULL THEN c.call_date >= @start ELSE TRUE END AND CASE WHEN sqlc.narg('end')::TIMESTAMPTZ IS NOT NULL THEN - c.call_date <= sqlc.narg('end') ELSE TRUE END + c.call_date < sqlc.narg('end') ELSE TRUE END GROUP BY 2, 3, 4 ORDER BY 4 DESC; @@ -22,6 +22,6 @@ WHERE CASE WHEN sqlc.narg('start')::TIMESTAMPTZ IS NOT NULL THEN c.call_date >= @start ELSE TRUE END AND CASE WHEN sqlc.narg('end')::TIMESTAMPTZ IS NOT NULL THEN - c.call_date <= sqlc.narg('end') ELSE TRUE END + c.call_date < sqlc.narg('end') ELSE TRUE END GROUP BY date ORDER BY date DESC; From 2640dd9110a8a6c5f3921464c6d15f01a11adbc9 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Tue, 18 Feb 2025 09:02:39 -0500 Subject: [PATCH 3/5] remove bound log --- pkg/stats/stats.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go index e8c4d31..5504935 100644 --- a/pkg/stats/stats.go +++ b/pkg/stats/stats.go @@ -76,8 +76,6 @@ func (s *stats) GetCallStats(ctx context.Context, interval calls.StatsInterval) return nil, calls.ErrInvalidInterval } - log.Debug().Str("start", start.String()).Str("end", end.String()).Msg("bound") - st, err := s.cs.CallStats(ctx, interval, jsontypes.Time(start), jsontypes.Time(end)) if err != nil { return nil, err From 48e2dcd7724c658d63ff980c5d9d2945aca5aa66 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Fri, 21 Feb 2025 20:59:08 -0500 Subject: [PATCH 4/5] make it debug, emit the tg id --- pkg/alerting/alerting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/alerting/alerting.go b/pkg/alerting/alerting.go index 72b39e8..6247669 100644 --- a/pkg/alerting/alerting.go +++ b/pkg/alerting/alerting.go @@ -182,7 +182,7 @@ func (as *alerter) eval(ctx context.Context, now time.Time, testMode bool) ([]al origScore := s.Score tgr, err := as.tgCache.TG(ctx, s.ID) if err != nil { - log.Error().Err(err).Msg("alerting eval tg get") + log.Debug().Str("tg", s.ID.String()).Err(err).Msg("alerting eval tg get") continue } From adbcf2ae47d7ee5f1fb5703b8d4672ec2f356461 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 22 Feb 2025 17:27:46 -0500 Subject: [PATCH 5/5] Unknown TG --- pkg/calls/callstore/store.go | 3 +++ pkg/database/calls.sql.go | 18 ++++++++++++++---- pkg/stats/stats.go | 1 - sql/postgres/queries/calls.sql | 6 ++++++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/calls/callstore/store.go b/pkg/calls/callstore/store.go index 14e9716..e405ec2 100644 --- a/pkg/calls/callstore/store.go +++ b/pkg/calls/callstore/store.go @@ -204,6 +204,7 @@ type CallsParams struct { TagsNot []string `json:"tagsNot"` TGFilter *string `json:"tgFilter"` AtLeastSeconds *float32 `json:"atLeastSeconds"` + UnknownTG bool `json:"unknownTG"` } func (s *postgresStore) Calls(ctx context.Context, p CallsParams) (rows []database.ListCallsPRow, totalCount int, err error) { @@ -224,6 +225,7 @@ func (s *postgresStore) Calls(ctx context.Context, p CallsParams) (rows []databa PerPage: perPage, Direction: p.Direction.DirString(common.DirAsc), TGFilter: p.TGFilter, + UnknownTG: p.UnknownTG, } if p.AtLeastSeconds != nil { @@ -245,6 +247,7 @@ func (s *postgresStore) Calls(ctx context.Context, p CallsParams) (rows []databa TagsNot: par.TagsNot, TGFilter: par.TGFilter, LongerThan: par.LongerThan, + UnknownTG: par.UnknownTG, }) if err != nil { return err diff --git a/pkg/database/calls.sql.go b/pkg/database/calls.sql.go index 45d1702..120110f 100644 --- a/pkg/database/calls.sql.go +++ b/pkg/database/calls.sql.go @@ -319,6 +319,9 @@ CASE WHEN $4::TEXT[] IS NOT NULL THEN ) ELSE TRUE END) AND (CASE WHEN $6::NUMERIC IS NOT NULL THEN ( c.duration > $6 + ) ELSE TRUE END) AND +(CASE WHEN $7::BOOLEAN = TRUE THEN ( + tgs.tgid IS NULL ) ELSE TRUE END) ` @@ -329,6 +332,7 @@ type ListCallsCountParams struct { TagsNot []string `json:"tagsNot"` TGFilter *string `json:"tgFilter"` LongerThan pgtype.Numeric `json:"longerThan"` + UnknownTG bool `json:"unknownTg"` } func (q *Queries) ListCallsCount(ctx context.Context, arg ListCallsCountParams) (int64, error) { @@ -339,6 +343,7 @@ func (q *Queries) ListCallsCount(ctx context.Context, arg ListCallsCountParams) arg.TagsNot, arg.TGFilter, arg.LongerThan, + arg.UnknownTG, ) var count int64 err := row.Scan(&count) @@ -374,13 +379,16 @@ CASE WHEN $4::TEXT[] IS NOT NULL THEN ) ELSE TRUE END) AND (CASE WHEN $6::NUMERIC IS NOT NULL THEN ( c.duration > $6 + ) ELSE TRUE END) AND +(CASE WHEN $7::BOOLEAN = TRUE THEN ( + tgs.tgid IS NULL ) ELSE TRUE END) GROUP BY c.id, c.call_date ORDER BY -CASE WHEN $7::TEXT = 'asc' THEN c.call_date END ASC, -CASE WHEN $7 = 'desc' THEN c.call_date END DESC -OFFSET $8 ROWS -FETCH NEXT $9 ROWS ONLY +CASE WHEN $8::TEXT = 'asc' THEN c.call_date END ASC, +CASE WHEN $8 = 'desc' THEN c.call_date END DESC +OFFSET $9 ROWS +FETCH NEXT $10 ROWS ONLY ` type ListCallsPParams struct { @@ -390,6 +398,7 @@ type ListCallsPParams struct { TagsNot []string `json:"tagsNot"` TGFilter *string `json:"tgFilter"` LongerThan pgtype.Numeric `json:"longerThan"` + UnknownTG bool `json:"unknownTg"` Direction string `json:"direction"` Offset int32 `json:"offset"` PerPage int32 `json:"perPage"` @@ -413,6 +422,7 @@ func (q *Queries) ListCallsP(ctx context.Context, arg ListCallsPParams) ([]ListC arg.TagsNot, arg.TGFilter, arg.LongerThan, + arg.UnknownTG, arg.Direction, arg.Offset, arg.PerPage, diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go index 5504935..a82991c 100644 --- a/pkg/stats/stats.go +++ b/pkg/stats/stats.go @@ -10,7 +10,6 @@ import ( "dynatron.me/x/stillbox/pkg/calls" "dynatron.me/x/stillbox/pkg/calls/callstore" "dynatron.me/x/stillbox/pkg/services" - "github.com/rs/zerolog/log" ) const DefaultExpiration = 5 * time.Minute diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index 0898863..13798c4 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -127,6 +127,9 @@ CASE WHEN sqlc.narg('tags_not')::TEXT[] IS NOT NULL THEN ) ELSE TRUE END) AND (CASE WHEN sqlc.narg('longer_than')::NUMERIC IS NOT NULL THEN ( c.duration > @longer_than + ) ELSE TRUE END) AND +(CASE WHEN @unknown_tg::BOOLEAN = TRUE THEN ( + tgs.tgid IS NULL ) ELSE TRUE END) GROUP BY c.id, c.call_date ORDER BY @@ -157,6 +160,9 @@ CASE WHEN sqlc.narg('tags_not')::TEXT[] IS NOT NULL THEN ) ELSE TRUE END) AND (CASE WHEN sqlc.narg('longer_than')::NUMERIC IS NOT NULL THEN ( c.duration > @longer_than + ) ELSE TRUE END) AND +(CASE WHEN @unknown_tg::BOOLEAN = TRUE THEN ( + tgs.tgid IS NULL ) ELSE TRUE END) ;