From 8b6fd5b948ea954034d8c9de25265415ea0a00b0 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Tue, 26 Nov 2024 20:46:12 -0500 Subject: [PATCH 01/15] "Accept interfaces, return concrete" --- pkg/database/database.go | 2 +- pkg/notify/notify.go | 2 +- pkg/rest/api.go | 2 +- pkg/sinks/sinks.go | 2 +- pkg/talkgroups/tgstore/store.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 97aa7d5..7f38fd1 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -72,7 +72,7 @@ func Close(c Store) { } // NewClient creates a new DB using the provided config. -func NewClient(ctx context.Context, conf config.DB) (Store, error) { +func NewClient(ctx context.Context, conf config.DB) (*Database, error) { dir, err := iofs.New(sqlembed.Migrations, "postgres/migrations") if err != nil { return nil, err diff --git a/pkg/notify/notify.go b/pkg/notify/notify.go index 286c5a5..6711638 100644 --- a/pkg/notify/notify.go +++ b/pkg/notify/notify.go @@ -205,7 +205,7 @@ func (n *notifier) Send(ctx context.Context, alerts []alert.Alert) error { return nil } -func New(cfg config.Notify) (Notifier, error) { +func New(cfg config.Notify) (*notifier, error) { n := new(notifier) for _, s := range cfg { diff --git a/pkg/rest/api.go b/pkg/rest/api.go index 8db4f98..2e9b5ee 100644 --- a/pkg/rest/api.go +++ b/pkg/rest/api.go @@ -20,7 +20,7 @@ type API interface { type api struct { } -func New() API { +func New() *api { s := new(api) return s diff --git a/pkg/sinks/sinks.go b/pkg/sinks/sinks.go index 8259f30..4992a60 100644 --- a/pkg/sinks/sinks.go +++ b/pkg/sinks/sinks.go @@ -36,7 +36,7 @@ type sinks struct { sinks map[string]sinkInstance } -func NewSinkManager() Sinks { +func NewSinkManager() *sinks { return &sinks{ sinks: make(map[string]sinkInstance), } diff --git a/pkg/talkgroups/tgstore/store.go b/pkg/talkgroups/tgstore/store.go index 81ef67c..9c9a59b 100644 --- a/pkg/talkgroups/tgstore/store.go +++ b/pkg/talkgroups/tgstore/store.go @@ -150,7 +150,7 @@ type cache struct { } // NewCache returns a new cache Store. -func NewCache() Store { +func NewCache() *cache { tgc := &cache{ tgs: make(tgMap), systems: make(map[int32]string), -- 2.48.1 From a22d45dbb533746caef22f0373319fe862dc19cb Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Thu, 28 Nov 2024 10:11:46 -0500 Subject: [PATCH 02/15] pretest --- pkg/config/config.go | 42 ++++ pkg/database/calls.sql.go | 25 ++ pkg/database/database.go | 55 +++- pkg/database/mocks/Store.go | 230 ++++++++++++++++- pkg/database/models.go | 21 ++ pkg/database/partman/partman.go | 249 +++++++++++++++++++ pkg/database/querier.go | 1 + sql/postgres/migrations/002_partition.up.sql | 28 +++ sql/postgres/queries/calls.sql | 4 + 9 files changed, 641 insertions(+), 14 deletions(-) create mode 100644 pkg/database/partman/partman.go create mode 100644 sql/postgres/migrations/002_partition.up.sql diff --git a/pkg/config/config.go b/pkg/config/config.go index 7d67a9b..ae53c66 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -41,6 +41,48 @@ type CORS struct { type DB struct { Connect string `yaml:"connect"` LogQueries bool `yaml:"logQueries"` + Partition Partition `yaml:"partition"` +} + +type PartitionInterval string +const ( + PartDaily PartitionInterval = "daily" + PartWeekly PartitionInterval = "weekly" + PartMonthly PartitionInterval = "monthly" + PartQuarterly PartitionInterval = "quarterly" + PartYearly PartitionInterval = "yearly" +) + +func (p PartitionInterval) IsValid() bool { + switch p { + case PartDaily, PartWeekly, PartMonthly, PartQuarterly, PartYearly: + return true + } + + return false +} + +type PartitionPolicy string +const ( + PartPolDetach PartitionPolicy = "detach" + PartPolDrop PartitionPolicy = "drop" +) + +func (p PartitionPolicy) IsValid() bool { + switch p { + case PartPolDetach, PartPolDrop: + return true + } + + return false +} + +type Partition struct { + Enabled bool `yaml:"enabled"` + Schema string `yaml:"schema"` + Interval PartitionInterval `yaml:"interval"` + Retain int `yaml:"retain"` + Detach bool `yaml:"detach"` } type Logger struct { diff --git a/pkg/database/calls.sql.go b/pkg/database/calls.sql.go index 2c82ea8..7e535d0 100644 --- a/pkg/database/calls.sql.go +++ b/pkg/database/calls.sql.go @@ -135,6 +135,31 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) error { return err } +const getCallsPartitions = `-- name: GetCallsPartitions :many +SELECT tablename FROM pg_catalog.pg_tables +WHERE schemaname = $1 AND tablename LIKE 'calls\_p\_____\___' +` + +func (q *Queries) GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) { + rows, err := q.db.Query(ctx, getCallsPartitions, schemaName) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*string + for rows.Next() { + var tablename *string + if err := rows.Scan(&tablename); err != nil { + return nil, err + } + items = append(items, tablename) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getDatabaseSize = `-- name: GetDatabaseSize :one SELECT pg_size_pretty(pg_database_size(current_database())) ` diff --git a/pkg/database/database.go b/pkg/database/database.go index 7f38fd1..8f84ad1 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -24,20 +24,28 @@ type Store interface { Querier talkgroupQuerier - DB() *Database + GetSearchPath(ctx context.Context) (string, error) + GetSchemaName(ctx context.Context) (string, error) + + DB() *Postgres + DBTX() DBTX InTx(context.Context, func(Store) error, pgx.TxOptions) error } -type Database struct { +type Postgres struct { *pgxpool.Pool *Queries } -func (db *Database) DB() *Database { +func (q *Queries) DBTX() DBTX { + return q.db +} + +func (db *Postgres) DB() *Postgres { return db } -func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOptions) error { +func (db *Postgres) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOptions) error { tx, err := db.DB().Pool.BeginTx(ctx, opts) if err != nil { return fmt.Errorf("Tx begin: %w", err) @@ -46,7 +54,7 @@ func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOp //nolint:errcheck defer tx.Rollback(ctx) - dbtx := &Database{Pool: db.Pool, Queries: db.Queries.WithTx(tx)} + dbtx := &Postgres{Pool: db.Pool, Queries: db.Queries.WithTx(tx)} err = f(dbtx) if err != nil { @@ -61,6 +69,37 @@ func (db *Database) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOp return nil } +func (db *Postgres) GetSearchPath(ctx context.Context) (string, error) { + var searchPath *string + err := db.QueryRow(ctx, `SHOW SEARCH_PATH;`).Scan(&searchPath) + if err != nil { + return "", err + } + + if searchPath == nil { + return "", errors.New("search path was null!") + } + + return *searchPath, err +} + +func (db *Postgres) GetSchemaName(ctx context.Context) (string, error) { + searchPath, err := db.GetSearchPath(ctx) + if err != nil { + return "", err + } + // we only support either the default SEARCH_PATH or one + // with only one element in it that is not a variable substitution. + + schemaName := "public" + if searchPath != "$user, public" { + sar := strings.Split(searchPath, ",") + schemaName = sar[0] + } + + return schemaName, nil +} + type dbLogger struct{} func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) { @@ -68,11 +107,11 @@ func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, } func Close(c Store) { - c.(*Database).Pool.Close() + c.(*Postgres).Pool.Close() } // NewClient creates a new DB using the provided config. -func NewClient(ctx context.Context, conf config.DB) (*Database, error) { +func NewClient(ctx context.Context, conf config.DB) (*Postgres, error) { dir, err := iofs.New(sqlembed.Migrations, "postgres/migrations") if err != nil { return nil, err @@ -107,7 +146,7 @@ func NewClient(ctx context.Context, conf config.DB) (*Database, error) { return nil, err } - db := &Database{ + db := &Postgres{ Pool: pool, Queries: New(pool), } diff --git a/pkg/database/mocks/Store.go b/pkg/database/mocks/Store.go index 9630a74..8f01f33 100644 --- a/pkg/database/mocks/Store.go +++ b/pkg/database/mocks/Store.go @@ -344,19 +344,19 @@ func (_c *Store_CreateUser_Call) RunAndReturn(run func(context.Context, database } // DB provides a mock function with given fields: -func (_m *Store) DB() *database.Database { +func (_m *Store) DB() *database.Postgres { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for DB") } - var r0 *database.Database - if rf, ok := ret.Get(0).(func() *database.Database); ok { + var r0 *database.Postgres + if rf, ok := ret.Get(0).(func() *database.Postgres); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*database.Database) + r0 = ret.Get(0).(*database.Postgres) } } @@ -380,12 +380,59 @@ func (_c *Store_DB_Call) Run(run func()) *Store_DB_Call { return _c } -func (_c *Store_DB_Call) Return(_a0 *database.Database) *Store_DB_Call { +func (_c *Store_DB_Call) Return(_a0 *database.Postgres) *Store_DB_Call { _c.Call.Return(_a0) return _c } -func (_c *Store_DB_Call) RunAndReturn(run func() *database.Database) *Store_DB_Call { +func (_c *Store_DB_Call) RunAndReturn(run func() *database.Postgres) *Store_DB_Call { + _c.Call.Return(run) + return _c +} + +// DBTX provides a mock function with given fields: +func (_m *Store) DBTX() database.DBTX { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for DBTX") + } + + var r0 database.DBTX + if rf, ok := ret.Get(0).(func() database.DBTX); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(database.DBTX) + } + } + + return r0 +} + +// Store_DBTX_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DBTX' +type Store_DBTX_Call struct { + *mock.Call +} + +// DBTX is a helper method to define mock.On call +func (_e *Store_Expecter) DBTX() *Store_DBTX_Call { + return &Store_DBTX_Call{Call: _e.mock.On("DBTX")} +} + +func (_c *Store_DBTX_Call) Run(run func()) *Store_DBTX_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Store_DBTX_Call) Return(_a0 database.DBTX) *Store_DBTX_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_DBTX_Call) RunAndReturn(run func() database.DBTX) *Store_DBTX_Call { _c.Call.Return(run) return _c } @@ -541,6 +588,65 @@ func (_c *Store_GetAPIKey_Call) RunAndReturn(run func(context.Context, string) ( return _c } +// GetCallsPartitions provides a mock function with given fields: ctx, schemaName +func (_m *Store) GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) { + ret := _m.Called(ctx, schemaName) + + if len(ret) == 0 { + panic("no return value specified for GetCallsPartitions") + } + + var r0 []*string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *string) ([]*string, error)); ok { + return rf(ctx, schemaName) + } + if rf, ok := ret.Get(0).(func(context.Context, *string) []*string); ok { + r0 = rf(ctx, schemaName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*string) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *string) error); ok { + r1 = rf(ctx, schemaName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Store_GetCallsPartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCallsPartitions' +type Store_GetCallsPartitions_Call struct { + *mock.Call +} + +// GetCallsPartitions is a helper method to define mock.On call +// - ctx context.Context +// - schemaName *string +func (_e *Store_Expecter) GetCallsPartitions(ctx interface{}, schemaName interface{}) *Store_GetCallsPartitions_Call { + return &Store_GetCallsPartitions_Call{Call: _e.mock.On("GetCallsPartitions", ctx, schemaName)} +} + +func (_c *Store_GetCallsPartitions_Call) Run(run func(ctx context.Context, schemaName *string)) *Store_GetCallsPartitions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*string)) + }) + return _c +} + +func (_c *Store_GetCallsPartitions_Call) Return(_a0 []*string, _a1 error) *Store_GetCallsPartitions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Store_GetCallsPartitions_Call) RunAndReturn(run func(context.Context, *string) ([]*string, error)) *Store_GetCallsPartitions_Call { + _c.Call.Return(run) + return _c +} + // GetDatabaseSize provides a mock function with given fields: ctx func (_m *Store) GetDatabaseSize(ctx context.Context) (string, error) { ret := _m.Called(ctx) @@ -597,6 +703,118 @@ func (_c *Store_GetDatabaseSize_Call) RunAndReturn(run func(context.Context) (st return _c } +// GetSchemaName provides a mock function with given fields: ctx +func (_m *Store) GetSchemaName(ctx context.Context) (string, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetSchemaName") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) string); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Store_GetSchemaName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSchemaName' +type Store_GetSchemaName_Call struct { + *mock.Call +} + +// GetSchemaName is a helper method to define mock.On call +// - ctx context.Context +func (_e *Store_Expecter) GetSchemaName(ctx interface{}) *Store_GetSchemaName_Call { + return &Store_GetSchemaName_Call{Call: _e.mock.On("GetSchemaName", ctx)} +} + +func (_c *Store_GetSchemaName_Call) Run(run func(ctx context.Context)) *Store_GetSchemaName_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Store_GetSchemaName_Call) Return(_a0 string, _a1 error) *Store_GetSchemaName_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Store_GetSchemaName_Call) RunAndReturn(run func(context.Context) (string, error)) *Store_GetSchemaName_Call { + _c.Call.Return(run) + return _c +} + +// GetSearchPath provides a mock function with given fields: ctx +func (_m *Store) GetSearchPath(ctx context.Context) (string, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetSearchPath") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) string); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Store_GetSearchPath_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSearchPath' +type Store_GetSearchPath_Call struct { + *mock.Call +} + +// GetSearchPath is a helper method to define mock.On call +// - ctx context.Context +func (_e *Store_Expecter) GetSearchPath(ctx interface{}) *Store_GetSearchPath_Call { + return &Store_GetSearchPath_Call{Call: _e.mock.On("GetSearchPath", ctx)} +} + +func (_c *Store_GetSearchPath_Call) Run(run func(ctx context.Context)) *Store_GetSearchPath_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Store_GetSearchPath_Call) Return(_a0 string, _a1 error) *Store_GetSearchPath_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Store_GetSearchPath_Call) RunAndReturn(run func(context.Context) (string, error)) *Store_GetSearchPath_Call { + _c.Call.Return(run) + return _c +} + // GetSystemName provides a mock function with given fields: ctx, systemID func (_m *Store) GetSystemName(ctx context.Context, systemID int) (string, error) { ret := _m.Called(ctx, systemID) diff --git a/pkg/database/models.go b/pkg/database/models.go index 042b84b..e4e0caf 100644 --- a/pkg/database/models.go +++ b/pkg/database/models.go @@ -55,6 +55,27 @@ type Call struct { Transcript *string `json:"transcript,omitempty"` } +type CallsUnpart struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + type Incident struct { ID uuid.UUID `json:"id,omitempty"` Name string `json:"name,omitempty"` diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go new file mode 100644 index 0000000..e1b092d --- /dev/null +++ b/pkg/database/partman/partman.go @@ -0,0 +1,249 @@ +package partman + +import ( + "fmt" + "context" + "strconv" + "strings" + "time" + + "dynatron.me/x/stillbox/pkg/config" + "dynatron.me/x/stillbox/pkg/database" + + "github.com/jackc/pgx/v5" + "github.com/rs/zerolog/log" +) + +type PartitionError string + +func (pe PartitionError) Error() string { + return fmt.Sprintf("bad partition '%s'", string(pe)) +} + +type ErrInvalidInterval string +func (e ErrInvalidInterval) Error() string { return fmt.Sprintf("invalid interval '%s'", string(e)) } + +type Interval string +const ( + Daily Interval = "daily" + Weekly Interval = "weekly" + Monthly Interval = "monthly" + Quarterly Interval = "quarterly" + Yearly Interval = "yearly" +) + +func (p Interval) IsValid() bool { + switch p { + case Daily, Weekly, Monthly, Quarterly, Yearly: + return true + } + + return false +} + + + +type PartitionManager interface { + Go(ctx context.Context) + Check(ctx context.Context) error +} + +type partman struct { + db *database.Postgres + cfg config.Partition + intv Interval +} + + +type partition interface { + PartitionName() string + Next() partition + Range() (string, string) +} + +type monthlyPartition struct { + year int + month time.Month +} + +func (d monthlyPartition) PartitionName() string { + return fmt.Sprintf("calls_p_%d_%02d", d.year, d.month) +} + +func (d monthlyPartition) Next() partition { + return d.next() +} + +func (d monthlyPartition) next() monthlyPartition { + if d.month == time.December { + d.year++ + d.month = time.January + } else { + d.month++ + } + + return d +} + +func (d monthlyPartition) Range() (string, string) { + next := d.next() + return fmt.Sprintf("%d-%02d-01", d.year, d.month), fmt.Sprintf("%d-%02d-01", next.year, next.month) +} + +func NewPartman(db *database.Postgres, cfg config.Partition) (*partman, error) { + pm := &partman{ + cfg: cfg, + db: db, + intv: Interval(cfg.Interval), + } + + if !pm.intv.IsValid() { + return nil, ErrInvalidInterval(pm.intv) + } + + return pm, nil +} + +var _ PartitionManager = (*partman)(nil) + +func (pm *partman) Go(ctx context.Context) { + go func(ctx context.Context) { + tick := time.NewTicker(60*time.Minute) + + select { + case <-tick.C: + err := pm.Check(ctx) + if err != nil { + log.Error().Err(err).Msg("partman check failed") + } + case <-ctx.Done(): + return + } + }(ctx) +} + +func (pm *partman) newPartition(t time.Time) partition { + return monthlyPartition{month: t.Month(), year: t.Year()} +} + +func (pm *partman) Check(ctx context.Context) error { + err := pm.db.InTx(ctx, func(db database.Store) error { + // by default, we want to make sure a partition exists for this and next month + // since we run this at startup, it's safe to do only that. + schemaName, err := db.GetSearchPath(ctx) + if err != nil { + return err + } + + partitions, err := db.GetCallsPartitions(ctx, &schemaName) + if err != nil { + return err + } + + // this could be done in SQL + partsMap, err := partitionsMap(partitions) + if err != nil { + return err + } + + now := time.Now() + + thisPart := pm.newPartition(now) + nextPart := thisPart.Next() + + mustExist := []partition{thisPart, nextPart} + + for _, ch := range mustExist { + if !partsMap.exists(ch) { + + err := pm.createPartition(ctx, db, ch) + if err != nil { + return err + } + + log.Info().Str("part", ch.PartitionName()).Msg("created partition") + } + } + + return nil + + }, pgx.TxOptions{}) + + if err != nil { + return err + } + + return nil +} + +func parsePartName(p *string) (monthlyPartition, error) { + r := monthlyPartition{} + dateAr := strings.Split(*p, "calls_p_") + if len(dateAr) != 2 { + return r, PartitionError(*p) + } + + dateAr = strings.Split(dateAr[1], "_") + if len(dateAr) != 2 { + return r, PartitionError(*p) + } + + year, err := strconv.Atoi(dateAr[0]) + if err != nil { + return r, PartitionError(*p) + } + + r.year = year + + month, err := strconv.Atoi(dateAr[1]) + if err != nil { + return r, PartitionError(*p) + } + + r.month = time.Month(month) + + return r, nil +} + +type partMap map[partition]struct{} + +func (pm partMap) exists(dt partition) bool { + _ , ex := pm[dt] + + return ex +} + +func partitionsMap(partitions []*string) (partMap, error) { + partsDate := make(partMap, len(partitions)) + + for _, p := range partitions { + if p == nil { + panic(PartitionError("")) + } + + dt, err := parsePartName(p) + if err != nil { + return nil, err + } + + partsDate[dt] = struct{}{} + } + + + return partsDate, nil +} + +func (pm *partman) createPartition(ctx context.Context, db database.Store, d partition) error { + t, n := d.Range() + + _, err := db.DBTX().Exec(ctx, fmt.Sprintf(`CREATE TABLE %s PARTITION OF calls FOR VALUES FROM ($1) TO ($2);`, d.PartitionName()), t, n) + if err != nil { + return err + } + + return nil +} + +func monthPart(m time.Month, y int) monthlyPartition { + return monthlyPartition{year: y, month: m} +} diff --git a/pkg/database/querier.go b/pkg/database/querier.go index 167df43..911cc22 100644 --- a/pkg/database/querier.go +++ b/pkg/database/querier.go @@ -20,6 +20,7 @@ type Querier interface { DeleteAPIKey(ctx context.Context, apiKey string) error DeleteUser(ctx context.Context, username string) error GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error) + GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) GetDatabaseSize(ctx context.Context) (string, error) GetSystemName(ctx context.Context, systemID int) (string, error) GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error) diff --git a/sql/postgres/migrations/002_partition.up.sql b/sql/postgres/migrations/002_partition.up.sql new file mode 100644 index 0000000..7ae6d80 --- /dev/null +++ b/sql/postgres/migrations/002_partition.up.sql @@ -0,0 +1,28 @@ +BEGIN; + +ALTER TABLE calls RENAME TO calls_unpart; + +CREATE TABLE calls ( + id UUID, + submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, + system INTEGER NOT NULL, + talkgroup INTEGER NOT NULL, + call_date TIMESTAMPTZ NOT NULL, + audio_name TEXT, + audio_blob BYTEA, + duration INTEGER, + audio_type TEXT, + audio_url TEXT, + frequency INTEGER NOT NULL, + frequencies INTEGER[], + patches INTEGER[], + tg_label TEXT, + tg_alpha_tag TEXT, + tg_group TEXT, + source INTEGER NOT NULL, + transcript TEXT, + PRIMARY KEY (id, call_date), + FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) +) PARTITION BY RANGE (call_date); + +COMMIT; diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index 5253556..a78ce12 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -56,3 +56,7 @@ VALUES -- name: GetDatabaseSize :one SELECT pg_size_pretty(pg_database_size(current_database())); + +-- name: GetCallsPartitions :many +SELECT tablename FROM pg_catalog.pg_tables +WHERE schemaname = @schema_name AND tablename LIKE 'calls\_p\_____\___'; -- 2.48.1 From 9c63368c7d4600a98aeed09f8dc7a94c130105f6 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Fri, 29 Nov 2024 19:22:50 -0500 Subject: [PATCH 03/15] partman monthly test passes --- .mockery.yaml | 1 + pkg/config/config.go | 48 +- pkg/database/calls.sql.go | 47 +- pkg/database/database.go | 35 +- pkg/database/mocks/DBTX.go | 287 +++++++++++ pkg/database/mocks/Store.go | 473 ++++++++++++------- pkg/database/models.go | 116 ++++- pkg/database/partitions.go | 87 ++++ pkg/database/partman/partman.go | 312 ++++++++---- pkg/database/partman/partman_test.go | 148 ++++++ pkg/database/querier.go | 3 +- sql/postgres/migrations/002_partition.up.sql | 52 ++ sql/postgres/queries/calls.sql | 20 +- sql/sqlc.yaml | 3 + 14 files changed, 1262 insertions(+), 370 deletions(-) create mode 100644 pkg/database/mocks/DBTX.go create mode 100644 pkg/database/partitions.go create mode 100644 pkg/database/partman/partman_test.go diff --git a/.mockery.yaml b/.mockery.yaml index fa78e2c..a0b5082 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -8,3 +8,4 @@ packages: config: interfaces: Store: + DBTX: diff --git a/pkg/config/config.go b/pkg/config/config.go index ae53c66..5a8d110 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,50 +39,18 @@ type CORS struct { } type DB struct { - Connect string `yaml:"connect"` - LogQueries bool `yaml:"logQueries"` + Connect string `yaml:"connect"` + LogQueries bool `yaml:"logQueries"` Partition Partition `yaml:"partition"` } -type PartitionInterval string -const ( - PartDaily PartitionInterval = "daily" - PartWeekly PartitionInterval = "weekly" - PartMonthly PartitionInterval = "monthly" - PartQuarterly PartitionInterval = "quarterly" - PartYearly PartitionInterval = "yearly" -) - -func (p PartitionInterval) IsValid() bool { - switch p { - case PartDaily, PartWeekly, PartMonthly, PartQuarterly, PartYearly: - return true - } - - return false -} - -type PartitionPolicy string -const ( - PartPolDetach PartitionPolicy = "detach" - PartPolDrop PartitionPolicy = "drop" -) - -func (p PartitionPolicy) IsValid() bool { - switch p { - case PartPolDetach, PartPolDrop: - return true - } - - return false -} - type Partition struct { - Enabled bool `yaml:"enabled"` - Schema string `yaml:"schema"` - Interval PartitionInterval `yaml:"interval"` - Retain int `yaml:"retain"` - Detach bool `yaml:"detach"` + Enabled bool `yaml:"enabled"` + Schema string `yaml:"schema"` + Interval string `yaml:"interval"` + Retain int `yaml:"retain"` + PreProvision *int `yaml:"preProvision"` + Drop bool `yaml:"detach"` } type Logger struct { diff --git a/pkg/database/calls.sql.go b/pkg/database/calls.sql.go index 7e535d0..308f7f1 100644 --- a/pkg/database/calls.sql.go +++ b/pkg/database/calls.sql.go @@ -135,29 +135,21 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) error { return err } -const getCallsPartitions = `-- name: GetCallsPartitions :many -SELECT tablename FROM pg_catalog.pg_tables -WHERE schemaname = $1 AND tablename LIKE 'calls\_p\_____\___' +const cleanupSweptCalls = `-- name: CleanupSweptCalls :exec +WITH to_sweep AS ( + SELECT id FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= $1 AND calls.call_date < $2 +) UPDATE incidents_calls + SET + swept_call_id = call_id, + calls_tbl_id = NULL + WHERE call_id IN (SELECT id FROM to_sweep) ` -func (q *Queries) GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) { - rows, err := q.db.Query(ctx, getCallsPartitions, schemaName) - if err != nil { - return nil, err - } - defer rows.Close() - var items []*string - for rows.Next() { - var tablename *string - if err := rows.Scan(&tablename); err != nil { - return nil, err - } - items = append(items, tablename) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil +func (q *Queries) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, cleanupSweptCalls, rangeStart, rangeEnd) + return err } const getDatabaseSize = `-- name: GetDatabaseSize :one @@ -179,3 +171,16 @@ func (q *Queries) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcrip _, err := q.db.Exec(ctx, setCallTranscript, iD, transcript) return err } + +const sweepCalls = `-- name: SweepCalls :exec +WITH to_sweep AS ( + SELECT id, submitter, system, talkgroup, calls.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, ic.call_date, notes FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= $1 AND calls.call_date < $2 +) INSERT INTO swept_calls SELECT id, submitter, system, talkgroup, to_sweep.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, to_sweep.call_date, notes FROM to_sweep +` + +func (q *Queries) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, sweepCalls, rangeStart, rangeEnd) + return err +} diff --git a/pkg/database/database.go b/pkg/database/database.go index 8f84ad1..fa3a25b 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -23,9 +23,7 @@ import ( type Store interface { Querier talkgroupQuerier - - GetSearchPath(ctx context.Context) (string, error) - GetSchemaName(ctx context.Context) (string, error) + partitionsQuerier DB() *Postgres DBTX() DBTX @@ -69,37 +67,6 @@ func (db *Postgres) InTx(ctx context.Context, f func(Store) error, opts pgx.TxOp return nil } -func (db *Postgres) GetSearchPath(ctx context.Context) (string, error) { - var searchPath *string - err := db.QueryRow(ctx, `SHOW SEARCH_PATH;`).Scan(&searchPath) - if err != nil { - return "", err - } - - if searchPath == nil { - return "", errors.New("search path was null!") - } - - return *searchPath, err -} - -func (db *Postgres) GetSchemaName(ctx context.Context) (string, error) { - searchPath, err := db.GetSearchPath(ctx) - if err != nil { - return "", err - } - // we only support either the default SEARCH_PATH or one - // with only one element in it that is not a variable substitution. - - schemaName := "public" - if searchPath != "$user, public" { - sar := strings.Split(searchPath, ",") - schemaName = sar[0] - } - - return schemaName, nil -} - type dbLogger struct{} func (m dbLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) { diff --git a/pkg/database/mocks/DBTX.go b/pkg/database/mocks/DBTX.go new file mode 100644 index 0000000..4de20fe --- /dev/null +++ b/pkg/database/mocks/DBTX.go @@ -0,0 +1,287 @@ +// Code generated by mockery v2.47.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + pgconn "github.com/jackc/pgx/v5/pgconn" + + pgx "github.com/jackc/pgx/v5" +) + +// DBTX is an autogenerated mock type for the DBTX type +type DBTX struct { + mock.Mock +} + +type DBTX_Expecter struct { + mock *mock.Mock +} + +func (_m *DBTX) EXPECT() *DBTX_Expecter { + return &DBTX_Expecter{mock: &_m.Mock} +} + +// Exec provides a mock function with given fields: _a0, _a1, _a2 +func (_m *DBTX) Exec(_a0 context.Context, _a1 string, _a2 ...interface{}) (pgconn.CommandTag, error) { + var _ca []interface{} + _ca = append(_ca, _a0, _a1) + _ca = append(_ca, _a2...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Exec") + } + + var r0 pgconn.CommandTag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, ...interface{}) (pgconn.CommandTag, error)); ok { + return rf(_a0, _a1, _a2...) + } + if rf, ok := ret.Get(0).(func(context.Context, string, ...interface{}) pgconn.CommandTag); ok { + r0 = rf(_a0, _a1, _a2...) + } else { + r0 = ret.Get(0).(pgconn.CommandTag) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, ...interface{}) error); ok { + r1 = rf(_a0, _a1, _a2...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DBTX_Exec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Exec' +type DBTX_Exec_Call struct { + *mock.Call +} + +// Exec is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +// - _a2 ...interface{} +func (_e *DBTX_Expecter) Exec(_a0 interface{}, _a1 interface{}, _a2 ...interface{}) *DBTX_Exec_Call { + return &DBTX_Exec_Call{Call: _e.mock.On("Exec", + append([]interface{}{_a0, _a1}, _a2...)...)} +} + +func (_c *DBTX_Exec_Call) Run(run func(_a0 context.Context, _a1 string, _a2 ...interface{})) *DBTX_Exec_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]interface{}, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(interface{}) + } + } + run(args[0].(context.Context), args[1].(string), variadicArgs...) + }) + return _c +} + +func (_c *DBTX_Exec_Call) Return(_a0 pgconn.CommandTag, _a1 error) *DBTX_Exec_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DBTX_Exec_Call) RunAndReturn(run func(context.Context, string, ...interface{}) (pgconn.CommandTag, error)) *DBTX_Exec_Call { + _c.Call.Return(run) + return _c +} + +// Query provides a mock function with given fields: _a0, _a1, _a2 +func (_m *DBTX) Query(_a0 context.Context, _a1 string, _a2 ...interface{}) (pgx.Rows, error) { + var _ca []interface{} + _ca = append(_ca, _a0, _a1) + _ca = append(_ca, _a2...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Query") + } + + var r0 pgx.Rows + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, ...interface{}) (pgx.Rows, error)); ok { + return rf(_a0, _a1, _a2...) + } + if rf, ok := ret.Get(0).(func(context.Context, string, ...interface{}) pgx.Rows); ok { + r0 = rf(_a0, _a1, _a2...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(pgx.Rows) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, ...interface{}) error); ok { + r1 = rf(_a0, _a1, _a2...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DBTX_Query_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Query' +type DBTX_Query_Call struct { + *mock.Call +} + +// Query is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +// - _a2 ...interface{} +func (_e *DBTX_Expecter) Query(_a0 interface{}, _a1 interface{}, _a2 ...interface{}) *DBTX_Query_Call { + return &DBTX_Query_Call{Call: _e.mock.On("Query", + append([]interface{}{_a0, _a1}, _a2...)...)} +} + +func (_c *DBTX_Query_Call) Run(run func(_a0 context.Context, _a1 string, _a2 ...interface{})) *DBTX_Query_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]interface{}, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(interface{}) + } + } + run(args[0].(context.Context), args[1].(string), variadicArgs...) + }) + return _c +} + +func (_c *DBTX_Query_Call) Return(_a0 pgx.Rows, _a1 error) *DBTX_Query_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DBTX_Query_Call) RunAndReturn(run func(context.Context, string, ...interface{}) (pgx.Rows, error)) *DBTX_Query_Call { + _c.Call.Return(run) + return _c +} + +// QueryRow provides a mock function with given fields: _a0, _a1, _a2 +func (_m *DBTX) QueryRow(_a0 context.Context, _a1 string, _a2 ...interface{}) pgx.Row { + var _ca []interface{} + _ca = append(_ca, _a0, _a1) + _ca = append(_ca, _a2...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for QueryRow") + } + + var r0 pgx.Row + if rf, ok := ret.Get(0).(func(context.Context, string, ...interface{}) pgx.Row); ok { + r0 = rf(_a0, _a1, _a2...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(pgx.Row) + } + } + + return r0 +} + +// DBTX_QueryRow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueryRow' +type DBTX_QueryRow_Call struct { + *mock.Call +} + +// QueryRow is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +// - _a2 ...interface{} +func (_e *DBTX_Expecter) QueryRow(_a0 interface{}, _a1 interface{}, _a2 ...interface{}) *DBTX_QueryRow_Call { + return &DBTX_QueryRow_Call{Call: _e.mock.On("QueryRow", + append([]interface{}{_a0, _a1}, _a2...)...)} +} + +func (_c *DBTX_QueryRow_Call) Run(run func(_a0 context.Context, _a1 string, _a2 ...interface{})) *DBTX_QueryRow_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]interface{}, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(interface{}) + } + } + run(args[0].(context.Context), args[1].(string), variadicArgs...) + }) + return _c +} + +func (_c *DBTX_QueryRow_Call) Return(_a0 pgx.Row) *DBTX_QueryRow_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DBTX_QueryRow_Call) RunAndReturn(run func(context.Context, string, ...interface{}) pgx.Row) *DBTX_QueryRow_Call { + _c.Call.Return(run) + return _c +} + +// SendBatch provides a mock function with given fields: _a0, _a1 +func (_m *DBTX) SendBatch(_a0 context.Context, _a1 *pgx.Batch) pgx.BatchResults { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for SendBatch") + } + + var r0 pgx.BatchResults + if rf, ok := ret.Get(0).(func(context.Context, *pgx.Batch) pgx.BatchResults); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(pgx.BatchResults) + } + } + + return r0 +} + +// DBTX_SendBatch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendBatch' +type DBTX_SendBatch_Call struct { + *mock.Call +} + +// SendBatch is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *pgx.Batch +func (_e *DBTX_Expecter) SendBatch(_a0 interface{}, _a1 interface{}) *DBTX_SendBatch_Call { + return &DBTX_SendBatch_Call{Call: _e.mock.On("SendBatch", _a0, _a1)} +} + +func (_c *DBTX_SendBatch_Call) Run(run func(_a0 context.Context, _a1 *pgx.Batch)) *DBTX_SendBatch_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*pgx.Batch)) + }) + return _c +} + +func (_c *DBTX_SendBatch_Call) Return(_a0 pgx.BatchResults) *DBTX_SendBatch_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DBTX_SendBatch_Call) RunAndReturn(run func(context.Context, *pgx.Batch) pgx.BatchResults) *DBTX_SendBatch_Call { + _c.Call.Return(run) + return _c +} + +// NewDBTX creates a new instance of DBTX. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewDBTX(t interface { + mock.TestingT + Cleanup(func()) +}) *DBTX { + mock := &DBTX{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/database/mocks/Store.go b/pkg/database/mocks/Store.go index 8f01f33..29c5fbe 100644 --- a/pkg/database/mocks/Store.go +++ b/pkg/database/mocks/Store.go @@ -12,6 +12,8 @@ import ( pgx "github.com/jackc/pgx/v5" + time "time" + uuid "github.com/google/uuid" ) @@ -227,6 +229,54 @@ func (_c *Store_BulkSetTalkgroupTags_Call) RunAndReturn(run func(context.Context return _c } +// CleanupSweptCalls provides a mock function with given fields: ctx, rangeStart, rangeEnd +func (_m *Store) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { + ret := _m.Called(ctx, rangeStart, rangeEnd) + + if len(ret) == 0 { + panic("no return value specified for CleanupSweptCalls") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + r0 = rf(ctx, rangeStart, rangeEnd) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Store_CleanupSweptCalls_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanupSweptCalls' +type Store_CleanupSweptCalls_Call struct { + *mock.Call +} + +// CleanupSweptCalls is a helper method to define mock.On call +// - ctx context.Context +// - rangeStart pgtype.Timestamptz +// - rangeEnd pgtype.Timestamptz +func (_e *Store_Expecter) CleanupSweptCalls(ctx interface{}, rangeStart interface{}, rangeEnd interface{}) *Store_CleanupSweptCalls_Call { + return &Store_CleanupSweptCalls_Call{Call: _e.mock.On("CleanupSweptCalls", ctx, rangeStart, rangeEnd)} +} + +func (_c *Store_CleanupSweptCalls_Call) Run(run func(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz)) *Store_CleanupSweptCalls_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(pgtype.Timestamptz), args[2].(pgtype.Timestamptz)) + }) + return _c +} + +func (_c *Store_CleanupSweptCalls_Call) Return(_a0 error) *Store_CleanupSweptCalls_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_CleanupSweptCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error) *Store_CleanupSweptCalls_Call { + _c.Call.Return(run) + return _c +} + // CreateAPIKey provides a mock function with given fields: ctx, owner, expires, disabled func (_m *Store) CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (database.ApiKey, error) { ret := _m.Called(ctx, owner, expires, disabled) @@ -286,6 +336,56 @@ func (_c *Store_CreateAPIKey_Call) RunAndReturn(run func(context.Context, int, p return _c } +// CreatePartition provides a mock function with given fields: ctx, parentTable, partitionName, start, end +func (_m *Store) CreatePartition(ctx context.Context, parentTable string, partitionName string, start time.Time, end time.Time) error { + ret := _m.Called(ctx, parentTable, partitionName, start, end) + + if len(ret) == 0 { + panic("no return value specified for CreatePartition") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, time.Time, time.Time) error); ok { + r0 = rf(ctx, parentTable, partitionName, start, end) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Store_CreatePartition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreatePartition' +type Store_CreatePartition_Call struct { + *mock.Call +} + +// CreatePartition is a helper method to define mock.On call +// - ctx context.Context +// - parentTable string +// - partitionName string +// - start time.Time +// - end time.Time +func (_e *Store_Expecter) CreatePartition(ctx interface{}, parentTable interface{}, partitionName interface{}, start interface{}, end interface{}) *Store_CreatePartition_Call { + return &Store_CreatePartition_Call{Call: _e.mock.On("CreatePartition", ctx, parentTable, partitionName, start, end)} +} + +func (_c *Store_CreatePartition_Call) Run(run func(ctx context.Context, parentTable string, partitionName string, start time.Time, end time.Time)) *Store_CreatePartition_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(time.Time), args[4].(time.Time)) + }) + return _c +} + +func (_c *Store_CreatePartition_Call) Return(_a0 error) *Store_CreatePartition_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_CreatePartition_Call) RunAndReturn(run func(context.Context, string, string, time.Time, time.Time) error) *Store_CreatePartition_Call { + _c.Call.Return(run) + return _c +} + // CreateUser provides a mock function with given fields: ctx, arg func (_m *Store) CreateUser(ctx context.Context, arg database.CreateUserParams) (database.User, error) { ret := _m.Called(ctx, arg) @@ -531,6 +631,100 @@ func (_c *Store_DeleteUser_Call) RunAndReturn(run func(context.Context, string) return _c } +// DetachPartition provides a mock function with given fields: ctx, partitionName +func (_m *Store) DetachPartition(ctx context.Context, partitionName string) error { + ret := _m.Called(ctx, partitionName) + + if len(ret) == 0 { + panic("no return value specified for DetachPartition") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, partitionName) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Store_DetachPartition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DetachPartition' +type Store_DetachPartition_Call struct { + *mock.Call +} + +// DetachPartition is a helper method to define mock.On call +// - ctx context.Context +// - partitionName string +func (_e *Store_Expecter) DetachPartition(ctx interface{}, partitionName interface{}) *Store_DetachPartition_Call { + return &Store_DetachPartition_Call{Call: _e.mock.On("DetachPartition", ctx, partitionName)} +} + +func (_c *Store_DetachPartition_Call) Run(run func(ctx context.Context, partitionName string)) *Store_DetachPartition_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *Store_DetachPartition_Call) Return(_a0 error) *Store_DetachPartition_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_DetachPartition_Call) RunAndReturn(run func(context.Context, string) error) *Store_DetachPartition_Call { + _c.Call.Return(run) + return _c +} + +// DropPartition provides a mock function with given fields: ctx, partitionName +func (_m *Store) DropPartition(ctx context.Context, partitionName string) error { + ret := _m.Called(ctx, partitionName) + + if len(ret) == 0 { + panic("no return value specified for DropPartition") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, partitionName) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Store_DropPartition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DropPartition' +type Store_DropPartition_Call struct { + *mock.Call +} + +// DropPartition is a helper method to define mock.On call +// - ctx context.Context +// - partitionName string +func (_e *Store_Expecter) DropPartition(ctx interface{}, partitionName interface{}) *Store_DropPartition_Call { + return &Store_DropPartition_Call{Call: _e.mock.On("DropPartition", ctx, partitionName)} +} + +func (_c *Store_DropPartition_Call) Run(run func(ctx context.Context, partitionName string)) *Store_DropPartition_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *Store_DropPartition_Call) Return(_a0 error) *Store_DropPartition_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_DropPartition_Call) RunAndReturn(run func(context.Context, string) error) *Store_DropPartition_Call { + _c.Call.Return(run) + return _c +} + // GetAPIKey provides a mock function with given fields: ctx, apiKey func (_m *Store) GetAPIKey(ctx context.Context, apiKey string) (database.ApiKey, error) { ret := _m.Called(ctx, apiKey) @@ -588,65 +782,6 @@ func (_c *Store_GetAPIKey_Call) RunAndReturn(run func(context.Context, string) ( return _c } -// GetCallsPartitions provides a mock function with given fields: ctx, schemaName -func (_m *Store) GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) { - ret := _m.Called(ctx, schemaName) - - if len(ret) == 0 { - panic("no return value specified for GetCallsPartitions") - } - - var r0 []*string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *string) ([]*string, error)); ok { - return rf(ctx, schemaName) - } - if rf, ok := ret.Get(0).(func(context.Context, *string) []*string); ok { - r0 = rf(ctx, schemaName) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*string) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *string) error); ok { - r1 = rf(ctx, schemaName) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Store_GetCallsPartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCallsPartitions' -type Store_GetCallsPartitions_Call struct { - *mock.Call -} - -// GetCallsPartitions is a helper method to define mock.On call -// - ctx context.Context -// - schemaName *string -func (_e *Store_Expecter) GetCallsPartitions(ctx interface{}, schemaName interface{}) *Store_GetCallsPartitions_Call { - return &Store_GetCallsPartitions_Call{Call: _e.mock.On("GetCallsPartitions", ctx, schemaName)} -} - -func (_c *Store_GetCallsPartitions_Call) Run(run func(ctx context.Context, schemaName *string)) *Store_GetCallsPartitions_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*string)) - }) - return _c -} - -func (_c *Store_GetCallsPartitions_Call) Return(_a0 []*string, _a1 error) *Store_GetCallsPartitions_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Store_GetCallsPartitions_Call) RunAndReturn(run func(context.Context, *string) ([]*string, error)) *Store_GetCallsPartitions_Call { - _c.Call.Return(run) - return _c -} - // GetDatabaseSize provides a mock function with given fields: ctx func (_m *Store) GetDatabaseSize(ctx context.Context) (string, error) { ret := _m.Called(ctx) @@ -703,118 +838,6 @@ func (_c *Store_GetDatabaseSize_Call) RunAndReturn(run func(context.Context) (st return _c } -// GetSchemaName provides a mock function with given fields: ctx -func (_m *Store) GetSchemaName(ctx context.Context) (string, error) { - ret := _m.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for GetSchemaName") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) string); ok { - r0 = rf(ctx) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Store_GetSchemaName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSchemaName' -type Store_GetSchemaName_Call struct { - *mock.Call -} - -// GetSchemaName is a helper method to define mock.On call -// - ctx context.Context -func (_e *Store_Expecter) GetSchemaName(ctx interface{}) *Store_GetSchemaName_Call { - return &Store_GetSchemaName_Call{Call: _e.mock.On("GetSchemaName", ctx)} -} - -func (_c *Store_GetSchemaName_Call) Run(run func(ctx context.Context)) *Store_GetSchemaName_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Store_GetSchemaName_Call) Return(_a0 string, _a1 error) *Store_GetSchemaName_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Store_GetSchemaName_Call) RunAndReturn(run func(context.Context) (string, error)) *Store_GetSchemaName_Call { - _c.Call.Return(run) - return _c -} - -// GetSearchPath provides a mock function with given fields: ctx -func (_m *Store) GetSearchPath(ctx context.Context) (string, error) { - ret := _m.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for GetSearchPath") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) string); ok { - r0 = rf(ctx) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Store_GetSearchPath_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSearchPath' -type Store_GetSearchPath_Call struct { - *mock.Call -} - -// GetSearchPath is a helper method to define mock.On call -// - ctx context.Context -func (_e *Store_Expecter) GetSearchPath(ctx interface{}) *Store_GetSearchPath_Call { - return &Store_GetSearchPath_Call{Call: _e.mock.On("GetSearchPath", ctx)} -} - -func (_c *Store_GetSearchPath_Call) Run(run func(ctx context.Context)) *Store_GetSearchPath_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Store_GetSearchPath_Call) Return(_a0 string, _a1 error) *Store_GetSearchPath_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Store_GetSearchPath_Call) RunAndReturn(run func(context.Context) (string, error)) *Store_GetSearchPath_Call { - _c.Call.Return(run) - return _c -} - // GetSystemName provides a mock function with given fields: ctx, systemID func (_m *Store) GetSystemName(ctx context.Context, systemID int) (string, error) { ret := _m.Called(ctx, systemID) @@ -872,6 +895,66 @@ func (_c *Store_GetSystemName_Call) RunAndReturn(run func(context.Context, int) return _c } +// GetTablePartitions provides a mock function with given fields: ctx, schemaName, tableName +func (_m *Store) GetTablePartitions(ctx context.Context, schemaName string, tableName string) ([]string, error) { + ret := _m.Called(ctx, schemaName, tableName) + + if len(ret) == 0 { + panic("no return value specified for GetTablePartitions") + } + + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]string, error)); ok { + return rf(ctx, schemaName, tableName) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []string); ok { + r0 = rf(ctx, schemaName, tableName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, schemaName, tableName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Store_GetTablePartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTablePartitions' +type Store_GetTablePartitions_Call struct { + *mock.Call +} + +// GetTablePartitions is a helper method to define mock.On call +// - ctx context.Context +// - schemaName string +// - tableName string +func (_e *Store_Expecter) GetTablePartitions(ctx interface{}, schemaName interface{}, tableName interface{}) *Store_GetTablePartitions_Call { + return &Store_GetTablePartitions_Call{Call: _e.mock.On("GetTablePartitions", ctx, schemaName, tableName)} +} + +func (_c *Store_GetTablePartitions_Call) Run(run func(ctx context.Context, schemaName string, tableName string)) *Store_GetTablePartitions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *Store_GetTablePartitions_Call) Return(_a0 []string, _a1 error) *Store_GetTablePartitions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Store_GetTablePartitions_Call) RunAndReturn(run func(context.Context, string, string) ([]string, error)) *Store_GetTablePartitions_Call { + _c.Call.Return(run) + return _c +} + // GetTalkgroup provides a mock function with given fields: ctx, systemID, tGID func (_m *Store) GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (database.GetTalkgroupRow, error) { ret := _m.Called(ctx, systemID, tGID) @@ -2176,6 +2259,54 @@ func (_c *Store_StoreTGVersion_Call) RunAndReturn(run func(context.Context, []da return _c } +// SweepCalls provides a mock function with given fields: ctx, rangeStart, rangeEnd +func (_m *Store) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { + ret := _m.Called(ctx, rangeStart, rangeEnd) + + if len(ret) == 0 { + panic("no return value specified for SweepCalls") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + r0 = rf(ctx, rangeStart, rangeEnd) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Store_SweepCalls_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SweepCalls' +type Store_SweepCalls_Call struct { + *mock.Call +} + +// SweepCalls is a helper method to define mock.On call +// - ctx context.Context +// - rangeStart pgtype.Timestamptz +// - rangeEnd pgtype.Timestamptz +func (_e *Store_Expecter) SweepCalls(ctx interface{}, rangeStart interface{}, rangeEnd interface{}) *Store_SweepCalls_Call { + return &Store_SweepCalls_Call{Call: _e.mock.On("SweepCalls", ctx, rangeStart, rangeEnd)} +} + +func (_c *Store_SweepCalls_Call) Run(run func(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz)) *Store_SweepCalls_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(pgtype.Timestamptz), args[2].(pgtype.Timestamptz)) + }) + return _c +} + +func (_c *Store_SweepCalls_Call) Return(_a0 error) *Store_SweepCalls_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Store_SweepCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error) *Store_SweepCalls_Call { + _c.Call.Return(run) + return _c +} + // UpdatePassword provides a mock function with given fields: ctx, username, password func (_m *Store) UpdatePassword(ctx context.Context, username string, password string) error { ret := _m.Called(ctx, username, password) diff --git a/pkg/database/models.go b/pkg/database/models.go index e4e0caf..b76cbb8 100644 --- a/pkg/database/models.go +++ b/pkg/database/models.go @@ -55,7 +55,91 @@ type Call struct { Transcript *string `json:"transcript,omitempty"` } -type CallsUnpart struct { +type CallsP202407 struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + +type CallsP202408 struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + +type CallsP202409 struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + +type CallsP202410 struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + +type CallsP202411 struct { ID uuid.UUID `json:"id,omitempty"` Submitter *int32 `json:"submitter,omitempty"` System int `json:"system,omitempty"` @@ -87,9 +171,12 @@ type Incident struct { } type IncidentsCall struct { - IncidentID uuid.UUID `json:"incident_id,omitempty"` - CallID uuid.UUID `json:"call_id,omitempty"` - Notes []byte `json:"notes,omitempty"` + IncidentID uuid.UUID `json:"incident_id,omitempty"` + CallID uuid.UUID `json:"call_id,omitempty"` + CallsTblID pgtype.UUID `json:"calls_tbl_id,omitempty"` + SweptCallID pgtype.UUID `json:"swept_call_id,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + Notes []byte `json:"notes,omitempty"` } type Setting struct { @@ -98,6 +185,27 @@ type Setting struct { Value []byte `json:"value,omitempty"` } +type SweptCall struct { + ID uuid.UUID `json:"id,omitempty"` + Submitter *int32 `json:"submitter,omitempty"` + System int `json:"system,omitempty"` + Talkgroup int `json:"talkgroup,omitempty"` + CallDate pgtype.Timestamptz `json:"call_date,omitempty"` + AudioName *string `json:"audio_name,omitempty"` + AudioBlob []byte `json:"audio_blob,omitempty"` + Duration *int32 `json:"duration,omitempty"` + AudioType *string `json:"audio_type,omitempty"` + AudioUrl *string `json:"audio_url,omitempty"` + Frequency int `json:"frequency,omitempty"` + Frequencies []int `json:"frequencies,omitempty"` + Patches []int `json:"patches,omitempty"` + TGLabel *string `json:"tg_label,omitempty"` + TGAlphaTag *string `json:"tg_alpha_tag,omitempty"` + TGGroup *string `json:"tg_group,omitempty"` + Source int `json:"source,omitempty"` + Transcript *string `json:"transcript,omitempty"` +} + type System struct { ID int `json:"id,omitempty"` Name string `json:"name,omitempty"` diff --git a/pkg/database/partitions.go b/pkg/database/partitions.go new file mode 100644 index 0000000..52bf9c1 --- /dev/null +++ b/pkg/database/partitions.go @@ -0,0 +1,87 @@ +package database + +import ( + "context" + "time" +) + +type partitionsQuerier interface { + GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]string, error) + CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error + DetachPartition(ctx context.Context, partitionName string) error + DropPartition(ctx context.Context, partitionName string) error +} + +func (q *Queries) GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]string, error) { + const query = `SELECT child.relname + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace + WHERE + nmsp_parent.nspname = nmsp_child.nspname -- parent and child schemas are the same + AND nmsp_child.nspname = $2 -- schemaName + AND parent.relname = $1;` // tableName + rows, err := q.db.Query(ctx, query, schemaName, tableName) + if err != nil { + return nil, err + } + + defer rows.Close() + + var partitions []string + for rows.Next() { + var partitionName string + if err := rows.Scan(&partitionName); err != nil { + return nil, err + } + + partitions = append(partitions, partitionName) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return partitions, nil +} + +func (q *Queries) CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error { + _, err := q.db.Exec(ctx, `CREATE TABLE $1 PARTITION OF $2 FOR VALUES FROM ($2) TO ($3);`, partitionName, parentTable, start, end) + + return err +} + +func (q *Queries) DropPartition(ctx context.Context, partitionName string) error { + _, err := q.db.Exec(ctx, `DROP TABLE $1;`, partitionName) + + return err +} + +func (q *Queries) DetachPartition(ctx context.Context, partitionName string) error { + _, err := q.db.Exec(ctx, `ALTER TABLE $1 DETACH PARTITION;`, partitionName) + return err +} + +/* +func (q *Queries) SweepCalls(ctx context.context, start, end time.Time) { +const -- name: SweepCalls :exec +BEGIN; +WITH to_sweep AS ( + SELECT * FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= @range_start AND calls.call_date < @range_end +) INSERT INTO swept_calls SELECT * FROM todelete; + +WITH to_sweep AS ( + SELECT id FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= @range_start AND calls.call_date < @range_end +) UPDATE incidents_calls + SET + swept_call_id = call_id, + calls_tbl_id = NULL + WHERE call_id IN (SELECT id FROM to_sweep); +COMMIT;} +*/ diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index e1b092d..025a634 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -1,8 +1,8 @@ package partman import ( - "fmt" "context" + "fmt" "strconv" "strings" "time" @@ -11,9 +11,25 @@ import ( "dynatron.me/x/stillbox/pkg/database" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/rs/zerolog/log" ) +const ( + callsTable = "calls" + + preProvisionDefault = 1 +) + +/* + * Partition scheme names: + * daily: calls_p_2024_11_28 + * weekly: calls_p_2024_w48 + * monthly: calls_p_2024_11 + * quarterly: calls_p_2024_q4 + * yearly: calls_p_2024 + */ + type PartitionError string func (pe PartitionError) Error() string { @@ -21,15 +37,19 @@ func (pe PartitionError) Error() string { } type ErrInvalidInterval string -func (e ErrInvalidInterval) Error() string { return fmt.Sprintf("invalid interval '%s'", string(e)) } + +func (in ErrInvalidInterval) Error() string { + return fmt.Sprintf("invalid interval '%s'", string(in)) +} type Interval string + const ( - Daily Interval = "daily" - Weekly Interval = "weekly" - Monthly Interval = "monthly" + Daily Interval = "daily" + Weekly Interval = "weekly" + Monthly Interval = "monthly" Quarterly Interval = "quarterly" - Yearly Interval = "yearly" + Yearly Interval = "yearly" ) func (p Interval) IsValid() bool { @@ -41,59 +61,67 @@ func (p Interval) IsValid() bool { return false } - - type PartitionManager interface { Go(ctx context.Context) - Check(ctx context.Context) error + Check(ctx context.Context, now time.Time) error } type partman struct { - db *database.Postgres - cfg config.Partition + db database.Store + cfg config.Partition intv Interval } - type partition interface { PartitionName() string - Next() partition - Range() (string, string) + Next(i int) partition + Prev(i int) partition + Range() (time.Time, time.Time) } type monthlyPartition struct { - year int - month time.Month + t time.Time } func (d monthlyPartition) PartitionName() string { - return fmt.Sprintf("calls_p_%d_%02d", d.year, d.month) + return fmt.Sprintf("calls_p_%d_%02d", d.t.Year(), d.t.Month()) } -func (d monthlyPartition) Next() partition { - return d.next() +func (d monthlyPartition) Next(i int) partition { + return d.next(i) } -func (d monthlyPartition) next() monthlyPartition { - if d.month == time.December { - d.year++ - d.month = time.January - } else { - d.month++ +func (d monthlyPartition) Prev(i int) partition { + return d.prev(i) +} + +func (d monthlyPartition) Range() (start, end time.Time) { + start = time.Date(d.t.Year(), d.t.Month(), 1, 0, 0, 0, 0, time.UTC) + end = start.AddDate(0, 1, 0) + + return +} + +func (d monthlyPartition) next(i int) monthlyPartition { + year, month, _ := d.t.Date() + + return monthlyPartition{ + t: time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, time.UTC), } - - return d } -func (d monthlyPartition) Range() (string, string) { - next := d.next() - return fmt.Sprintf("%d-%02d-01", d.year, d.month), fmt.Sprintf("%d-%02d-01", next.year, next.month) +func (d monthlyPartition) prev(i int) monthlyPartition { + year, month, _ := d.t.Date() + + return monthlyPartition{ + t: time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, d.t.Location()), + } } -func NewPartman(db *database.Postgres, cfg config.Partition) (*partman, error) { +func New(db database.Store, cfg config.Partition) (*partman, error) { pm := &partman{ - cfg: cfg, - db: db, + cfg: cfg, + db: db, intv: Interval(cfg.Interval), } @@ -108,11 +136,11 @@ var _ PartitionManager = (*partman)(nil) func (pm *partman) Go(ctx context.Context) { go func(ctx context.Context) { - tick := time.NewTicker(60*time.Minute) + tick := time.NewTicker(60 * time.Minute) select { - case <-tick.C: - err := pm.Check(ctx) + case now := <-tick.C: + err := pm.Check(ctx, now) if err != nil { log.Error().Err(err).Msg("partman check failed") } @@ -123,84 +151,185 @@ func (pm *partman) Go(ctx context.Context) { } func (pm *partman) newPartition(t time.Time) partition { - return monthlyPartition{month: t.Month(), year: t.Year()} + switch pm.intv { + case Monthly: + return monthlyPartition{t} + } + + return nil } -func (pm *partman) Check(ctx context.Context) error { - err := pm.db.InTx(ctx, func(db database.Store) error { +func (pm *partman) retentionPartitions(cur partition) []partition { + partitions := make([]partition, 0, pm.cfg.Retain) + for i := 1; i <= pm.cfg.Retain; i++ { + prev := cur.Prev(i) + partitions = append(partitions, prev) + } + + return partitions +} + +func (pm *partman) futurePartitions(cur partition) []partition { + preProv := preProvisionDefault + if pm.cfg.PreProvision != nil { + preProv = *pm.cfg.PreProvision + } + + partitions := make([]partition, 0, pm.cfg.Retain) + for i := 1; i <= preProv; i++ { + next := cur.Next(i) + partitions = append(partitions, next) + } + + return partitions +} + +func (pm *partman) expectedPartitions(now time.Time) []partition { + curPart := pm.newPartition(now) + + retain := pm.retentionPartitions(curPart) + + future := pm.futurePartitions(curPart) + + shouldExist := append(retain, curPart) + shouldExist = append(shouldExist, future...) + + return shouldExist +} + +func (pm *partman) comparePartitions(existingTables, expectedTables []partition) (unexpectedTables, missingTables []partition) { + existing := make(map[string]partition) + expectedAndExists := make(map[string]bool) + + for _, t := range existingTables { + existing[t.PartitionName()] = t + } + + for _, t := range expectedTables { + if _, found := existing[t.PartitionName()]; found { + expectedAndExists[t.PartitionName()] = true + } else { + missingTables = append(missingTables, t) + } + } + + for _, t := range existingTables { + if _, found := expectedAndExists[t.PartitionName()]; !found { + // Only in existingTables and not in both + unexpectedTables = append(unexpectedTables, t) + } + } + + return unexpectedTables, missingTables +} + +func (pm *partman) existingPartitions(parts []string) ([]partition, error) { + existing := make([]partition, 0, len(parts)) + for _, v := range parts { + p, err := parsePartName(v) + if err != nil { + return nil, err + } + + existing = append(existing, p) + } + return existing, nil +} + +func (pm *partman) fullTableName(s string) string { + return fmt.Sprintf("%s.%s", pm.cfg.Schema, s) +} + +func (pm *partman) prunePartition(ctx context.Context, tx database.Store, p partition) error { + s, e := p.Range() + start := pgtype.Timestamptz{Time: s, Valid: true} + end := pgtype.Timestamptz{Time: e, Valid: true} + err := tx.SweepCalls(ctx, start, end) + if err != nil { + return err + } + + err = tx.CleanupSweptCalls(ctx, start, end) + if err != nil { + return err + } + + err = tx.DetachPartition(ctx, pm.fullTableName(p.PartitionName())) + if err != nil { + return err + } + + if pm.cfg.Drop { + return tx.DropPartition(ctx, pm.fullTableName(p.PartitionName())) + } + + return nil +} + +func (pm *partman) Check(ctx context.Context, now time.Time) error { + return pm.db.InTx(ctx, func(db database.Store) error { // by default, we want to make sure a partition exists for this and next month - // since we run this at startup, it's safe to do only that. - schemaName, err := db.GetSearchPath(ctx) + // since we run this at startup, it's safe to do only that. + partitions, err := db.GetTablePartitions(ctx, pm.cfg.Schema, callsTable) if err != nil { return err } - partitions, err := db.GetCallsPartitions(ctx, &schemaName) + existing, err := pm.existingPartitions(partitions) if err != nil { return err } - // this could be done in SQL - partsMap, err := partitionsMap(partitions) - if err != nil { - return err + expected := pm.expectedPartitions(now) + + unexpected, missing := pm.comparePartitions(existing, expected) + + for _, p := range unexpected { + err := pm.prunePartition(ctx, db, p) + if err != nil { + return err + } } - now := time.Now() - - thisPart := pm.newPartition(now) - nextPart := thisPart.Next() - - mustExist := []partition{thisPart, nextPart} - - for _, ch := range mustExist { - if !partsMap.exists(ch) { - - err := pm.createPartition(ctx, db, ch) - if err != nil { - return err - } - - log.Info().Str("part", ch.PartitionName()).Msg("created partition") + for _, p := range missing { + err := pm.createPartition(ctx, db, p) + if err != nil { + return err } } return nil }, pgx.TxOptions{}) - - if err != nil { - return err - } - - return nil } -func parsePartName(p *string) (monthlyPartition, error) { - r := monthlyPartition{} - dateAr := strings.Split(*p, "calls_p_") +func (pm *partman) createPartition(ctx context.Context, tx database.Store, part partition) error { + start, end := part.Range() + return tx.CreatePartition(ctx, callsTable, part.PartitionName(), start, end) +} + +func parsePartName(p string) (partition, error) { + dateAr := strings.Split(p, "calls_p_") if len(dateAr) != 2 { - return r, PartitionError(*p) + return nil, PartitionError(p) } dateAr = strings.Split(dateAr[1], "_") if len(dateAr) != 2 { - return r, PartitionError(*p) + return nil, PartitionError(p) } year, err := strconv.Atoi(dateAr[0]) if err != nil { - return r, PartitionError(*p) + return nil, PartitionError(p) } - r.year = year - month, err := strconv.Atoi(dateAr[1]) if err != nil { - return r, PartitionError(*p) + return nil, PartitionError(p) } - r.month = time.Month(month) + r := monthlyPartition{time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC)} return r, nil } @@ -208,19 +337,15 @@ func parsePartName(p *string) (monthlyPartition, error) { type partMap map[partition]struct{} func (pm partMap) exists(dt partition) bool { - _ , ex := pm[dt] + _, ex := pm[dt] return ex } -func partitionsMap(partitions []*string) (partMap, error) { +func partitionsMap(partitions []string, mustExist map[partition]struct{}) (partMap, error) { partsDate := make(partMap, len(partitions)) for _, p := range partitions { - if p == nil { - panic(PartitionError("")) - } - dt, err := parsePartName(p) if err != nil { return nil, err @@ -229,21 +354,16 @@ func partitionsMap(partitions []*string) (partMap, error) { partsDate[dt] = struct{}{} } - return partsDate, nil } -func (pm *partman) createPartition(ctx context.Context, db database.Store, d partition) error { - t, n := d.Range() - - _, err := db.DBTX().Exec(ctx, fmt.Sprintf(`CREATE TABLE %s PARTITION OF calls FOR VALUES FROM ($1) TO ($2);`, d.PartitionName()), t, n) - if err != nil { - return err - } - - return nil +func runRetention() { + // make sure to check if partition was attached first + // before dropping. don't want to accidentally drop pre-detached partitions. } -func monthPart(m time.Month, y int) monthlyPartition { - return monthlyPartition{year: y, month: m} +func dropPart() { + // intx + // SweepCalls + // DropPart } diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go new file mode 100644 index 0000000..984bc04 --- /dev/null +++ b/pkg/database/partman/partman_test.go @@ -0,0 +1,148 @@ +package partman_test + +import ( + "context" + "testing" + "time" + + "dynatron.me/x/stillbox/internal/common" + "dynatron.me/x/stillbox/pkg/config" + "dynatron.me/x/stillbox/pkg/database" + "dynatron.me/x/stillbox/pkg/database/mocks" + "dynatron.me/x/stillbox/pkg/database/partman" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +var mctx = mock.Anything + +func inTx(s *mocks.Store, dbtx *mocks.DBTX) { + s.EXPECT().InTx(mctx, mock.AnythingOfType("func(database.Store) error"), mock.AnythingOfType("pgx.TxOptions")).RunAndReturn(func(ctx context.Context, f func(db database.Store) error, po pgx.TxOptions) error { + return f(s) + }) + +} + +type timeRange struct { + start time.Time + end time.Time +} + +func TestPartman(t *testing.T) { + ctx := context.Background() + + timeInUTC := func(s string) time.Time { + t, err := time.ParseInLocation("2006-01-02 15:04:05", s, time.UTC) + if err != nil { + panic(err) + } + + return t + } + + dateInUTC := func(s string) time.Time { + t, err := time.ParseInLocation("2006-01-02", s, time.UTC) + if err != nil { + panic(err) + } + + return t + } + + tests := []struct { + name string + now time.Time + cfg config.Partition + extant []string + expectCreate []string + expectDrop []string + expectDetach []string + expectSweep []timeRange + expectCleanup []timeRange + }{ + { + name: "monthly base", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "monthly", + Retain: 2, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []string{ + "calls_p_2024_10", + "calls_p_2024_09", + "calls_p_2024_08", + "calls_p_2024_07", + }, + expectCreate: []string{ + "calls_p_2024_11", + "calls_p_2024_12", + "calls_p_2025_01", + }, + expectDrop: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + expectSweep: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectCleanup: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectDetach: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + db := mocks.NewStore(t) + dbtx := mocks.NewDBTX(t) + createdPartitions := make([]string, 0, len(tc.expectCreate)) + sweptRanges := make([]timeRange, 0, len(tc.expectSweep)) + droppedPartitions := make([]string, 0, len(tc.expectDrop)) + cleanupRanges := make([]timeRange, 0, len(tc.expectCleanup)) + detachedPartitions := make([]string, 0, len(tc.expectDetach)) + + db.EXPECT().CreatePartition(mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), mock.AnythingOfType("time.Time")).Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { + createdPartitions = append(createdPartitions, partitionName) + }).Return(nil) + db.EXPECT().SweepCalls(mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz")).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + sweptRanges = append(sweptRanges, timeRange{start: start.Time, end: end.Time}) + }).Return(nil) + db.EXPECT().CleanupSweptCalls(mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz")).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + cleanupRanges = append(cleanupRanges, timeRange{start: start.Time, end: end.Time}) + }).Return(nil) + db.EXPECT().DropPartition(mctx, mock.AnythingOfType("string")).Run(func(ctx context.Context, partName string) { + droppedPartitions = append(droppedPartitions, partName) + }).Return(nil) + db.EXPECT().DetachPartition(mctx, mock.AnythingOfType("string")).Run(func(ctx context.Context, partName string) { + detachedPartitions = append(detachedPartitions, partName) + }).Return(nil) + inTx(db, dbtx) + db.EXPECT().GetTablePartitions(mctx, "public", "calls").Return(tc.extant, nil) + pm, err := partman.New(db, tc.cfg) + require.NoError(t, err) + + err = pm.Check(ctx, tc.now) + require.NoError(t, err) + + assert.ElementsMatch(t, tc.expectCreate, createdPartitions) + assert.ElementsMatch(t, tc.expectSweep, sweptRanges) + assert.ElementsMatch(t, tc.expectDrop, droppedPartitions) + assert.ElementsMatch(t, tc.expectCleanup, cleanupRanges) + assert.ElementsMatch(t, tc.expectDetach, detachedPartitions) + }) + } +} diff --git a/pkg/database/querier.go b/pkg/database/querier.go index 911cc22..e9425ca 100644 --- a/pkg/database/querier.go +++ b/pkg/database/querier.go @@ -15,12 +15,12 @@ type Querier interface { AddAlert(ctx context.Context, arg AddAlertParams) error AddCall(ctx context.Context, arg AddCallParams) error AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error) + CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) DeleteAPIKey(ctx context.Context, apiKey string) error DeleteUser(ctx context.Context, username string) error GetAPIKey(ctx context.Context, apiKey string) (ApiKey, error) - GetCallsPartitions(ctx context.Context, schemaName *string) ([]*string, error) GetDatabaseSize(ctx context.Context) (string, error) GetSystemName(ctx context.Context, systemID int) (string, error) GetTalkgroup(ctx context.Context, systemID int32, tGID int32) (GetTalkgroupRow, error) @@ -43,6 +43,7 @@ type Querier interface { SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults + SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error UpdatePassword(ctx context.Context, username string, password string) error UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) UpsertTalkgroup(ctx context.Context, arg []UpsertTalkgroupParams) *UpsertTalkgroupBatchResults diff --git a/sql/postgres/migrations/002_partition.up.sql b/sql/postgres/migrations/002_partition.up.sql index 7ae6d80..3ac1dc2 100644 --- a/sql/postgres/migrations/002_partition.up.sql +++ b/sql/postgres/migrations/002_partition.up.sql @@ -25,4 +25,56 @@ CREATE TABLE calls ( FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) ) PARTITION BY RANGE (call_date); +-- for our prod +create table calls_p_2024_07 partition of calls for values from ('2024-07-01') to ('2024-08-01'); +create table calls_p_2024_08 partition of calls for values from ('2024-08-01') to ('2024-09-01'); +create table calls_p_2024_09 partition of calls for values from ('2024-09-01') to ('2024-10-01'); +create table calls_p_2024_10 partition of calls for values from ('2024-10-01') to ('2024-11-01'); +create table calls_p_2024_11 partition of calls for values from ('2024-11-01') to ('2024-12-01'); + + +insert into calls (id, submitter, system, talkgroup, call_date, audio_name, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration) select id, submitter, system, talkgroup, call_date, audio_name, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration from calls_unpart; + +drop table calls_unpart cascade; + +CREATE TABLE swept_calls ( + id UUID PRIMARY KEY, + submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, + system INTEGER NOT NULL, + talkgroup INTEGER NOT NULL, + call_date TIMESTAMPTZ NOT NULL, + audio_name TEXT, + audio_blob BYTEA, + duration INTEGER, + audio_type TEXT, + audio_url TEXT, + frequency INTEGER NOT NULL, + frequencies INTEGER[], + patches INTEGER[], + tg_label TEXT, + tg_alpha_tag TEXT, + tg_group TEXT, + source INTEGER NOT NULL, + transcript TEXT, + FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) +); + +DROP TABLE IF EXISTS incidents_calls; -- DATA LOSS + +CREATE TABLE IF NOT EXISTS incidents_calls( + incident_id UUID NOT NULL REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, + call_id UUID NOT NULL, + calls_tbl_id UUID NULL, + swept_call_id UUID NULL REFERENCES swept_calls(id), + call_date TIMESTAMPTZ NULL, + notes JSONB, + FOREIGN KEY (calls_tbl_id, call_date) REFERENCES calls(id, call_date), + PRIMARY KEY (incident_id, call_id) +); + + +-- ALTER TABLE incidents_calls ADD COLUMN call_date TIMESTAMPTZ NOT NULL; +-- ALTER TABLE incidents_calls DROP CONSTRAINT incidents_calls_call_id_fkey; +-- ALTER TABLE incidents_calls ADD CONSTRAINT incidents_calls_call_id_call_date_fkey FOREIGN KEY (call_id, call_date) REFERENCES calls(id, call_date) ON UPDATE CASCADE; + COMMIT; diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index a78ce12..3955614 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -57,6 +57,20 @@ VALUES -- name: GetDatabaseSize :one SELECT pg_size_pretty(pg_database_size(current_database())); --- name: GetCallsPartitions :many -SELECT tablename FROM pg_catalog.pg_tables -WHERE schemaname = @schema_name AND tablename LIKE 'calls\_p\_____\___'; +-- name: SweepCalls :exec +WITH to_sweep AS ( + SELECT * FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= @range_start AND calls.call_date < @range_end +) INSERT INTO swept_calls SELECT * FROM to_sweep; + +-- name: CleanupSweptCalls :exec +WITH to_sweep AS ( + SELECT id FROM calls + JOIN incidents_calls ic ON ic.call_id = calls.id + WHERE calls.call_date >= @range_start AND calls.call_date < @range_end +) UPDATE incidents_calls + SET + swept_call_id = call_id, + calls_tbl_id = NULL + WHERE call_id IN (SELECT id FROM to_sweep); diff --git a/sql/sqlc.yaml b/sql/sqlc.yaml index c233814..0b51630 100644 --- a/sql/sqlc.yaml +++ b/sql/sqlc.yaml @@ -37,3 +37,6 @@ sql: import: "dynatron.me/x/stillbox/internal/jsontypes" type: "Metadata" nullable: true + - column: "pg_catalog.pg_tables.tablename" + go_type: string + nullable: false -- 2.48.1 From f251f0c1d41e482ce5f2783f474522eb2ce9a793 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Fri, 29 Nov 2024 23:24:43 -0500 Subject: [PATCH 04/15] before refact --- pkg/database/partitions.go | 167 ++++++++++++----- pkg/database/partman/intervals.go | 103 +++++++++++ pkg/database/partman/partman.go | 181 +++++++++---------- pkg/database/partman/partman_test.go | 37 +++- sql/postgres/migrations/002_partition.up.sql | 2 +- 5 files changed, 343 insertions(+), 147 deletions(-) create mode 100644 pkg/database/partman/intervals.go diff --git a/pkg/database/partitions.go b/pkg/database/partitions.go index 52bf9c1..e11d699 100644 --- a/pkg/database/partitions.go +++ b/pkg/database/partitions.go @@ -1,10 +1,27 @@ package database import ( + "fmt" "context" + "errors" "time" + + "github.com/jackc/pgx/v5" ) +var ( + ErrLowerBoundAfterUpperBound = errors.New("lower bound after upper bound") + ErrCantDecodePartitionBounds = errors.New("cannot decode partition bounds") +) + +type PartitionResult struct { + ParentTable string + Schema string + Name string + LowerBound string + UpperBound string +} + type partitionsQuerier interface { GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]string, error) CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error @@ -12,36 +29,36 @@ type partitionsQuerier interface { DropPartition(ctx context.Context, partitionName string) error } -func (q *Queries) GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]string, error) { - const query = `SELECT child.relname - FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid - JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace - JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace - WHERE - nmsp_parent.nspname = nmsp_child.nspname -- parent and child schemas are the same - AND nmsp_child.nspname = $2 -- schemaName - AND parent.relname = $1;` // tableName - rows, err := q.db.Query(ctx, query, schemaName, tableName) +func (q *Queries) GetTablePartitions(ctx context.Context, schemaName, tableName string) (partitions []PartitionResult, err error) { + query := fmt.Sprintf(` + WITH parts as ( + SELECT + relnamespace::regnamespace as schema, + c.oid::pg_catalog.regclass AS part_name, + regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FOR VALUES FROM \(''(.*)''\) TO \(''(.*)''\)') AS bounds + FROM + pg_catalog.pg_class c JOIN pg_catalog.pg_inherits i ON (c.oid = i.inhrelid) + WHERE i.inhparent = '%s.%s'::regclass + AND c.relkind='r' + ) + SELECT + schema, + part_name as name, + '%s' as parentTable, + bounds[1]::text AS lowerBound, + bounds[2]::text AS upperBound + FROM parts + ORDER BY part_name;`, schemaName, tableName, tableName) + + rows, err := q.db.Query(ctx, query) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get partitions: %w", err) } - defer rows.Close() - - var partitions []string - for rows.Next() { - var partitionName string - if err := rows.Scan(&partitionName); err != nil { - return nil, err - } - - partitions = append(partitions, partitionName) - } - - if err := rows.Err(); err != nil { - return nil, err + partitions, err = pgx.CollectRows(rows, pgx.RowToStructByName[PartitionResult]) + if err != nil { + return nil, fmt.Errorf("failed to cast list: %w", err) } return partitions, nil @@ -64,24 +81,80 @@ func (q *Queries) DetachPartition(ctx context.Context, partitionName string) err return err } -/* -func (q *Queries) SweepCalls(ctx context.context, start, end time.Time) { -const -- name: SweepCalls :exec -BEGIN; -WITH to_sweep AS ( - SELECT * FROM calls - JOIN incidents_calls ic ON ic.call_id = calls.id - WHERE calls.call_date >= @range_start AND calls.call_date < @range_end -) INSERT INTO swept_calls SELECT * FROM todelete; +func (partition PartitionResult) ParseBounds() (lowerBound time.Time, upperBound time.Time, err error) { + lowerBound, upperBound, err = parseBoundAsDate(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsDateTime(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsDateTimeWithTimezone(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + if lowerBound.After(lowerBound) { + return time.Time{}, time.Time{}, ErrLowerBoundAfterUpperBound + } + + return time.Time{}, time.Time{}, ErrCantDecodePartitionBounds +} + +func parseBoundAsDate(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02", partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as date: %w", err) + } + + upperBound, err = time.Parse("2006-01-02", partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as date: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsDateTime(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02 15:04:05", partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime: %w", err) + } + + upperBound, err = time.Parse("2006-01-02 15:04:05", partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsDateTimeWithTimezone(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02 15:04:05Z07", partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime with timezone: %w", err) + } + + upperBound, err = time.Parse("2006-01-02 15:04:05Z07", partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime with timezone: %w", err) + } + + lowerBound = convertToDateTimeWithoutTimezone(lowerBound) + upperBound = convertToDateTimeWithoutTimezone(upperBound) + + return lowerBound, upperBound, nil +} + +func convertToDateTimeWithoutTimezone(bound time.Time) time.Time { + parsedTime, err := time.Parse("2006-01-02 15:04:05", bound.UTC().Format("2006-01-02 15:04:05")) + if err != nil { + return time.Time{} + } + + return parsedTime +} -WITH to_sweep AS ( - SELECT id FROM calls - JOIN incidents_calls ic ON ic.call_id = calls.id - WHERE calls.call_date >= @range_start AND calls.call_date < @range_end -) UPDATE incidents_calls - SET - swept_call_id = call_id, - calls_tbl_id = NULL - WHERE call_id IN (SELECT id FROM to_sweep); -COMMIT;} -*/ diff --git a/pkg/database/partman/intervals.go b/pkg/database/partman/intervals.go new file mode 100644 index 0000000..4ef78dc --- /dev/null +++ b/pkg/database/partman/intervals.go @@ -0,0 +1,103 @@ +package partman + +import ( + "fmt" + "time" +) + +const ( + daysInWeek = 7 +) + +type weeklyPartition struct { + t time.Time +} + +func (d weeklyPartition) PartitionName() string { + return fmt.Sprintf("calls_p_%d_w%02d_%02d", d.t.Year(), d.t.Month(), d.t.Day()) +} + +func (d weeklyPartition) Range() (start, end time.Time) { + start = time.Date(d.t.Year(), d.t.Month(), d.t.Day(), 0, 0, 0, 0, d.t.UTC().Location()).AddDate(0, 0, -int(d.t.Weekday()-time.Monday)) + start = start.AddDate(0, 0, daysInWeek) + + return +} + +func (d weeklyPartition) Next(i int) partition { + year, month, day := d.t.Date() + + return weeklyPartition{ + t: time.Date(year, month, day+i, 0, 0, 0, 0, time.UTC), + } +} + +func (d weeklyPartition) Prev(i int) partition { + year, month, day := d.t.Date() + + return weeklyPartition{ + t: time.Date(year, month, day-i, 0, 0, 0, 0, d.t.Location()), + } +} + +type dailyPartition struct { + t time.Time +} + +func (d dailyPartition) PartitionName() string { + return fmt.Sprintf("calls_p_%d_%02d_%02d", d.t.Year(), d.t.Month(), d.t.Day()) +} + +func (d dailyPartition) Range() (start, end time.Time) { + start = time.Date(d.t.Year(), d.t.Month(), d.t.Day(), 0, 0, 0, 0, time.UTC) + end = start.AddDate(0, 0, 1) + + return +} + +func (d dailyPartition) Next(i int) partition { + year, month, day := d.t.Date() + + return dailyPartition{ + t: time.Date(year, month, day+i, 0, 0, 0, 0, time.UTC), + } +} + +func (d dailyPartition) Prev(i int) partition { + year, month, day := d.t.Date() + + return dailyPartition{ + t: time.Date(year, month, day-i, 0, 0, 0, 0, d.t.Location()), + } +} + +type monthlyPartition struct { + t time.Time +} + +func (d monthlyPartition) PartitionName() string { + return fmt.Sprintf("calls_p_%d_%02d", d.t.Year(), d.t.Month()) +} + +func (d monthlyPartition) Range() (start, end time.Time) { + start = time.Date(d.t.Year(), d.t.Month(), 1, 0, 0, 0, 0, time.UTC) + end = start.AddDate(0, 1, 0) + + return +} + +func (d monthlyPartition) Next(i int) partition { + year, month, _ := d.t.Date() + + return monthlyPartition{ + t: time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, time.UTC), + } +} + +func (d monthlyPartition) Prev(i int) partition { + year, month, _ := d.t.Date() + + return monthlyPartition{ + t: time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, d.t.Location()), + } +} diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index 025a634..e973649 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -21,19 +21,30 @@ const ( preProvisionDefault = 1 ) -/* - * Partition scheme names: - * daily: calls_p_2024_11_28 - * weekly: calls_p_2024_w48 - * monthly: calls_p_2024_11 - * quarterly: calls_p_2024_q4 - * yearly: calls_p_2024 - */ +type partitionError struct { + p string + err error +} -type PartitionError string +func (pe partitionError) Error() string { + r := fmt.Sprintf("bad partition '%s'", pe.p) + if pe.err != nil { + r += ": " + pe.err.Error() + } -func (pe PartitionError) Error() string { - return fmt.Sprintf("bad partition '%s'", string(pe)) + return r +} + +func (pe partitionError) Unwrap() error { + return pe.err +} + +func PartitionError(pname string, err ...error) partitionError { + if len(err) > 0 { + return partitionError{p: pname, err: err[0]} + } + + return partitionError{p: pname} } type ErrInvalidInterval string @@ -45,6 +56,7 @@ func (in ErrInvalidInterval) Error() string { type Interval string const ( + Unknown Interval = "" Daily Interval = "daily" Weekly Interval = "weekly" Monthly Interval = "monthly" @@ -79,45 +91,6 @@ type partition interface { Range() (time.Time, time.Time) } -type monthlyPartition struct { - t time.Time -} - -func (d monthlyPartition) PartitionName() string { - return fmt.Sprintf("calls_p_%d_%02d", d.t.Year(), d.t.Month()) -} - -func (d monthlyPartition) Next(i int) partition { - return d.next(i) -} - -func (d monthlyPartition) Prev(i int) partition { - return d.prev(i) -} - -func (d monthlyPartition) Range() (start, end time.Time) { - start = time.Date(d.t.Year(), d.t.Month(), 1, 0, 0, 0, 0, time.UTC) - end = start.AddDate(0, 1, 0) - - return -} - -func (d monthlyPartition) next(i int) monthlyPartition { - year, month, _ := d.t.Date() - - return monthlyPartition{ - t: time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, time.UTC), - } -} - -func (d monthlyPartition) prev(i int) monthlyPartition { - year, month, _ := d.t.Date() - - return monthlyPartition{ - t: time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, d.t.Location()), - } -} - func New(db database.Store, cfg config.Partition) (*partman, error) { pm := &partman{ cfg: cfg, @@ -223,10 +196,13 @@ func (pm *partman) comparePartitions(existingTables, expectedTables []partition) return unexpectedTables, missingTables } -func (pm *partman) existingPartitions(parts []string) ([]partition, error) { +func (pm *partman) existingPartitions(parts []database.PartitionResult) ([]partition, error) { existing := make([]partition, 0, len(parts)) for _, v := range parts { - p, err := parsePartName(v) + if v.Schema != pm.cfg.Schema { + return nil, PartitionError(v.Schema+"."+v.Name+" has wrong schema") + } + p, _, err := pm.parsePart(v) if err != nil { return nil, err } @@ -308,30 +284,69 @@ func (pm *partman) createPartition(ctx context.Context, tx database.Store, part return tx.CreatePartition(ctx, callsTable, part.PartitionName(), start, end) } -func parsePartName(p string) (partition, error) { - dateAr := strings.Split(p, "calls_p_") +/* + * Partition scheme names: + * daily: calls_p_2024_11_28 + * weekly: calls_p_2024_w48 + * monthly: calls_p_2024_11 + * quarterly: calls_p_2024_q4 + * yearly: calls_p_2024 + */ + +func (pm *partman) parsePart(pr database.PartitionResult) (partition, Interval, error) { + intv := Unknown + pn := pr.Name + low, up, err := pr.ParseBounds() + if err != nil { + return nil, intv, err + } + dateAr := strings.Split(pn, "calls_p_") if len(dateAr) != 2 { - return nil, PartitionError(p) + return nil, intv, PartitionError(pn) } dateAr = strings.Split(dateAr[1], "_") - if len(dateAr) != 2 { - return nil, PartitionError(p) + switch len(dateAr) { + case 3: // daily + intv = Daily + ymd := [3]int{} + for i := 0; i < 3; i++ { + r, err := strconv.Atoi(dateAr[i]) + if err != nil { + return nil, intv, PartitionError(pn, err) + } + + ymd[i] = r + } + return dailyPartition{time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC)}, Daily, nil + case 2: + year, err := strconv.Atoi(dateAr[0]) + if err != nil { + return nil, intv, PartitionError(pn, err) + } + if strings.HasPrefix(dateAr[1], "w") { + intv = Weekly + weekNum, err := strconv.Atoi(dateAr[1][1:]) + if err != nil { + return nil, intv, PartitionError(pn, err) + } + + return weeklyPartition{time.Date(year, 0, daysInWeek*weekNum, 0, 0, 0, 0, time.UTC)}, intv, nil + } else if strings.HasPrefix(dateAr[1], "q") { + panic("not implemented") + } + // monthly + month, err := strconv.Atoi(dateAr[1]) + if err != nil { + return nil, intv, PartitionError(pn) + } + + return monthlyPartition{time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC)}, intv, nil + case 1: // yearly + panic("not implemented") } - year, err := strconv.Atoi(dateAr[0]) - if err != nil { - return nil, PartitionError(p) - } - - month, err := strconv.Atoi(dateAr[1]) - if err != nil { - return nil, PartitionError(p) - } - - r := monthlyPartition{time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC)} - - return r, nil + return nil, intv, PartitionError(pn) } type partMap map[partition]struct{} @@ -341,29 +356,3 @@ func (pm partMap) exists(dt partition) bool { return ex } - -func partitionsMap(partitions []string, mustExist map[partition]struct{}) (partMap, error) { - partsDate := make(partMap, len(partitions)) - - for _, p := range partitions { - dt, err := parsePartName(p) - if err != nil { - return nil, err - } - - partsDate[dt] = struct{}{} - } - - return partsDate, nil -} - -func runRetention() { - // make sure to check if partition was attached first - // before dropping. don't want to accidentally drop pre-detached partitions. -} - -func dropPart() { - // intx - // SweepCalls - // DropPart -} diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index 984bc04..a515157 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -20,7 +20,7 @@ import ( var mctx = mock.Anything -func inTx(s *mocks.Store, dbtx *mocks.DBTX) { +func inTx(s *mocks.Store) { s.EXPECT().InTx(mctx, mock.AnythingOfType("func(database.Store) error"), mock.AnythingOfType("pgx.TxOptions")).RunAndReturn(func(ctx context.Context, f func(db database.Store) error, po pgx.TxOptions) error { return f(s) }) @@ -103,12 +103,43 @@ func TestPartman(t *testing.T) { "public.calls_p_2024_08", }, }, + { + name: "weekly base", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "weekly", + Retain: 2, + Drop: false, + PreProvision: common.PtrTo(2), + }, + extant: []string{ + "calls_p_2024_w44", + "calls_p_2024_w45", + "calls_p_2024_w46", + }, + expectCreate: []string{ + "calls_p_2024_w48", + "calls_p_2024_w49", + "calls_p_2024_w50", + }, + expectSweep: []timeRange{ + // TODO + }, + expectCleanup: []timeRange{ + // TODO + }, + expectDetach: []string{ + "public.calls_p_2024_w44", + "public.calls_p_2024_w45", + }, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { db := mocks.NewStore(t) - dbtx := mocks.NewDBTX(t) createdPartitions := make([]string, 0, len(tc.expectCreate)) sweptRanges := make([]timeRange, 0, len(tc.expectSweep)) droppedPartitions := make([]string, 0, len(tc.expectDrop)) @@ -130,7 +161,7 @@ func TestPartman(t *testing.T) { db.EXPECT().DetachPartition(mctx, mock.AnythingOfType("string")).Run(func(ctx context.Context, partName string) { detachedPartitions = append(detachedPartitions, partName) }).Return(nil) - inTx(db, dbtx) + inTx(db) db.EXPECT().GetTablePartitions(mctx, "public", "calls").Return(tc.extant, nil) pm, err := partman.New(db, tc.cfg) require.NoError(t, err) diff --git a/sql/postgres/migrations/002_partition.up.sql b/sql/postgres/migrations/002_partition.up.sql index 3ac1dc2..0fc1d80 100644 --- a/sql/postgres/migrations/002_partition.up.sql +++ b/sql/postgres/migrations/002_partition.up.sql @@ -33,7 +33,7 @@ create table calls_p_2024_10 partition of calls for values from ('2024-10-01') t create table calls_p_2024_11 partition of calls for values from ('2024-11-01') to ('2024-12-01'); -insert into calls (id, submitter, system, talkgroup, call_date, audio_name, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration) select id, submitter, system, talkgroup, call_date, audio_name, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration from calls_unpart; +insert into calls (id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration) select id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration from calls_unpart; drop table calls_unpart cascade; -- 2.48.1 From da6fbe89af532023804037c3ee379b3267c850cd Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 11:06:40 -0500 Subject: [PATCH 05/15] Monthly and weekly passing --- internal/isoweek/LICENSE | 22 +++ internal/isoweek/isoweek.go | 93 ++++++++++++ internal/isoweek/isoweek_test.go | 96 +++++++++++++ internal/isoweek/julian.go | 32 +++++ internal/isoweek/julian_test.go | 52 +++++++ pkg/database/mocks/Store.go | 14 +- pkg/database/partitions.go | 5 +- pkg/database/partman/intervals.go | 202 +++++++++++++++++---------- pkg/database/partman/partman.go | 117 +++++++++++----- pkg/database/partman/partman_test.go | 99 +++++++++---- 10 files changed, 581 insertions(+), 151 deletions(-) create mode 100644 internal/isoweek/LICENSE create mode 100644 internal/isoweek/isoweek.go create mode 100644 internal/isoweek/isoweek_test.go create mode 100644 internal/isoweek/julian.go create mode 100644 internal/isoweek/julian_test.go diff --git a/internal/isoweek/LICENSE b/internal/isoweek/LICENSE new file mode 100644 index 0000000..31d0b01 --- /dev/null +++ b/internal/isoweek/LICENSE @@ -0,0 +1,22 @@ +License for github.com/snabb/isoweek: + +Copyright © 2016-2023 Janne Snabb snabb AT epipe.com + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/internal/isoweek/isoweek.go b/internal/isoweek/isoweek.go new file mode 100644 index 0000000..c65d196 --- /dev/null +++ b/internal/isoweek/isoweek.go @@ -0,0 +1,93 @@ +// Package isoweek calculates a starting date and time of [ISO 8601] week. +// +// ISO 8601 standard defines the common [week number] system used in Europe +// and many other countries. Monday is the first day of a week. +// +// The Go standard library [time] package has [time.Time.ISOWeek] function +// for getting ISO 8601 week number of a given [time.Time], but there is no +// reverse functionality for getting a date from a week number. This package +// implements that. +// +// Invalid input is silently accepted. There is a separate [Validate] +// function if week number validation is needed. +// +// There are also functions for working with [Julian day numbers]. Using Julian +// day numbers is often the easiest and fastest way to do date calculations. +// +// This package does not work with the "traditional" week system used in +// US/Canada/Japan/etc. (weeks starting on Sundays). However the Julian day +// number functions may be still useful. +// +// [ISO 8601]: https://en.wikipedia.org/wiki/ISO_8601 +// [week number]: https://en.wikipedia.org/wiki/ISO_week_date +// [Julian day numbers]: https://en.wikipedia.org/wiki/Julian_day +package isoweek + +import "time" + +// ISOWeekday returns the ISO 8601 weekday number of given day. +// (1 = Mon, 2 = Tue,.. 7 = Sun) +// +// This is different from Go's standard [time.Weekday]. +func ISOWeekday(year int, month time.Month, day int) (weekday int) { + // Richards, E. G. (2013) pp. 592, 618 + + return DateToJulian(year, month, day)%7 + 1 +} + +// startOffset returns the offset (in days) from the start of a year to +// Monday of the given week. Offset may be negative. +func startOffset(y, week int) (offset int) { + // This is optimized version of the following: + // + // return week*7 - ISOWeekday(y, 1, 4) - 3 + // + // Uses Tomohiko Sakamoto's algorithm for calculating the weekday. + + y = y - 1 + return week*7 - (y+y/4-y/100+y/400+3)%7 - 4 +} + +// StartTime returns the starting time (Monday 00:00) of the given +// ISO 8601 week. +func StartTime(wyear, week int, loc *time.Location) (start time.Time) { + y, m, d := StartDate(wyear, week) + return time.Date(y, m, d, 0, 0, 0, 0, loc) +} + +// StartDate returns the starting date (Monday) of the given ISO 8601 week. +func StartDate(wyear, week int) (year int, month time.Month, day int) { + return JulianToDate( + DateToJulian(wyear, 1, 1) + startOffset(wyear, week)) +} + +// ordinalInYear returns the ordinal (within a year) day number. +func ordinalInYear(year int, month time.Month, day int) (dayNo int) { + return DateToJulian(year, month, day) - DateToJulian(year, 1, 1) + 1 +} + +// FromDate returns ISO 8601 week number of a date. +func FromDate(year int, month time.Month, day int) (wyear, week int) { + week = (ordinalInYear(year, month, day) - ISOWeekday(year, month, day) + 10) / 7 + if week < 1 { + return FromDate(year-1, 12, 31) // last week of preceding year + } + if week == 53 && + DateToJulian(StartDate(year+1, 1)) <= DateToJulian(year, month, day) { + return year + 1, 1 // first week of following year + } + return year, week +} + +// Validate checks if a week number is valid. Returns true if it is valid. +func Validate(wyear, week int) (ok bool) { + if week < 1 || week > 53 { + return false + } + wyear2, week2 := FromDate(StartDate(wyear, week)) + + if wyear == wyear2 && week == week2 { + return true + } + return false +} diff --git a/internal/isoweek/isoweek_test.go b/internal/isoweek/isoweek_test.go new file mode 100644 index 0000000..907b237 --- /dev/null +++ b/internal/isoweek/isoweek_test.go @@ -0,0 +1,96 @@ +package isoweek_test + +import ( + "fmt" + "testing" + "time" + + "dynatron.me/x/stillbox/internal/isoweek" +) + +// TestISOWeekday tests all days from year 1 until year 4000. +// Ensures that behaviour matches the Go standard library Weekday. +func TestISOWeekday(test *testing.T) { + t := time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) + var wd1, wd2 int + for t.Year() < 4000 { + wd1 = int(t.Weekday()) + wd2 = isoweek.ISOWeekday(t.Date()) + + if wd2 == 7 { + wd2 = 0 + } + if wd1 != wd2 { + test.Errorf("mismatch on %s", t.Format("2006-01-02")) + } + t = t.AddDate(0, 0, 1) + } +} + +func ExampleISOWeekday() { + fmt.Println(isoweek.ISOWeekday(1984, 1, 1)) + // Output: 7 +} + +// TestStartTime tests all weeks from year 1 until year 4000. +// Ensures that behaviour matches the Go standard library ISOWeek. +func TestStartTime(test *testing.T) { + t := time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) + for t.Weekday() != time.Monday { + t = t.AddDate(0, 0, 1) + } + for t.Year() < 4000 { + wy, ww := t.ISOWeek() + wst := isoweek.StartTime(wy, ww, time.UTC) + if !wst.Equal(t) { + test.Errorf("mismatch: %v != %v (wy = %d, ww = %d)", + t, wst, wy, ww) + } + t = t.AddDate(0, 0, 7) + } +} + +func ExampleStartTime() { + t := isoweek.StartTime(1985, 1, time.UTC) + fmt.Println(t) + // Output: 1984-12-31 00:00:00 +0000 UTC +} + +func ExampleStartDate() { + y, m, d := isoweek.StartDate(2000, 1) + fmt.Println(d, m, y) + // Output: 3 January 2000 +} + +// TestFromDate tests all days from year 1 until year 4000. +// Ensures that behaviour matches the Go standard library ISOWeek. +func TestFromDate(test *testing.T) { + t := time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) + for t.Year() < 4000 { + wy, ww := t.ISOWeek() + wy2, ww2 := isoweek.FromDate(t.Date()) + if wy != wy2 || ww != ww2 { + test.Errorf("mismatch on %s", t.Format("2006-01-02")) + } + t = t.AddDate(0, 0, 1) + } +} + +func ExampleFromDate() { + fmt.Println(isoweek.FromDate(1984, 1, 1)) + // Output: 1983 52 +} + +func ExampleValidate() { + fmt.Println( + isoweek.Validate(2015, 52), isoweek.Validate(2015, 53), + isoweek.Validate(2015, 54), isoweek.Validate(2016, 0), + isoweek.Validate(2016, 1)) + fmt.Println( + isoweek.Validate(2016, 52), isoweek.Validate(2016, 53), + isoweek.Validate(2016, 54), isoweek.Validate(2017, 0), + isoweek.Validate(2017, 1)) + // Output: + // true true false false true + // true false false false true +} diff --git a/internal/isoweek/julian.go b/internal/isoweek/julian.go new file mode 100644 index 0000000..128c555 --- /dev/null +++ b/internal/isoweek/julian.go @@ -0,0 +1,32 @@ +package isoweek + +import "time" + +// DateToJulian converts a date to a Julian day number. +func DateToJulian(year int, month time.Month, day int) (jdn int) { + // Claus Tøndering's Calendar FAQ + // http://www.tondering.dk/claus/cal/julperiod.php#formula + + if month < 3 { + year = year - 1 + month = month + 12 + } + year = year + 4800 + + return day + (153*(int(month)-3)+2)/5 + 365*year + + year/4 - year/100 + year/400 - 32045 +} + +// JulianToDate converts a Julian day number to a date. +func JulianToDate(jdn int) (year int, month time.Month, day int) { + // Richards, E. G. (2013) pp. 585–624 + + e := 4*(jdn+1401+(4*jdn+274277)/146097*3/4-38) + 3 + h := e%1461/4*5 + 2 + + day = h%153/5 + 1 + month = time.Month((h/153+2)%12 + 1) + year = e/1461 - 4716 + (14-int(month))/12 + + return year, month, day +} diff --git a/internal/isoweek/julian_test.go b/internal/isoweek/julian_test.go new file mode 100644 index 0000000..abc0add --- /dev/null +++ b/internal/isoweek/julian_test.go @@ -0,0 +1,52 @@ +package isoweek_test + +import ( + "fmt" + "testing" + "time" + + "dynatron.me/x/stillbox/internal/isoweek" +) + +func TestJulianToDate(test *testing.T) { + j := isoweek.DateToJulian(1, time.January, 1) + + for { + y, m, d := isoweek.JulianToDate(j) + if y >= 4000 { + break + } + if j != isoweek.DateToJulian(y, m, d) { + test.Errorf("mismatch on %04d-%02d-%02d", y, m, d) + } + j++ + } +} + +func TestDateToJulian(test *testing.T) { + t := time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) + + for t.Year() < 4000 { + j := isoweek.DateToJulian(t.Date()) + + y, m, d := isoweek.JulianToDate(j) + + if y != t.Year() || m != t.Month() || d != t.Day() { + test.Errorf("mismatch on %s", t.Format("2006-01-02")) + } + if j+1 != isoweek.DateToJulian(y, m, d+1) { + test.Errorf("mismatch 2 on %s", t.Format("2006-01-02")) + } + t = t.AddDate(0, 0, 1) + } +} + +func ExampleDateToJulian() { + fmt.Println(isoweek.DateToJulian(2006, 1, 2)) + // Output: 2453738 +} + +func ExampleJulianToDate() { + fmt.Println(isoweek.JulianToDate(2453738)) + // Output: 2006 January 2 +} diff --git a/pkg/database/mocks/Store.go b/pkg/database/mocks/Store.go index 29c5fbe..e4778bf 100644 --- a/pkg/database/mocks/Store.go +++ b/pkg/database/mocks/Store.go @@ -896,23 +896,23 @@ func (_c *Store_GetSystemName_Call) RunAndReturn(run func(context.Context, int) } // GetTablePartitions provides a mock function with given fields: ctx, schemaName, tableName -func (_m *Store) GetTablePartitions(ctx context.Context, schemaName string, tableName string) ([]string, error) { +func (_m *Store) GetTablePartitions(ctx context.Context, schemaName string, tableName string) ([]database.PartitionResult, error) { ret := _m.Called(ctx, schemaName, tableName) if len(ret) == 0 { panic("no return value specified for GetTablePartitions") } - var r0 []string + var r0 []database.PartitionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]string, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]database.PartitionResult, error)); ok { return rf(ctx, schemaName, tableName) } - if rf, ok := ret.Get(0).(func(context.Context, string, string) []string); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, string) []database.PartitionResult); ok { r0 = rf(ctx, schemaName, tableName) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) + r0 = ret.Get(0).([]database.PartitionResult) } } @@ -945,12 +945,12 @@ func (_c *Store_GetTablePartitions_Call) Run(run func(ctx context.Context, schem return _c } -func (_c *Store_GetTablePartitions_Call) Return(_a0 []string, _a1 error) *Store_GetTablePartitions_Call { +func (_c *Store_GetTablePartitions_Call) Return(_a0 []database.PartitionResult, _a1 error) *Store_GetTablePartitions_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Store_GetTablePartitions_Call) RunAndReturn(run func(context.Context, string, string) ([]string, error)) *Store_GetTablePartitions_Call { +func (_c *Store_GetTablePartitions_Call) RunAndReturn(run func(context.Context, string, string) ([]database.PartitionResult, error)) *Store_GetTablePartitions_Call { _c.Call.Return(run) return _c } diff --git a/pkg/database/partitions.go b/pkg/database/partitions.go index e11d699..f93c749 100644 --- a/pkg/database/partitions.go +++ b/pkg/database/partitions.go @@ -1,9 +1,9 @@ package database import ( - "fmt" "context" "errors" + "fmt" "time" "github.com/jackc/pgx/v5" @@ -23,7 +23,7 @@ type PartitionResult struct { } type partitionsQuerier interface { - GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]string, error) + GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]PartitionResult, error) CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error DetachPartition(ctx context.Context, partitionName string) error DropPartition(ctx context.Context, partitionName string) error @@ -157,4 +157,3 @@ func convertToDateTimeWithoutTimezone(bound time.Time) time.Time { return parsedTime } - diff --git a/pkg/database/partman/intervals.go b/pkg/database/partman/intervals.go index 4ef78dc..3f37012 100644 --- a/pkg/database/partman/intervals.go +++ b/pkg/database/partman/intervals.go @@ -6,98 +6,148 @@ import ( ) const ( - daysInWeek = 7 + daysInWeek = 7 + monthsInQuarter = 3 ) -type weeklyPartition struct { - t time.Time -} - -func (d weeklyPartition) PartitionName() string { - return fmt.Sprintf("calls_p_%d_w%02d_%02d", d.t.Year(), d.t.Month(), d.t.Day()) -} - -func (d weeklyPartition) Range() (start, end time.Time) { - start = time.Date(d.t.Year(), d.t.Month(), d.t.Day(), 0, 0, 0, 0, d.t.UTC().Location()).AddDate(0, 0, -int(d.t.Weekday()-time.Monday)) - start = start.AddDate(0, 0, daysInWeek) +func getDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC) + upperBound = lowerBound.AddDate(0, 0, 1) return } -func (d weeklyPartition) Next(i int) partition { - year, month, day := d.t.Date() - - return weeklyPartition{ - t: time.Date(year, month, day+i, 0, 0, 0, 0, time.UTC), - } -} - -func (d weeklyPartition) Prev(i int) partition { - year, month, day := d.t.Date() - - return weeklyPartition{ - t: time.Date(year, month, day-i, 0, 0, 0, 0, d.t.Location()), - } -} - -type dailyPartition struct { - t time.Time -} - -func (d dailyPartition) PartitionName() string { - return fmt.Sprintf("calls_p_%d_%02d_%02d", d.t.Year(), d.t.Month(), d.t.Day()) -} - -func (d dailyPartition) Range() (start, end time.Time) { - start = time.Date(d.t.Year(), d.t.Month(), d.t.Day(), 0, 0, 0, 0, time.UTC) - end = start.AddDate(0, 0, 1) +func getWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, -int(date.Weekday()-time.Monday)) + upperBound = lowerBound.AddDate(0, 0, daysInWeek) return } -func (d dailyPartition) Next(i int) partition { - year, month, day := d.t.Date() - - return dailyPartition{ - t: time.Date(year, month, day+i, 0, 0, 0, 0, time.UTC), - } -} - -func (d dailyPartition) Prev(i int) partition { - year, month, day := d.t.Date() - - return dailyPartition{ - t: time.Date(year, month, day-i, 0, 0, 0, 0, d.t.Location()), - } -} - -type monthlyPartition struct { - t time.Time -} - -func (d monthlyPartition) PartitionName() string { - return fmt.Sprintf("calls_p_%d_%02d", d.t.Year(), d.t.Month()) -} - -func (d monthlyPartition) Range() (start, end time.Time) { - start = time.Date(d.t.Year(), d.t.Month(), 1, 0, 0, 0, 0, time.UTC) - end = start.AddDate(0, 1, 0) +func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC) + upperBound = lowerBound.AddDate(0, 1, 0) return } -func (d monthlyPartition) Next(i int) partition { - year, month, _ := d.t.Date() +func getQuarterlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + year, _, _ := date.Date() - return monthlyPartition{ - t: time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, time.UTC), - } + quarter := (int(date.Month()) - 1) / monthsInQuarter + firstMonthOfTheQuarter := time.Month(quarter*monthsInQuarter + 1) + + lowerBound = time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) + upperBound = lowerBound.AddDate(0, monthsInQuarter, 0) + + return } -func (d monthlyPartition) Prev(i int) partition { - year, month, _ := d.t.Date() +func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + upperBound = lowerBound.AddDate(1, 0, 0) + + return +} + +func (p partition) Next(i int) partition { + + var t time.Time + switch p.Interval { + case Daily: + t = p.Time.AddDate(0, 0, i) + case Weekly: + t = p.Time.AddDate(0, 0, i*daysInWeek) + case Monthly: + year, month, _ := p.Time.Date() + + t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, p.Time.Location()) + case Quarterly: + t = p.Time.AddDate(0, i*monthsInQuarter, 0) + case Yearly: + year, _, _ := p.Time.Date() + + t = time.Date(year+i, 1, 1, 0, 0, 0, 0, p.Time.Location()) + } + np := partition{ + ParentTable: p.ParentTable, + Name: p.Name, + Schema: p.Schema, + Interval: p.Interval, + Time: t, + } + + np.setName() + + return np +} + +func (p *partition) setName() { + t := p.Time + var suffix string + + switch p.Interval { + case Daily: + suffix = t.Format("2006_01_02") + case Weekly: + year, week := t.ISOWeek() + suffix = fmt.Sprintf("%d_w%02d", year, week) + case Monthly: + suffix = t.Format("2006_01") + case Quarterly: + year, month, _ := t.Date() + + var quarter int + + switch { + case month >= 1 && month <= 3: + quarter = 1 + case month >= 4 && month <= 6: + quarter = 2 + case month >= 7 && month <= 9: + quarter = 3 + case month >= 10 && month <= 12: + quarter = 4 + } + + suffix = fmt.Sprintf("%d_q%d", year, quarter) + case Yearly: + suffix = t.Format("2006") + default: + panic(ErrInvalidInterval(p.Interval)) + } + + p.Name = fmt.Sprintf("%s_p_%s", p.ParentTable, suffix) +} + +func (p partition) Prev(i int) partition { + var t time.Time + switch p.Interval { + case Daily: + t = p.Time.AddDate(0, 0, -i) + case Weekly: + t = p.Time.AddDate(0, 0, -i*daysInWeek) + case Monthly: + year, month, _ := p.Time.Date() + + t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, p.Time.Location()) + case Quarterly: + t = p.Time.AddDate(0, -i*monthsInQuarter, 0) + case Yearly: + year, _, _ := p.Time.Date() + + t = time.Date(year-i, 1, 1, 0, 0, 0, 0, p.Time.Location()) + } + + pp := partition{ + ParentTable: p.ParentTable, + Name: p.Name, + Schema: p.Schema, + Interval: p.Interval, + Time: t, + } + pp.setName() + + return pp - return monthlyPartition{ - t: time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, d.t.Location()), - } } diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index e973649..640b547 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -1,5 +1,7 @@ package partman +// portions lifted gratefully from github.com/qonto/postgresql-partition-manager, MIT license. + import ( "context" "fmt" @@ -7,6 +9,7 @@ import ( "strings" "time" + "dynatron.me/x/stillbox/internal/isoweek" "dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/database" @@ -22,7 +25,7 @@ const ( ) type partitionError struct { - p string + p string err error } @@ -39,6 +42,14 @@ func (pe partitionError) Unwrap() error { return pe.err } +type parsedIntervalError struct { + parsed, start time.Time +} + +func (pie parsedIntervalError) Error() string { + return fmt.Sprintf("parsed interval (%s) does not match start (%s)", pie.parsed, pie.start) +} + func PartitionError(pname string, err ...error) partitionError { if len(err) > 0 { return partitionError{p: pname, err: err[0]} @@ -84,11 +95,12 @@ type partman struct { intv Interval } -type partition interface { - PartitionName() string - Next(i int) partition - Prev(i int) partition - Range() (time.Time, time.Time) +type partition struct { + ParentTable string + Schema string + Name string + Interval Interval + Time time.Time } func New(db database.Store, cfg config.Partition) (*partman, error) { @@ -124,12 +136,16 @@ func (pm *partman) Go(ctx context.Context) { } func (pm *partman) newPartition(t time.Time) partition { - switch pm.intv { - case Monthly: - return monthlyPartition{t} + p := partition{ + ParentTable: callsTable, + Schema: pm.cfg.Schema, + Interval: Interval(pm.cfg.Interval), + Time: t, } - return nil + p.setName() + + return p } func (pm *partman) retentionPartitions(cur partition) []partition { @@ -200,9 +216,9 @@ func (pm *partman) existingPartitions(parts []database.PartitionResult) ([]parti existing := make([]partition, 0, len(parts)) for _, v := range parts { if v.Schema != pm.cfg.Schema { - return nil, PartitionError(v.Schema+"."+v.Name+" has wrong schema") + return nil, PartitionError(v.Schema + "." + v.Name + " has wrong schema") } - p, _, err := pm.parsePart(v) + p, err := pm.parsePart(v) if err != nil { return nil, err } @@ -279,6 +295,27 @@ func (pm *partman) Check(ctx context.Context, now time.Time) error { }, pgx.TxOptions{}) } +func (p partition) Range() (time.Time, time.Time) { + switch p.Interval { + case Daily: + return getDailyBounds(p.Time) + case Weekly: + return getWeeklyBounds(p.Time) + case Monthly: + return getMonthlyBounds(p.Time) + case Quarterly: + return getQuarterlyBounds(p.Time) + case Yearly: + return getYearlyBounds(p.Time) + } + + panic("unknown interval!") +} + +func (p partition) PartitionName() string { + return p.Name +} + func (pm *partman) createPartition(ctx context.Context, tx database.Store, part partition) error { start, end := part.Range() return tx.CreatePartition(ctx, callsTable, part.PartitionName(), start, end) @@ -293,66 +330,76 @@ func (pm *partman) createPartition(ctx context.Context, tx database.Store, part * yearly: calls_p_2024 */ -func (pm *partman) parsePart(pr database.PartitionResult) (partition, Interval, error) { - intv := Unknown +func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err error) { pn := pr.Name - low, up, err := pr.ParseBounds() + low, _, err := pr.ParseBounds() if err != nil { - return nil, intv, err + return + } + p = partition{ + ParentTable: pr.ParentTable, + Name: pr.Name, + Schema: pr.Schema, + Time: low, } dateAr := strings.Split(pn, "calls_p_") if len(dateAr) != 2 { - return nil, intv, PartitionError(pn) + return p, PartitionError(pn) } dateAr = strings.Split(dateAr[1], "_") switch len(dateAr) { case 3: // daily - intv = Daily + p.Interval = Daily ymd := [3]int{} for i := 0; i < 3; i++ { r, err := strconv.Atoi(dateAr[i]) if err != nil { - return nil, intv, PartitionError(pn, err) + return p, PartitionError(pn, err) } ymd[i] = r } - return dailyPartition{time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC)}, Daily, nil + parsed := time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC) + if parsed != p.Time { + return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + } + return p, nil case 2: year, err := strconv.Atoi(dateAr[0]) if err != nil { - return nil, intv, PartitionError(pn, err) + return p, PartitionError(pn, err) } if strings.HasPrefix(dateAr[1], "w") { - intv = Weekly + p.Interval = Weekly weekNum, err := strconv.Atoi(dateAr[1][1:]) if err != nil { - return nil, intv, PartitionError(pn, err) + return p, PartitionError(pn, err) } - return weeklyPartition{time.Date(year, 0, daysInWeek*weekNum, 0, 0, 0, 0, time.UTC)}, intv, nil + parsed := isoweek.StartTime(year, weekNum, time.UTC) + if parsed != p.Time { + return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + } + return p, nil } else if strings.HasPrefix(dateAr[1], "q") { panic("not implemented") } // monthly + p.Interval = Monthly month, err := strconv.Atoi(dateAr[1]) if err != nil { - return nil, intv, PartitionError(pn) + return p, PartitionError(pn) } - return monthlyPartition{time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC)}, intv, nil + parsed := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + if parsed != p.Time { + return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + } + return p, nil case 1: // yearly panic("not implemented") } - return nil, intv, PartitionError(pn) -} - -type partMap map[partition]struct{} - -func (pm partMap) exists(dt partition) bool { - _, ex := pm[dt] - - return ex + return p, PartitionError(pn) } diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index a515157..bccea4d 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -53,11 +53,21 @@ func TestPartman(t *testing.T) { return t } + etb := func(name, low, up string) database.PartitionResult { + return database.PartitionResult{ + ParentTable: "calls", + Schema: "public", + Name: name, + LowerBound: low, + UpperBound: up, + } + } + tests := []struct { name string now time.Time cfg config.Partition - extant []string + extant []database.PartitionResult expectCreate []string expectDrop []string expectDetach []string @@ -75,11 +85,11 @@ func TestPartman(t *testing.T) { Drop: true, PreProvision: common.PtrTo(2), }, - extant: []string{ - "calls_p_2024_10", - "calls_p_2024_09", - "calls_p_2024_08", - "calls_p_2024_07", + extant: []database.PartitionResult{ + etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), + etb("calls_p_2024_09", "2024-09-01", "2024-10-01"), + etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), + etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, expectCreate: []string{ "calls_p_2024_11", @@ -114,21 +124,24 @@ func TestPartman(t *testing.T) { Drop: false, PreProvision: common.PtrTo(2), }, - extant: []string{ - "calls_p_2024_w44", - "calls_p_2024_w45", - "calls_p_2024_w46", + extant: []database.PartitionResult{ + etb("calls_p_2024_w44", "2024-10-28", "2024-11-04"), + etb("calls_p_2024_w45", "2024-11-04", "2024-11-11"), + etb("calls_p_2024_w46", "2024-11-11", "2024-11-18"), }, expectCreate: []string{ + "calls_p_2024_w47", "calls_p_2024_w48", "calls_p_2024_w49", "calls_p_2024_w50", }, expectSweep: []timeRange{ - // TODO + timeRange{start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, + timeRange{start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, }, expectCleanup: []timeRange{ - // TODO + timeRange{start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, + timeRange{start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, }, expectDetach: []string{ "public.calls_p_2024_w44", @@ -146,34 +159,60 @@ func TestPartman(t *testing.T) { cleanupRanges := make([]timeRange, 0, len(tc.expectCleanup)) detachedPartitions := make([]string, 0, len(tc.expectDetach)) - db.EXPECT().CreatePartition(mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), mock.AnythingOfType("time.Time")).Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { - createdPartitions = append(createdPartitions, partitionName) - }).Return(nil) - db.EXPECT().SweepCalls(mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz")).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { - sweptRanges = append(sweptRanges, timeRange{start: start.Time, end: end.Time}) - }).Return(nil) - db.EXPECT().CleanupSweptCalls(mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz")).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + db.EXPECT(). + CreatePartition( + mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), + mock.AnythingOfType("time.Time"), + ). + Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { + createdPartitions = append(createdPartitions, partitionName) + }).Return(nil) + + db.EXPECT(). + SweepCalls( + mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), + ). + Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + sweptRanges = append(sweptRanges, timeRange{start: start.Time, end: end.Time}) + }).Return(nil) + + db.EXPECT(). + CleanupSweptCalls( + mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), + ).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { cleanupRanges = append(cleanupRanges, timeRange{start: start.Time, end: end.Time}) }).Return(nil) - db.EXPECT().DropPartition(mctx, mock.AnythingOfType("string")).Run(func(ctx context.Context, partName string) { - droppedPartitions = append(droppedPartitions, partName) - }).Return(nil) - db.EXPECT().DetachPartition(mctx, mock.AnythingOfType("string")).Run(func(ctx context.Context, partName string) { - detachedPartitions = append(detachedPartitions, partName) - }).Return(nil) + + if tc.cfg.Drop { + db.EXPECT(). + DropPartition(mctx, mock.AnythingOfType("string")). + Run(func(ctx context.Context, partName string) { + droppedPartitions = append(droppedPartitions, partName) + }).Return(nil) + } + + db.EXPECT(). + DetachPartition( + mctx, mock.AnythingOfType("string")). + Run(func(ctx context.Context, partName string) { + detachedPartitions = append(detachedPartitions, partName) + }).Return(nil) + inTx(db) + db.EXPECT().GetTablePartitions(mctx, "public", "calls").Return(tc.extant, nil) + pm, err := partman.New(db, tc.cfg) require.NoError(t, err) err = pm.Check(ctx, tc.now) require.NoError(t, err) - assert.ElementsMatch(t, tc.expectCreate, createdPartitions) - assert.ElementsMatch(t, tc.expectSweep, sweptRanges) - assert.ElementsMatch(t, tc.expectDrop, droppedPartitions) - assert.ElementsMatch(t, tc.expectCleanup, cleanupRanges) - assert.ElementsMatch(t, tc.expectDetach, detachedPartitions) + assert.ElementsMatch(t, tc.expectCreate, createdPartitions, "created partitions") + assert.ElementsMatch(t, tc.expectSweep, sweptRanges, "swept ranges") + assert.ElementsMatch(t, tc.expectDrop, droppedPartitions, "dropped partitions") + assert.ElementsMatch(t, tc.expectCleanup, cleanupRanges, "cleaned up ranges") + assert.ElementsMatch(t, tc.expectDetach, detachedPartitions, "detached partitions") }) } } -- 2.48.1 From ecf282ce0ad9e8d6d9dda24688d342a728ab976a Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 11:10:17 -0500 Subject: [PATCH 06/15] fix lint --- pkg/config/parse.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/config/parse.go b/pkg/config/parse.go index f8e9ac0..e8ba75f 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -40,10 +40,13 @@ func (c *Configuration) read() error { return err } - k.Load(env.Provider(common.EnvPrefix, ".", func(s string) string { + err = k.Load(env.Provider(common.EnvPrefix, ".", func(s string) string { return strings.Replace(strings.ToLower( strings.TrimPrefix(s, common.EnvPrefix)), "_", ".", -1) }), nil) + if err != nil { + return err + } err = k.UnmarshalWithConf("", &c.Config, koanf.UnmarshalConf{ -- 2.48.1 From a22c716d9961ec8ab7e2ef81c2ea2ebeef9a2d08 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 12:27:59 -0500 Subject: [PATCH 07/15] quarter math is all wrong --- pkg/database/partman/intervals.go | 1 - pkg/database/partman/partman.go | 46 +++-- pkg/database/partman/partman_test.go | 255 ++++++++++++++++++++++----- 3 files changed, 246 insertions(+), 56 deletions(-) diff --git a/pkg/database/partman/intervals.go b/pkg/database/partman/intervals.go index 3f37012..b80d330 100644 --- a/pkg/database/partman/intervals.go +++ b/pkg/database/partman/intervals.go @@ -51,7 +51,6 @@ func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { } func (p partition) Next(i int) partition { - var t time.Time switch p.Interval { case Daily: diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index 640b547..b2fd65f 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -4,6 +4,7 @@ package partman import ( "context" + "errors" "fmt" "strconv" "strings" @@ -24,12 +25,17 @@ const ( preProvisionDefault = 1 ) -type partitionError struct { +var ( + ErrWrongSchema = errors.New("wrong schema name") + ErrDifferentInterval = errors.New("stored partition interval differs from configured") +) + +type PartitionErr struct { p string err error } -func (pe partitionError) Error() string { +func (pe PartitionErr) Error() string { r := fmt.Sprintf("bad partition '%s'", pe.p) if pe.err != nil { r += ": " + pe.err.Error() @@ -38,24 +44,24 @@ func (pe partitionError) Error() string { return r } -func (pe partitionError) Unwrap() error { +func (pe PartitionErr) Unwrap() error { return pe.err } -type parsedIntervalError struct { +type ParsedIntvlErr struct { parsed, start time.Time } -func (pie parsedIntervalError) Error() string { +func (pie ParsedIntvlErr) Error() string { return fmt.Sprintf("parsed interval (%s) does not match start (%s)", pie.parsed, pie.start) } -func PartitionError(pname string, err ...error) partitionError { +func PartitionError(pname string, err ...error) PartitionErr { if len(err) > 0 { - return partitionError{p: pname, err: err[0]} + return PartitionErr{p: pname, err: err[0]} } - return partitionError{p: pname} + return PartitionErr{p: pname} } type ErrInvalidInterval string @@ -216,13 +222,17 @@ func (pm *partman) existingPartitions(parts []database.PartitionResult) ([]parti existing := make([]partition, 0, len(parts)) for _, v := range parts { if v.Schema != pm.cfg.Schema { - return nil, PartitionError(v.Schema + "." + v.Name + " has wrong schema") + return nil, PartitionError(v.Schema+"."+v.Name, ErrWrongSchema) } p, err := pm.parsePart(v) if err != nil { return nil, err } + if p.Interval != Interval(pm.cfg.Interval) { + return nil, PartitionError(v.Schema+"."+v.Name, ErrDifferentInterval) + } + existing = append(existing, p) } return existing, nil @@ -362,7 +372,7 @@ func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err erro } parsed := time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC) if parsed != p.Time { - return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) } return p, nil case 2: @@ -379,11 +389,21 @@ func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err erro parsed := isoweek.StartTime(year, weekNum, time.UTC) if parsed != p.Time { - return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) } return p, nil } else if strings.HasPrefix(dateAr[1], "q") { - panic("not implemented") + p.Interval = Quarterly + quarterNum, err := strconv.Atoi(dateAr[1][1:]) + if err != nil { + return p, PartitionError(pn, err) + } + firstMonthOfTheQuarter := time.Month(quarterNum*monthsInQuarter + 1) + parsed := time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) + if parsed != p.Time { + return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) + } + return p, nil } // monthly p.Interval = Monthly @@ -394,7 +414,7 @@ func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err erro parsed := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) if parsed != p.Time { - return p, PartitionError(pn, parsedIntervalError{parsed: parsed, start: p.Time}) + return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) } return p, nil case 1: // yearly diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index bccea4d..371024e 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -53,16 +53,20 @@ func TestPartman(t *testing.T) { return t } - etb := func(name, low, up string) database.PartitionResult { + etbWithSchema := func(schema, name, low, up string) database.PartitionResult { return database.PartitionResult{ ParentTable: "calls", - Schema: "public", + Schema: schema, Name: name, LowerBound: low, UpperBound: up, } } + etb := func(name, low, up string) database.PartitionResult { + return etbWithSchema("public", name, low, up) + } + tests := []struct { name string now time.Time @@ -73,6 +77,7 @@ func TestPartman(t *testing.T) { expectDetach []string expectSweep []timeRange expectCleanup []timeRange + expectErr error }{ { name: "monthly base", @@ -148,6 +153,160 @@ func TestPartman(t *testing.T) { "public.calls_p_2024_w45", }, }, + { + name: "daily base", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "daily", + Retain: 2, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + etb("calls_p_2024_11_24", "2024-11-24", "2024-11-25"), + etb("calls_p_2024_11_25", "2024-11-25", "2024-11-26"), + etb("calls_p_2024_11_26", "2024-11-26", "2024-11-27"), + etb("calls_p_2024_11_27", "2024-11-27", "2024-11-28"), + }, + expectCreate: []string{ + "calls_p_2024_11_28", + "calls_p_2024_11_29", + "calls_p_2024_11_30", + }, + expectDrop: []string{ + "public.calls_p_2024_11_24", + "public.calls_p_2024_11_25", + }, + expectSweep: []timeRange{ + timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + }, + expectCleanup: []timeRange{ + timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + }, + expectDetach: []string{ + "public.calls_p_2024_11_24", + "public.calls_p_2024_11_25", + }, + }, + { + name: "quarterly base", + now: timeInUTC("2025-07-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "quarterly", + Retain: 2, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + etb("calls_p_2024_q3", "2024-07-01", "2024-10-01"), + etb("calls_p_2024_q4", "2024-10-01", "2025-01-01"), + etb("calls_p_2025_q1", "2025-01-01", "2024-04-01"), + etb("calls_p_2025_q2", "2025-04-01", "2024-07-01"), + }, + expectCreate: []string{}, + expectDrop: []string{ + "public.calls_p_2024_11_24", + "public.calls_p_2024_11_25", + }, + expectSweep: []timeRange{ + timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + }, + expectCleanup: []timeRange{ + timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + }, + expectDetach: []string{ + "public.calls_p_2024_11_24", + "public.calls_p_2024_11_25", + }, + }, + { + name: "changed monthly to daily", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "daily", + Retain: 2, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), + etb("calls_p_2024_09", "2024-09-01", "2024-10-01"), + etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), + etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), + }, + expectCreate: []string{ + "calls_p_2024_11", + "calls_p_2024_12", + "calls_p_2025_01", + }, + expectDrop: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + expectSweep: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectCleanup: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectDetach: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + expectErr: partman.ErrDifferentInterval, + }, + { + name: "monthly wrong schema", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "monthly", + Retain: 2, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), + etbWithSchema("reid", "calls_p_2024_09", "2024-09-01", "2024-10-01"), + etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), + etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), + }, + expectCreate: []string{ + "calls_p_2024_11", + "calls_p_2024_12", + "calls_p_2025_01", + }, + expectDrop: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + expectSweep: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectCleanup: []timeRange{ + timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + }, + expectDetach: []string{ + "public.calls_p_2024_07", + "public.calls_p_2024_08", + }, + expectErr: partman.ErrWrongSchema, + }, } for _, tc := range tests { @@ -158,46 +317,54 @@ func TestPartman(t *testing.T) { droppedPartitions := make([]string, 0, len(tc.expectDrop)) cleanupRanges := make([]timeRange, 0, len(tc.expectCleanup)) detachedPartitions := make([]string, 0, len(tc.expectDetach)) + sweepMap := make(map[timeRange]struct{}) - db.EXPECT(). - CreatePartition( - mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), - mock.AnythingOfType("time.Time"), - ). - Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { - createdPartitions = append(createdPartitions, partitionName) - }).Return(nil) - - db.EXPECT(). - SweepCalls( - mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), - ). - Run(func(ctx context.Context, start, end pgtype.Timestamptz) { - sweptRanges = append(sweptRanges, timeRange{start: start.Time, end: end.Time}) - }).Return(nil) - - db.EXPECT(). - CleanupSweptCalls( - mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), - ).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { - cleanupRanges = append(cleanupRanges, timeRange{start: start.Time, end: end.Time}) - }).Return(nil) - - if tc.cfg.Drop { + if tc.expectErr == nil { db.EXPECT(). - DropPartition(mctx, mock.AnythingOfType("string")). + CreatePartition( + mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), + mock.AnythingOfType("time.Time"), + ). + Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { + createdPartitions = append(createdPartitions, partitionName) + }).Return(nil) + + db.EXPECT(). + SweepCalls( + mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), + ). + Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + tr := timeRange{start: start.Time, end: end.Time} + sweepMap[tr] = struct{}{} + sweptRanges = append(sweptRanges, tr) + }).Return(nil) + + db.EXPECT(). + CleanupSweptCalls( + mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), + ).Run(func(ctx context.Context, start, end pgtype.Timestamptz) { + tr := timeRange{start: start.Time, end: end.Time} + require.Contains(t, sweepMap, tr) + + cleanupRanges = append(cleanupRanges, tr) + }).Return(nil) + + if tc.cfg.Drop { + db.EXPECT(). + DropPartition(mctx, mock.AnythingOfType("string")). + Run(func(ctx context.Context, partName string) { + droppedPartitions = append(droppedPartitions, partName) + }).Return(nil) + } + + db.EXPECT(). + DetachPartition( + mctx, mock.AnythingOfType("string")). Run(func(ctx context.Context, partName string) { - droppedPartitions = append(droppedPartitions, partName) + detachedPartitions = append(detachedPartitions, partName) }).Return(nil) } - db.EXPECT(). - DetachPartition( - mctx, mock.AnythingOfType("string")). - Run(func(ctx context.Context, partName string) { - detachedPartitions = append(detachedPartitions, partName) - }).Return(nil) - inTx(db) db.EXPECT().GetTablePartitions(mctx, "public", "calls").Return(tc.extant, nil) @@ -206,13 +373,17 @@ func TestPartman(t *testing.T) { require.NoError(t, err) err = pm.Check(ctx, tc.now) - require.NoError(t, err) + if tc.expectErr != nil { + assert.ErrorIs(t, err, tc.expectErr) + } else { + require.NoError(t, err) - assert.ElementsMatch(t, tc.expectCreate, createdPartitions, "created partitions") - assert.ElementsMatch(t, tc.expectSweep, sweptRanges, "swept ranges") - assert.ElementsMatch(t, tc.expectDrop, droppedPartitions, "dropped partitions") - assert.ElementsMatch(t, tc.expectCleanup, cleanupRanges, "cleaned up ranges") - assert.ElementsMatch(t, tc.expectDetach, detachedPartitions, "detached partitions") + assert.ElementsMatch(t, tc.expectCreate, createdPartitions, "created partitions") + assert.ElementsMatch(t, tc.expectSweep, sweptRanges, "swept ranges") + assert.ElementsMatch(t, tc.expectDrop, droppedPartitions, "dropped partitions") + assert.ElementsMatch(t, tc.expectCleanup, cleanupRanges, "cleaned up ranges") + assert.ElementsMatch(t, tc.expectDetach, detachedPartitions, "detached partitions") + } }) } } -- 2.48.1 From 76cd199ec596e535af441073b48a78e677789c1c Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 16:53:40 -0500 Subject: [PATCH 08/15] weekly pass --- pkg/database/partman/partman.go | 17 +++- pkg/database/partman/partman_test.go | 116 +++++++++------------------ 2 files changed, 53 insertions(+), 80 deletions(-) diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index b2fd65f..668e071 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -224,7 +224,7 @@ func (pm *partman) existingPartitions(parts []database.PartitionResult) ([]parti if v.Schema != pm.cfg.Schema { return nil, PartitionError(v.Schema+"."+v.Name, ErrWrongSchema) } - p, err := pm.parsePart(v) + p, err := pm.verifyPartName(v) if err != nil { return nil, err } @@ -340,7 +340,7 @@ func (pm *partman) createPartition(ctx context.Context, tx database.Store, part * yearly: calls_p_2024 */ -func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err error) { +func (pm *partman) verifyPartName(pr database.PartitionResult) (p partition, err error) { pn := pr.Name low, _, err := pr.ParseBounds() if err != nil { @@ -398,7 +398,7 @@ func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err erro if err != nil { return p, PartitionError(pn, err) } - firstMonthOfTheQuarter := time.Month(quarterNum*monthsInQuarter + 1) + firstMonthOfTheQuarter := time.Month((quarterNum-1)*monthsInQuarter + 1) parsed := time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) if parsed != p.Time { return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) @@ -418,7 +418,16 @@ func (pm *partman) parsePart(pr database.PartitionResult) (p partition, err erro } return p, nil case 1: // yearly - panic("not implemented") + year, err := strconv.Atoi(dateAr[0]) + if err != nil { + return p, PartitionError(pn, err) + } + parsed := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC) + if parsed != p.Time { + return p, PartitionError(pn, ParsedIntvlErr{parsed: parsed, start: p.Time}) + } + + return p, nil } return p, PartitionError(pn) diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index 371024e..5719e7f 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -32,6 +32,11 @@ type timeRange struct { end time.Time } +type partSpec struct { + name string + timeRange +} + func TestPartman(t *testing.T) { ctx := context.Background() @@ -72,7 +77,7 @@ func TestPartman(t *testing.T) { now time.Time cfg config.Partition extant []database.PartitionResult - expectCreate []string + expectCreate []partSpec expectDrop []string expectDetach []string expectSweep []timeRange @@ -96,22 +101,22 @@ func TestPartman(t *testing.T) { etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, - expectCreate: []string{ - "calls_p_2024_11", - "calls_p_2024_12", - "calls_p_2025_01", + expectCreate: []partSpec{ + {name:"calls_p_2024_11", timeRange: timeRange{start: dateInUTC("2024-11-01"), end: dateInUTC("2024-12-01")}}, + {name:"calls_p_2024_12", timeRange: timeRange{start: dateInUTC("2024-12-01"), end: dateInUTC("2025-01-01")}}, + {name:"calls_p_2025_01", timeRange: timeRange{start: dateInUTC("2025-01-01"), end: dateInUTC("2025-02-01")}}, }, expectDrop: []string{ "public.calls_p_2024_07", "public.calls_p_2024_08", }, expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + {start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + {start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, }, expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, + {start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, + {start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, }, expectDetach: []string{ "public.calls_p_2024_07", @@ -120,7 +125,7 @@ func TestPartman(t *testing.T) { }, { name: "weekly base", - now: timeInUTC("2024-11-28 11:37:04"), + now: timeInUTC("2024-11-28 11:37:04"), // week 48 cfg: config.Partition{ Enabled: true, Schema: "public", @@ -133,20 +138,21 @@ func TestPartman(t *testing.T) { etb("calls_p_2024_w44", "2024-10-28", "2024-11-04"), etb("calls_p_2024_w45", "2024-11-04", "2024-11-11"), etb("calls_p_2024_w46", "2024-11-11", "2024-11-18"), + // missing week 47 }, - expectCreate: []string{ - "calls_p_2024_w47", - "calls_p_2024_w48", - "calls_p_2024_w49", - "calls_p_2024_w50", + expectCreate: []partSpec{ + {name: "calls_p_2024_w47", timeRange: timeRange{start: dateInUTC("2024-11-18"), end: dateInUTC("2024-11-25")}}, + {name: "calls_p_2024_w48", timeRange: timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-12-02")}}, + {name: "calls_p_2024_w49", timeRange: timeRange{start: dateInUTC("2024-12-02"), end: dateInUTC("2024-12-09")}}, + {name: "calls_p_2024_w50", timeRange: timeRange{start: dateInUTC("2024-12-09"), end: dateInUTC("2024-12-16")}}, }, expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, - timeRange{start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, + {start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, + {start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, }, expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, - timeRange{start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, + {start: dateInUTC("2024-10-28"), end: dateInUTC("2024-11-04")}, + {start: dateInUTC("2024-11-04"), end: dateInUTC("2024-11-11")}, }, expectDetach: []string{ "public.calls_p_2024_w44", @@ -170,22 +176,22 @@ func TestPartman(t *testing.T) { etb("calls_p_2024_11_26", "2024-11-26", "2024-11-27"), etb("calls_p_2024_11_27", "2024-11-27", "2024-11-28"), }, - expectCreate: []string{ - "calls_p_2024_11_28", - "calls_p_2024_11_29", - "calls_p_2024_11_30", + expectCreate: []partSpec{ + {name: "calls_p_2024_11_28"}, + {name: "calls_p_2024_11_29"}, + {name: "calls_p_2024_11_30"}, }, expectDrop: []string{ "public.calls_p_2024_11_24", "public.calls_p_2024_11_25", }, expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, }, expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, }, expectDetach: []string{ "public.calls_p_2024_11_24", @@ -209,18 +215,17 @@ func TestPartman(t *testing.T) { etb("calls_p_2025_q1", "2025-01-01", "2024-04-01"), etb("calls_p_2025_q2", "2025-04-01", "2024-07-01"), }, - expectCreate: []string{}, expectDrop: []string{ "public.calls_p_2024_11_24", "public.calls_p_2024_11_25", }, expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, }, expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - timeRange{start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, + {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, }, expectDetach: []string{ "public.calls_p_2024_11_24", @@ -244,27 +249,6 @@ func TestPartman(t *testing.T) { etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, - expectCreate: []string{ - "calls_p_2024_11", - "calls_p_2024_12", - "calls_p_2025_01", - }, - expectDrop: []string{ - "public.calls_p_2024_07", - "public.calls_p_2024_08", - }, - expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, - }, - expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, - }, - expectDetach: []string{ - "public.calls_p_2024_07", - "public.calls_p_2024_08", - }, expectErr: partman.ErrDifferentInterval, }, { @@ -284,27 +268,6 @@ func TestPartman(t *testing.T) { etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, - expectCreate: []string{ - "calls_p_2024_11", - "calls_p_2024_12", - "calls_p_2025_01", - }, - expectDrop: []string{ - "public.calls_p_2024_07", - "public.calls_p_2024_08", - }, - expectSweep: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, - }, - expectCleanup: []timeRange{ - timeRange{start: dateInUTC("2024-07-01"), end: dateInUTC("2024-08-01")}, - timeRange{start: dateInUTC("2024-08-01"), end: dateInUTC("2024-09-01")}, - }, - expectDetach: []string{ - "public.calls_p_2024_07", - "public.calls_p_2024_08", - }, expectErr: partman.ErrWrongSchema, }, } @@ -312,7 +275,7 @@ func TestPartman(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { db := mocks.NewStore(t) - createdPartitions := make([]string, 0, len(tc.expectCreate)) + createdPartitions := make([]partSpec, 0, len(tc.expectCreate)) sweptRanges := make([]timeRange, 0, len(tc.expectSweep)) droppedPartitions := make([]string, 0, len(tc.expectDrop)) cleanupRanges := make([]timeRange, 0, len(tc.expectCleanup)) @@ -326,7 +289,8 @@ func TestPartman(t *testing.T) { mock.AnythingOfType("time.Time"), ). Run(func(ctx context.Context, tableName, partitionName string, start, end time.Time) { - createdPartitions = append(createdPartitions, partitionName) + ps := partSpec{name: partitionName, timeRange: timeRange{start: start, end: end}} + createdPartitions = append(createdPartitions, ps) }).Return(nil) db.EXPECT(). -- 2.48.1 From af9cd3ff48c6cf59a92a76af342f777fa2c1b2f0 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 17:33:15 -0500 Subject: [PATCH 09/15] daily --- pkg/database/partman/partman.go | 3 + pkg/database/partman/partman_test.go | 82 +++++++++++++++------------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index 668e071..0c84b60 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -398,6 +398,9 @@ func (pm *partman) verifyPartName(pr database.PartitionResult) (p partition, err if err != nil { return p, PartitionError(pn, err) } + if quarterNum > 4 { + return p, PartitionError(pn, errors.New("invalid quarter")) + } firstMonthOfTheQuarter := time.Month((quarterNum-1)*monthsInQuarter + 1) parsed := time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, time.UTC) if parsed != p.Time { diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index 5719e7f..28bf8e8 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -58,7 +58,7 @@ func TestPartman(t *testing.T) { return t } - etbWithSchema := func(schema, name, low, up string) database.PartitionResult { + partResultWithSchema := func(schema, name, low, up string) database.PartitionResult { return database.PartitionResult{ ParentTable: "calls", Schema: schema, @@ -68,8 +68,14 @@ func TestPartman(t *testing.T) { } } - etb := func(name, low, up string) database.PartitionResult { - return etbWithSchema("public", name, low, up) + partResult := func(name, low, up string) database.PartitionResult { + return partResultWithSchema("public", name, low, up) + } + + dailyTR := func(tr string) timeRange { + dtr := dateInUTC(tr) + etr := dtr.AddDate(0, 0, 1) + return timeRange{start: dtr, end: etr} } tests := []struct { @@ -96,10 +102,10 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), - etb("calls_p_2024_09", "2024-09-01", "2024-10-01"), - etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), - etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), + partResult("calls_p_2024_10", "2024-10-01", "2024-11-01"), + partResult("calls_p_2024_09", "2024-09-01", "2024-10-01"), + partResult("calls_p_2024_08", "2024-08-01", "2024-09-01"), + partResult("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, expectCreate: []partSpec{ {name:"calls_p_2024_11", timeRange: timeRange{start: dateInUTC("2024-11-01"), end: dateInUTC("2024-12-01")}}, @@ -135,9 +141,9 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_w44", "2024-10-28", "2024-11-04"), - etb("calls_p_2024_w45", "2024-11-04", "2024-11-11"), - etb("calls_p_2024_w46", "2024-11-11", "2024-11-18"), + partResult("calls_p_2024_w44", "2024-10-28", "2024-11-04"), + partResult("calls_p_2024_w45", "2024-11-04", "2024-11-11"), + partResult("calls_p_2024_w46", "2024-11-11", "2024-11-18"), // missing week 47 }, expectCreate: []partSpec{ @@ -161,7 +167,7 @@ func TestPartman(t *testing.T) { }, { name: "daily base", - now: timeInUTC("2024-11-28 11:37:04"), + now: timeInUTC("2024-12-31 11:37:04"), cfg: config.Partition{ Enabled: true, Schema: "public", @@ -171,31 +177,31 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_11_24", "2024-11-24", "2024-11-25"), - etb("calls_p_2024_11_25", "2024-11-25", "2024-11-26"), - etb("calls_p_2024_11_26", "2024-11-26", "2024-11-27"), - etb("calls_p_2024_11_27", "2024-11-27", "2024-11-28"), + partResult("calls_p_2024_12_26", "2024-12-26", "2024-12-27"), + partResult("calls_p_2024_12_27", "2024-12-27", "2024-12-28"), + partResult("calls_p_2024_12_30", "2024-12-30", "2024-12-31"), + partResult("calls_p_2024_12_31", "2024-12-31", "2025-01-01"), }, expectCreate: []partSpec{ - {name: "calls_p_2024_11_28"}, - {name: "calls_p_2024_11_29"}, - {name: "calls_p_2024_11_30"}, + {name: "calls_p_2024_12_29", timeRange: dailyTR("2024-12-29")}, + {name: "calls_p_2025_01_01", timeRange: dailyTR("2025-01-01")}, + {name: "calls_p_2025_01_02", timeRange: dailyTR("2025-01-02")}, }, expectDrop: []string{ - "public.calls_p_2024_11_24", - "public.calls_p_2024_11_25", + "public.calls_p_2024_12_26", + "public.calls_p_2024_12_27", }, expectSweep: []timeRange{ - {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-12-26"), end: dateInUTC("2024-12-27")}, + {start: dateInUTC("2024-12-27"), end: dateInUTC("2024-12-28")}, }, expectCleanup: []timeRange{ - {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-12-26"), end: dateInUTC("2024-12-27")}, + {start: dateInUTC("2024-12-27"), end: dateInUTC("2024-12-28")}, }, expectDetach: []string{ - "public.calls_p_2024_11_24", - "public.calls_p_2024_11_25", + "public.calls_p_2024_12_26", + "public.calls_p_2024_12_27", }, }, { @@ -210,10 +216,10 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_q3", "2024-07-01", "2024-10-01"), - etb("calls_p_2024_q4", "2024-10-01", "2025-01-01"), - etb("calls_p_2025_q1", "2025-01-01", "2024-04-01"), - etb("calls_p_2025_q2", "2025-04-01", "2024-07-01"), + partResult("calls_p_2024_q3", "2024-07-01", "2024-10-01"), + partResult("calls_p_2024_q4", "2024-10-01", "2025-01-01"), + partResult("calls_p_2025_q1", "2025-01-01", "2024-04-01"), + partResult("calls_p_2025_q2", "2025-04-01", "2024-07-01"), }, expectDrop: []string{ "public.calls_p_2024_11_24", @@ -244,10 +250,10 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), - etb("calls_p_2024_09", "2024-09-01", "2024-10-01"), - etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), - etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), + partResult("calls_p_2024_10", "2024-10-01", "2024-11-01"), + partResult("calls_p_2024_09", "2024-09-01", "2024-10-01"), + partResult("calls_p_2024_08", "2024-08-01", "2024-09-01"), + partResult("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, expectErr: partman.ErrDifferentInterval, }, @@ -263,10 +269,10 @@ func TestPartman(t *testing.T) { PreProvision: common.PtrTo(2), }, extant: []database.PartitionResult{ - etb("calls_p_2024_10", "2024-10-01", "2024-11-01"), - etbWithSchema("reid", "calls_p_2024_09", "2024-09-01", "2024-10-01"), - etb("calls_p_2024_08", "2024-08-01", "2024-09-01"), - etb("calls_p_2024_07", "2024-07-01", "2024-08-01"), + partResult("calls_p_2024_10", "2024-10-01", "2024-11-01"), + partResultWithSchema("reid", "calls_p_2024_09", "2024-09-01", "2024-10-01"), + partResult("calls_p_2024_08", "2024-08-01", "2024-09-01"), + partResult("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, expectErr: partman.ErrWrongSchema, }, -- 2.48.1 From 4d4ce6e9a8aa50671083d84f8b24b502b763a920 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 17:44:08 -0500 Subject: [PATCH 10/15] quarterly pass --- pkg/database/partman/partman_test.go | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index 28bf8e8..81fc50c 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -108,9 +108,9 @@ func TestPartman(t *testing.T) { partResult("calls_p_2024_07", "2024-07-01", "2024-08-01"), }, expectCreate: []partSpec{ - {name:"calls_p_2024_11", timeRange: timeRange{start: dateInUTC("2024-11-01"), end: dateInUTC("2024-12-01")}}, - {name:"calls_p_2024_12", timeRange: timeRange{start: dateInUTC("2024-12-01"), end: dateInUTC("2025-01-01")}}, - {name:"calls_p_2025_01", timeRange: timeRange{start: dateInUTC("2025-01-01"), end: dateInUTC("2025-02-01")}}, + {name: "calls_p_2024_11", timeRange: timeRange{start: dateInUTC("2024-11-01"), end: dateInUTC("2024-12-01")}}, + {name: "calls_p_2024_12", timeRange: timeRange{start: dateInUTC("2024-12-01"), end: dateInUTC("2025-01-01")}}, + {name: "calls_p_2025_01", timeRange: timeRange{start: dateInUTC("2025-01-01"), end: dateInUTC("2025-02-01")}}, }, expectDrop: []string{ "public.calls_p_2024_07", @@ -206,7 +206,7 @@ func TestPartman(t *testing.T) { }, { name: "quarterly base", - now: timeInUTC("2025-07-28 11:37:04"), + now: timeInUTC("2025-07-28 11:37:04"), // q3 cfg: config.Partition{ Enabled: true, Schema: "public", @@ -222,20 +222,25 @@ func TestPartman(t *testing.T) { partResult("calls_p_2025_q2", "2025-04-01", "2024-07-01"), }, expectDrop: []string{ - "public.calls_p_2024_11_24", - "public.calls_p_2024_11_25", + "public.calls_p_2024_q3", + "public.calls_p_2024_q4", }, expectSweep: []timeRange{ - {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-07-01"), end: dateInUTC("2024-10-01")}, + {start: dateInUTC("2024-10-01"), end: dateInUTC("2025-01-01")}, }, expectCleanup: []timeRange{ - {start: dateInUTC("2024-11-24"), end: dateInUTC("2024-11-25")}, - {start: dateInUTC("2024-11-25"), end: dateInUTC("2024-11-26")}, + {start: dateInUTC("2024-07-01"), end: dateInUTC("2024-10-01")}, + {start: dateInUTC("2024-10-01"), end: dateInUTC("2025-01-01")}, + }, + expectCreate: []partSpec{ + {name: "calls_p_2025_q3", timeRange: timeRange{dateInUTC("2025-07-01"), dateInUTC("2025-10-01")}}, + {name: "calls_p_2025_q4", timeRange: timeRange{dateInUTC("2025-10-01"), dateInUTC("2026-01-01")}}, + {name: "calls_p_2026_q1", timeRange: timeRange{dateInUTC("2026-01-01"), dateInUTC("2026-04-01")}}, }, expectDetach: []string{ - "public.calls_p_2024_11_24", - "public.calls_p_2024_11_25", + "public.calls_p_2024_q3", + "public.calls_p_2024_q4", }, }, { -- 2.48.1 From 70f9eeec571b8b741c10f5e4d1956c0a54c98a62 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 18:01:52 -0500 Subject: [PATCH 11/15] yearly passes --- pkg/database/partman/partman.go | 1 + pkg/database/partman/partman_test.go | 33 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index 0c84b60..ce5646a 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -421,6 +421,7 @@ func (pm *partman) verifyPartName(pr database.PartitionResult) (p partition, err } return p, nil case 1: // yearly + p.Interval = Yearly year, err := strconv.Atoi(dateAr[0]) if err != nil { return p, PartitionError(pn, err) diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index 81fc50c..c42b63f 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -243,6 +243,39 @@ func TestPartman(t *testing.T) { "public.calls_p_2024_q4", }, }, + { + name: "yearly base", + now: timeInUTC("2023-04-28 11:37:04"), // q3 + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "yearly", + Retain: 3, + Drop: false, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + partResult("calls_p_2019", "2019-01-01", "2020-01-01"), + partResult("calls_p_2020", "2020-01-01", "2021-01-01"), + partResult("calls_p_2021", "2021-01-01", "2022-01-01"), + partResult("calls_p_2022", "2022-01-01", "2023-01-01"), + partResult("calls_p_2023", "2023-01-01", "2024-01-01"), + }, + expectDrop: []string{}, + expectSweep: []timeRange{ + {start: dateInUTC("2019-01-01"), end: dateInUTC("2020-01-01")}, + }, + expectCleanup: []timeRange{ + {start: dateInUTC("2019-01-01"), end: dateInUTC("2020-01-01")}, + }, + expectCreate: []partSpec{ + {name: "calls_p_2024", timeRange: timeRange{dateInUTC("2024-01-01"), dateInUTC("2025-01-01")}}, + {name: "calls_p_2025", timeRange: timeRange{dateInUTC("2025-01-01"), dateInUTC("2026-01-01")}}, + }, + expectDetach: []string{ + "public.calls_p_2019", + }, + }, { name: "changed monthly to daily", now: timeInUTC("2024-11-28 11:37:04"), -- 2.48.1 From aa3f54c4526c753d9e20758cbcda75e0c5983224 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sat, 30 Nov 2024 18:22:31 -0500 Subject: [PATCH 12/15] works except local timestamps --- pkg/config/config.go | 1 + pkg/database/calls.sql.go | 22 +++++++++----- pkg/database/mocks/Store.go | 52 +++++++++++++++++++++++---------- pkg/database/partman/partman.go | 38 ++++++++++++++---------- pkg/database/querier.go | 4 +-- pkg/server/server.go | 15 ++++++++++ sql/postgres/queries/calls.sql | 4 +-- 7 files changed, 92 insertions(+), 44 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 5a8d110..bb2bf23 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,6 +51,7 @@ type Partition struct { Retain int `yaml:"retain"` PreProvision *int `yaml:"preProvision"` Drop bool `yaml:"detach"` + LocalTime bool `yaml:"local"` } type Logger struct { diff --git a/pkg/database/calls.sql.go b/pkg/database/calls.sql.go index 308f7f1..daa7312 100644 --- a/pkg/database/calls.sql.go +++ b/pkg/database/calls.sql.go @@ -135,7 +135,7 @@ func (q *Queries) AddCall(ctx context.Context, arg AddCallParams) error { return err } -const cleanupSweptCalls = `-- name: CleanupSweptCalls :exec +const cleanupSweptCalls = `-- name: CleanupSweptCalls :execrows WITH to_sweep AS ( SELECT id FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id @@ -147,9 +147,12 @@ WITH to_sweep AS ( WHERE call_id IN (SELECT id FROM to_sweep) ` -func (q *Queries) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { - _, err := q.db.Exec(ctx, cleanupSweptCalls, rangeStart, rangeEnd) - return err +func (q *Queries) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) { + result, err := q.db.Exec(ctx, cleanupSweptCalls, rangeStart, rangeEnd) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil } const getDatabaseSize = `-- name: GetDatabaseSize :one @@ -172,7 +175,7 @@ func (q *Queries) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcrip return err } -const sweepCalls = `-- name: SweepCalls :exec +const sweepCalls = `-- name: SweepCalls :execrows WITH to_sweep AS ( SELECT id, submitter, system, talkgroup, calls.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, ic.call_date, notes FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id @@ -180,7 +183,10 @@ WITH to_sweep AS ( ) INSERT INTO swept_calls SELECT id, submitter, system, talkgroup, to_sweep.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, to_sweep.call_date, notes FROM to_sweep ` -func (q *Queries) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { - _, err := q.db.Exec(ctx, sweepCalls, rangeStart, rangeEnd) - return err +func (q *Queries) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) { + result, err := q.db.Exec(ctx, sweepCalls, rangeStart, rangeEnd) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil } diff --git a/pkg/database/mocks/Store.go b/pkg/database/mocks/Store.go index e4778bf..4efa7ea 100644 --- a/pkg/database/mocks/Store.go +++ b/pkg/database/mocks/Store.go @@ -230,21 +230,31 @@ func (_c *Store_BulkSetTalkgroupTags_Call) RunAndReturn(run func(context.Context } // CleanupSweptCalls provides a mock function with given fields: ctx, rangeStart, rangeEnd -func (_m *Store) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { +func (_m *Store) CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) { ret := _m.Called(ctx, rangeStart, rangeEnd) if len(ret) == 0 { panic("no return value specified for CleanupSweptCalls") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) (int64, error)); ok { + return rf(ctx, rangeStart, rangeEnd) + } + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) int64); ok { r0 = rf(ctx, rangeStart, rangeEnd) } else { - r0 = ret.Error(0) + r0 = ret.Get(0).(int64) } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + r1 = rf(ctx, rangeStart, rangeEnd) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // Store_CleanupSweptCalls_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanupSweptCalls' @@ -267,12 +277,12 @@ func (_c *Store_CleanupSweptCalls_Call) Run(run func(ctx context.Context, rangeS return _c } -func (_c *Store_CleanupSweptCalls_Call) Return(_a0 error) *Store_CleanupSweptCalls_Call { - _c.Call.Return(_a0) +func (_c *Store_CleanupSweptCalls_Call) Return(_a0 int64, _a1 error) *Store_CleanupSweptCalls_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *Store_CleanupSweptCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error) *Store_CleanupSweptCalls_Call { +func (_c *Store_CleanupSweptCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) (int64, error)) *Store_CleanupSweptCalls_Call { _c.Call.Return(run) return _c } @@ -2260,21 +2270,31 @@ func (_c *Store_StoreTGVersion_Call) RunAndReturn(run func(context.Context, []da } // SweepCalls provides a mock function with given fields: ctx, rangeStart, rangeEnd -func (_m *Store) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error { +func (_m *Store) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) { ret := _m.Called(ctx, rangeStart, rangeEnd) if len(ret) == 0 { panic("no return value specified for SweepCalls") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) (int64, error)); ok { + return rf(ctx, rangeStart, rangeEnd) + } + if rf, ok := ret.Get(0).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) int64); ok { r0 = rf(ctx, rangeStart, rangeEnd) } else { - r0 = ret.Error(0) + r0 = ret.Get(0).(int64) } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error); ok { + r1 = rf(ctx, rangeStart, rangeEnd) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // Store_SweepCalls_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SweepCalls' @@ -2297,12 +2317,12 @@ func (_c *Store_SweepCalls_Call) Run(run func(ctx context.Context, rangeStart pg return _c } -func (_c *Store_SweepCalls_Call) Return(_a0 error) *Store_SweepCalls_Call { - _c.Call.Return(_a0) +func (_c *Store_SweepCalls_Call) Return(_a0 int64, _a1 error) *Store_SweepCalls_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *Store_SweepCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) error) *Store_SweepCalls_Call { +func (_c *Store_SweepCalls_Call) RunAndReturn(run func(context.Context, pgtype.Timestamptz, pgtype.Timestamptz) (int64, error)) *Store_SweepCalls_Call { _c.Call.Return(run) return _c } diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index ce5646a..ec8c04c 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -126,19 +126,17 @@ func New(db database.Store, cfg config.Partition) (*partman, error) { var _ PartitionManager = (*partman)(nil) func (pm *partman) Go(ctx context.Context) { - go func(ctx context.Context) { - tick := time.NewTicker(60 * time.Minute) + tick := time.NewTicker(60 * time.Minute) - select { - case now := <-tick.C: - err := pm.Check(ctx, now) - if err != nil { - log.Error().Err(err).Msg("partman check failed") - } - case <-ctx.Done(): - return + select { + case now := <-tick.C: + err := pm.Check(ctx, now) + if err != nil { + log.Error().Err(err).Msg("partman check failed") } - }(ctx) + case <-ctx.Done(): + return + } } func (pm *partman) newPartition(t time.Time) partition { @@ -246,23 +244,29 @@ func (pm *partman) prunePartition(ctx context.Context, tx database.Store, p part s, e := p.Range() start := pgtype.Timestamptz{Time: s, Valid: true} end := pgtype.Timestamptz{Time: e, Valid: true} - err := tx.SweepCalls(ctx, start, end) + fullPartName := pm.fullTableName(p.PartitionName()) + + swept, err := tx.SweepCalls(ctx, start, end) if err != nil { return err } + log.Info().Int64("rows", swept).Time("start", s).Time("end", e).Msg("swept calls") - err = tx.CleanupSweptCalls(ctx, start, end) + swept, err = tx.CleanupSweptCalls(ctx, start, end) if err != nil { return err } + log.Debug().Int64("rows", swept).Time("start", s).Time("end", e).Msg("cleaned up swept calls") - err = tx.DetachPartition(ctx, pm.fullTableName(p.PartitionName())) + log.Info().Str("partition", fullPartName).Msg("detaching partition") + err = tx.DetachPartition(ctx, fullPartName) if err != nil { return err } if pm.cfg.Drop { - return tx.DropPartition(ctx, pm.fullTableName(p.PartitionName())) + log.Info().Str("partition", fullPartName).Msg("dropping partition") + return tx.DropPartition(ctx, fullPartName) } return nil @@ -328,7 +332,9 @@ func (p partition) PartitionName() string { func (pm *partman) createPartition(ctx context.Context, tx database.Store, part partition) error { start, end := part.Range() - return tx.CreatePartition(ctx, callsTable, part.PartitionName(), start, end) + name := part.PartitionName() + log.Info().Str("partition", name).Time("start", start).Time("end", end).Msg("creating partition") + return tx.CreatePartition(ctx, callsTable, name, start, end) } /* diff --git a/pkg/database/querier.go b/pkg/database/querier.go index e9425ca..edeba22 100644 --- a/pkg/database/querier.go +++ b/pkg/database/querier.go @@ -15,7 +15,7 @@ type Querier interface { AddAlert(ctx context.Context, arg AddAlertParams) error AddCall(ctx context.Context, arg AddCallParams) error AddLearnedTalkgroup(ctx context.Context, arg AddLearnedTalkgroupParams) (Talkgroup, error) - CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error + CleanupSweptCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) CreateAPIKey(ctx context.Context, owner int, expires pgtype.Timestamp, disabled *bool) (ApiKey, error) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) DeleteAPIKey(ctx context.Context, apiKey string) error @@ -43,7 +43,7 @@ type Querier interface { SetCallTranscript(ctx context.Context, iD uuid.UUID, transcript *string) error SetTalkgroupTags(ctx context.Context, tags []string, systemID int32, tGID int32) error StoreTGVersion(ctx context.Context, arg []StoreTGVersionParams) *StoreTGVersionBatchResults - SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) error + SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) UpdatePassword(ctx context.Context, username string, password string) error UpdateTalkgroup(ctx context.Context, arg UpdateTalkgroupParams) (Talkgroup, error) UpsertTalkgroup(ctx context.Context, arg []UpsertTalkgroupParams) *UpsertTalkgroupBatchResults diff --git a/pkg/server/server.go b/pkg/server/server.go index 57a9048..0011a23 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,6 +10,7 @@ import ( "dynatron.me/x/stillbox/pkg/auth" "dynatron.me/x/stillbox/pkg/config" "dynatron.me/x/stillbox/pkg/database" + "dynatron.me/x/stillbox/pkg/database/partman" "dynatron.me/x/stillbox/pkg/nexus" "dynatron.me/x/stillbox/pkg/notify" "dynatron.me/x/stillbox/pkg/rest" @@ -40,6 +41,7 @@ type Server struct { hup chan os.Signal tgs tgstore.Store rest rest.API + partman partman.PartitionManager } func New(ctx context.Context, cfg *config.Configuration) (*Server, error) { @@ -62,6 +64,18 @@ func New(ctx context.Context, cfg *config.Configuration) (*Server, error) { return nil, err } + pm, err := partman.New(db, cfg.DB.Partition) + if err != nil { + return nil, err + } + + err = pm.Check(ctx, time.Now()) + if err != nil { + return nil, err + } + + go pm.Go(ctx) + tgCache := tgstore.NewCache() api := rest.New() @@ -77,6 +91,7 @@ func New(ctx context.Context, cfg *config.Configuration) (*Server, error) { tgs: tgCache, sinks: sinks.NewSinkManager(), rest: api, + partman: pm, } srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db, tgCache), true) diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index 3955614..4720797 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -57,14 +57,14 @@ VALUES -- name: GetDatabaseSize :one SELECT pg_size_pretty(pg_database_size(current_database())); --- name: SweepCalls :exec +-- name: SweepCalls :execrows WITH to_sweep AS ( SELECT * FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id WHERE calls.call_date >= @range_start AND calls.call_date < @range_end ) INSERT INTO swept_calls SELECT * FROM to_sweep; --- name: CleanupSweptCalls :exec +-- name: CleanupSweptCalls :execrows WITH to_sweep AS ( SELECT id FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id -- 2.48.1 From 16caa0a3a42749e3c08d461564add691e98aa04f Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sun, 1 Dec 2024 02:26:06 -0500 Subject: [PATCH 13/15] utc --- sql/postgres/migrations/002_partition.up.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/postgres/migrations/002_partition.up.sql b/sql/postgres/migrations/002_partition.up.sql index 0fc1d80..1c63d25 100644 --- a/sql/postgres/migrations/002_partition.up.sql +++ b/sql/postgres/migrations/002_partition.up.sql @@ -26,11 +26,12 @@ CREATE TABLE calls ( ) PARTITION BY RANGE (call_date); -- for our prod -create table calls_p_2024_07 partition of calls for values from ('2024-07-01') to ('2024-08-01'); -create table calls_p_2024_08 partition of calls for values from ('2024-08-01') to ('2024-09-01'); -create table calls_p_2024_09 partition of calls for values from ('2024-09-01') to ('2024-10-01'); -create table calls_p_2024_10 partition of calls for values from ('2024-10-01') to ('2024-11-01'); -create table calls_p_2024_11 partition of calls for values from ('2024-11-01') to ('2024-12-01'); +create table calls_p_2024_07 partition of calls for values from ('2024-07-01 00:00:00-00') to ('2024-08-01 00:00:00-00'); +create table calls_p_2024_08 partition of calls for values from ('2024-08-01 00:00:00-00') to ('2024-09-01 00:00:00-00'); +create table calls_p_2024_09 partition of calls for values from ('2024-09-01 00:00:00-00') to ('2024-10-01 00:00:00-00'); +create table calls_p_2024_10 partition of calls for values from ('2024-10-01 00:00:00-00') to ('2024-11-01 00:00:00-00'); +create table calls_p_2024_11 partition of calls for values from ('2024-11-01 00:00:00-00') to ('2024-12-01 00:00:00-00'); +create table calls_p_2024_12 partition of calls for values from ('2024-12-01 00:00:00-00') to ('2025-01-01 00:00:00-00'); insert into calls (id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration) select id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration from calls_unpart; -- 2.48.1 From 68f542d23a66102cf959a8d13fb8df4cb57bbdce Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sun, 1 Dec 2024 02:28:27 -0500 Subject: [PATCH 14/15] almost --- config.sample.yaml | 13 ++++ pkg/config/config.go | 1 - pkg/database/calls.sql.go | 5 +- pkg/database/database.go | 2 + pkg/database/mocks/Store.go | 21 ++--- pkg/database/partitions.go | 23 +++--- pkg/database/partman/partman.go | 2 +- pkg/server/server.go | 29 +++---- sql/postgres/migrations/001_initial.up.sql | 43 +++++++++-- sql/postgres/migrations/002_partition.up.sql | 81 -------------------- sql/postgres/queries/calls.sql | 4 +- 11 files changed, 99 insertions(+), 125 deletions(-) delete mode 100644 sql/postgres/migrations/002_partition.up.sql diff --git a/config.sample.yaml b/config.sample.yaml index 4e66425..500cb1c 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -1,5 +1,18 @@ db: connect: 'postgres://postgres:password@localhost:5432/example' + partition: + # whether to enable the built-in partition manager + enabled: true + # the postgres schema containing our tables + schema: public + # daily|weekly|monthly|quarterly|yearly + interval: monthly + # number of partitions to retain, -1 to keep all + retain: 3 + # whether to drop or simply detach + drop: true + # number of partitions to prepare ahead + preProvision: 3 cors: allowedOrigins: - 'http://localhost:*' diff --git a/pkg/config/config.go b/pkg/config/config.go index bb2bf23..5a8d110 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,7 +51,6 @@ type Partition struct { Retain int `yaml:"retain"` PreProvision *int `yaml:"preProvision"` Drop bool `yaml:"detach"` - LocalTime bool `yaml:"local"` } type Logger struct { diff --git a/pkg/database/calls.sql.go b/pkg/database/calls.sql.go index daa7312..af8be5c 100644 --- a/pkg/database/calls.sql.go +++ b/pkg/database/calls.sql.go @@ -177,10 +177,11 @@ func (q *Queries) SetCallTranscript(ctx context.Context, iD uuid.UUID, transcrip const sweepCalls = `-- name: SweepCalls :execrows WITH to_sweep AS ( - SELECT id, submitter, system, talkgroup, calls.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, ic.call_date, notes FROM calls + SELECT id, submitter, system, talkgroup, calls.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript + FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id WHERE calls.call_date >= $1 AND calls.call_date < $2 -) INSERT INTO swept_calls SELECT id, submitter, system, talkgroup, to_sweep.call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, incident_id, call_id, calls_tbl_id, swept_call_id, to_sweep.call_date, notes FROM to_sweep +) INSERT INTO swept_calls SELECT id, submitter, system, talkgroup, call_date, audio_name, audio_blob, duration, audio_type, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript FROM to_sweep ` func (q *Queries) SweepCalls(ctx context.Context, rangeStart pgtype.Timestamptz, rangeEnd pgtype.Timestamptz) (int64, error) { diff --git a/pkg/database/database.go b/pkg/database/database.go index fa3a25b..b833adb 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -94,6 +94,8 @@ func NewClient(ctx context.Context, conf config.DB) (*Postgres, error) { return nil, err } + log.Debug().Err(err).Msg("migrations done") + m.Close() pgConf, err := pgxpool.ParseConfig(conf.Connect) diff --git a/pkg/database/mocks/Store.go b/pkg/database/mocks/Store.go index 4efa7ea..564dcd1 100644 --- a/pkg/database/mocks/Store.go +++ b/pkg/database/mocks/Store.go @@ -641,17 +641,17 @@ func (_c *Store_DeleteUser_Call) RunAndReturn(run func(context.Context, string) return _c } -// DetachPartition provides a mock function with given fields: ctx, partitionName -func (_m *Store) DetachPartition(ctx context.Context, partitionName string) error { - ret := _m.Called(ctx, partitionName) +// DetachPartition provides a mock function with given fields: ctx, parentTable, partitionName +func (_m *Store) DetachPartition(ctx context.Context, parentTable string, partitionName string) error { + ret := _m.Called(ctx, parentTable, partitionName) if len(ret) == 0 { panic("no return value specified for DetachPartition") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(ctx, partitionName) + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, parentTable, partitionName) } else { r0 = ret.Error(0) } @@ -666,14 +666,15 @@ type Store_DetachPartition_Call struct { // DetachPartition is a helper method to define mock.On call // - ctx context.Context +// - parentTable string // - partitionName string -func (_e *Store_Expecter) DetachPartition(ctx interface{}, partitionName interface{}) *Store_DetachPartition_Call { - return &Store_DetachPartition_Call{Call: _e.mock.On("DetachPartition", ctx, partitionName)} +func (_e *Store_Expecter) DetachPartition(ctx interface{}, parentTable interface{}, partitionName interface{}) *Store_DetachPartition_Call { + return &Store_DetachPartition_Call{Call: _e.mock.On("DetachPartition", ctx, parentTable, partitionName)} } -func (_c *Store_DetachPartition_Call) Run(run func(ctx context.Context, partitionName string)) *Store_DetachPartition_Call { +func (_c *Store_DetachPartition_Call) Run(run func(ctx context.Context, parentTable string, partitionName string)) *Store_DetachPartition_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string)) + run(args[0].(context.Context), args[1].(string), args[2].(string)) }) return _c } @@ -683,7 +684,7 @@ func (_c *Store_DetachPartition_Call) Return(_a0 error) *Store_DetachPartition_C return _c } -func (_c *Store_DetachPartition_Call) RunAndReturn(run func(context.Context, string) error) *Store_DetachPartition_Call { +func (_c *Store_DetachPartition_Call) RunAndReturn(run func(context.Context, string, string) error) *Store_DetachPartition_Call { _c.Call.Return(run) return _c } diff --git a/pkg/database/partitions.go b/pkg/database/partitions.go index f93c749..636c549 100644 --- a/pkg/database/partitions.go +++ b/pkg/database/partitions.go @@ -25,7 +25,7 @@ type PartitionResult struct { type partitionsQuerier interface { GetTablePartitions(ctx context.Context, schemaName, tableName string) ([]PartitionResult, error) CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error - DetachPartition(ctx context.Context, partitionName string) error + DetachPartition(ctx context.Context, parentTable, partitionName string) error DropPartition(ctx context.Context, partitionName string) error } @@ -65,19 +65,20 @@ func (q *Queries) GetTablePartitions(ctx context.Context, schemaName, tableName } func (q *Queries) CreatePartition(ctx context.Context, parentTable, partitionName string, start, end time.Time) error { - _, err := q.db.Exec(ctx, `CREATE TABLE $1 PARTITION OF $2 FOR VALUES FROM ($2) TO ($3);`, partitionName, parentTable, start, end) + const boundFmt = "2006-01-02 00:00:00Z00" + _, err := q.db.Exec(ctx, fmt.Sprintf(`CREATE TABLE %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s');`, partitionName, parentTable, start.Format(boundFmt), end.Format(boundFmt))) return err } func (q *Queries) DropPartition(ctx context.Context, partitionName string) error { - _, err := q.db.Exec(ctx, `DROP TABLE $1;`, partitionName) + _, err := q.db.Exec(ctx, fmt.Sprintf(`DROP TABLE %s;`, partitionName)) return err } -func (q *Queries) DetachPartition(ctx context.Context, partitionName string) error { - _, err := q.db.Exec(ctx, `ALTER TABLE $1 DETACH PARTITION;`, partitionName) +func (q *Queries) DetachPartition(ctx context.Context, parentTable, partitionName string) error { + _, err := q.db.Exec(ctx, fmt.Sprintf(`ALTER TABLE %s DETACH PARTITION %s;`, parentTable, partitionName)) return err } @@ -105,12 +106,12 @@ func (partition PartitionResult) ParseBounds() (lowerBound time.Time, upperBound } func parseBoundAsDate(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { - lowerBound, err = time.Parse("2006-01-02", partition.LowerBound) + lowerBound, err = time.ParseInLocation("2006-01-02", partition.LowerBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as date: %w", err) } - upperBound, err = time.Parse("2006-01-02", partition.UpperBound) + upperBound, err = time.ParseInLocation("2006-01-02", partition.UpperBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as date: %w", err) } @@ -119,12 +120,12 @@ func parseBoundAsDate(partition PartitionResult) (lowerBound, upperBound time.Ti } func parseBoundAsDateTime(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { - lowerBound, err = time.Parse("2006-01-02 15:04:05", partition.LowerBound) + lowerBound, err = time.ParseInLocation("2006-01-02 15:04:05", partition.LowerBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime: %w", err) } - upperBound, err = time.Parse("2006-01-02 15:04:05", partition.UpperBound) + upperBound, err = time.ParseInLocation("2006-01-02 15:04:05", partition.UpperBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime: %w", err) } @@ -133,12 +134,12 @@ func parseBoundAsDateTime(partition PartitionResult) (lowerBound, upperBound tim } func parseBoundAsDateTimeWithTimezone(partition PartitionResult) (lowerBound, upperBound time.Time, err error) { - lowerBound, err = time.Parse("2006-01-02 15:04:05Z07", partition.LowerBound) + lowerBound, err = time.ParseInLocation("2006-01-02 15:04:05Z07", partition.LowerBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime with timezone: %w", err) } - upperBound, err = time.Parse("2006-01-02 15:04:05Z07", partition.UpperBound) + upperBound, err = time.ParseInLocation("2006-01-02 15:04:05Z07", partition.UpperBound, time.UTC) if err != nil { return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime with timezone: %w", err) } diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index ec8c04c..e158a13 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -259,7 +259,7 @@ func (pm *partman) prunePartition(ctx context.Context, tx database.Store, p part log.Debug().Int64("rows", swept).Time("start", s).Time("end", e).Msg("cleaned up swept calls") log.Info().Str("partition", fullPartName).Msg("detaching partition") - err = tx.DetachPartition(ctx, fullPartName) + err = tx.DetachPartition(ctx, callsTable, fullPartName) if err != nil { return err } diff --git a/pkg/server/server.go b/pkg/server/server.go index 0011a23..9166b23 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -64,18 +64,6 @@ func New(ctx context.Context, cfg *config.Configuration) (*Server, error) { return nil, err } - pm, err := partman.New(db, cfg.DB.Partition) - if err != nil { - return nil, err - } - - err = pm.Check(ctx, time.Now()) - if err != nil { - return nil, err - } - - go pm.Go(ctx) - tgCache := tgstore.NewCache() api := rest.New() @@ -91,7 +79,18 @@ func New(ctx context.Context, cfg *config.Configuration) (*Server, error) { tgs: tgCache, sinks: sinks.NewSinkManager(), rest: api, - partman: pm, + } + + if cfg.DB.Partition.Enabled { + srv.partman, err = partman.New(db, cfg.DB.Partition) + if err != nil { + return nil, err + } + + err = srv.partman.Check(ctx, time.Now()) + if err != nil { + return nil, err + } } srv.sinks.Register("database", sinks.NewDatabaseSink(srv.db, tgCache), true) @@ -143,6 +142,10 @@ func (s *Server) Go(ctx context.Context) error { go s.nex.Go(ctx) go s.alerter.Go(ctx) + if pm := s.partman; pm != nil { + go pm.Go(ctx) + } + var err error go func() { err = httpSrv.ListenAndServe() diff --git a/sql/postgres/migrations/001_initial.up.sql b/sql/postgres/migrations/001_initial.up.sql index 14695d5..4f493b0 100644 --- a/sql/postgres/migrations/001_initial.up.sql +++ b/sql/postgres/migrations/001_initial.up.sql @@ -78,7 +78,33 @@ CREATE TABLE IF NOT EXISTS alerts( metadata JSONB ); -CREATE TABLE IF NOT EXISTS calls( +CREATE TABLE calls ( + id UUID, + submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, + system INTEGER NOT NULL, + talkgroup INTEGER NOT NULL, + call_date TIMESTAMPTZ NOT NULL, + audio_name TEXT, + audio_blob BYTEA, + duration INTEGER, + audio_type TEXT, + audio_url TEXT, + frequency INTEGER NOT NULL, + frequencies INTEGER[], + patches INTEGER[], + tg_label TEXT, + tg_alpha_tag TEXT, + tg_group TEXT, + source INTEGER NOT NULL, + transcript TEXT, + PRIMARY KEY (id, call_date), + FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) +) PARTITION BY RANGE (call_date); + +CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript)); +CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date); + +CREATE TABLE swept_calls ( id UUID PRIMARY KEY, submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, system INTEGER NOT NULL, @@ -100,8 +126,9 @@ CREATE TABLE IF NOT EXISTS calls( FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) ); -CREATE INDEX IF NOT EXISTS calls_transcript_idx ON calls USING GIN (to_tsvector('english', transcript)); -CREATE INDEX IF NOT EXISTS calls_call_date_tg_idx ON calls(system, talkgroup, call_date); +CREATE INDEX IF NOT EXISTS swept_calls_transcript_idx ON swept_calls USING GIN (to_tsvector('english', transcript)); +CREATE INDEX IF NOT EXISTS swept_calls_call_date_tg_idx ON swept_calls(system, talkgroup, call_date); + CREATE TABLE IF NOT EXISTS settings( name TEXT PRIMARY KEY, @@ -125,8 +152,14 @@ CREATE INDEX IF NOT EXISTS incidents_name_description_idx ON incidents USING GIN ); CREATE TABLE IF NOT EXISTS incidents_calls( - incident_id UUID REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, - call_id UUID REFERENCES calls(id) ON UPDATE CASCADE, + incident_id UUID NOT NULL REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, + call_id UUID NOT NULL, + calls_tbl_id UUID NULL, + swept_call_id UUID NULL REFERENCES swept_calls(id), + call_date TIMESTAMPTZ NULL, notes JSONB, + FOREIGN KEY (calls_tbl_id, call_date) REFERENCES calls(id, call_date), PRIMARY KEY (incident_id, call_id) ); + + diff --git a/sql/postgres/migrations/002_partition.up.sql b/sql/postgres/migrations/002_partition.up.sql deleted file mode 100644 index 1c63d25..0000000 --- a/sql/postgres/migrations/002_partition.up.sql +++ /dev/null @@ -1,81 +0,0 @@ -BEGIN; - -ALTER TABLE calls RENAME TO calls_unpart; - -CREATE TABLE calls ( - id UUID, - submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, - system INTEGER NOT NULL, - talkgroup INTEGER NOT NULL, - call_date TIMESTAMPTZ NOT NULL, - audio_name TEXT, - audio_blob BYTEA, - duration INTEGER, - audio_type TEXT, - audio_url TEXT, - frequency INTEGER NOT NULL, - frequencies INTEGER[], - patches INTEGER[], - tg_label TEXT, - tg_alpha_tag TEXT, - tg_group TEXT, - source INTEGER NOT NULL, - transcript TEXT, - PRIMARY KEY (id, call_date), - FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) -) PARTITION BY RANGE (call_date); - --- for our prod -create table calls_p_2024_07 partition of calls for values from ('2024-07-01 00:00:00-00') to ('2024-08-01 00:00:00-00'); -create table calls_p_2024_08 partition of calls for values from ('2024-08-01 00:00:00-00') to ('2024-09-01 00:00:00-00'); -create table calls_p_2024_09 partition of calls for values from ('2024-09-01 00:00:00-00') to ('2024-10-01 00:00:00-00'); -create table calls_p_2024_10 partition of calls for values from ('2024-10-01 00:00:00-00') to ('2024-11-01 00:00:00-00'); -create table calls_p_2024_11 partition of calls for values from ('2024-11-01 00:00:00-00') to ('2024-12-01 00:00:00-00'); -create table calls_p_2024_12 partition of calls for values from ('2024-12-01 00:00:00-00') to ('2025-01-01 00:00:00-00'); - - -insert into calls (id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration) select id, submitter, system, talkgroup, call_date, audio_name, audio_blob, audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript, duration from calls_unpart; - -drop table calls_unpart cascade; - -CREATE TABLE swept_calls ( - id UUID PRIMARY KEY, - submitter INTEGER REFERENCES api_keys(id) ON DELETE SET NULL, - system INTEGER NOT NULL, - talkgroup INTEGER NOT NULL, - call_date TIMESTAMPTZ NOT NULL, - audio_name TEXT, - audio_blob BYTEA, - duration INTEGER, - audio_type TEXT, - audio_url TEXT, - frequency INTEGER NOT NULL, - frequencies INTEGER[], - patches INTEGER[], - tg_label TEXT, - tg_alpha_tag TEXT, - tg_group TEXT, - source INTEGER NOT NULL, - transcript TEXT, - FOREIGN KEY (system, talkgroup) REFERENCES talkgroups(system_id, tgid) -); - -DROP TABLE IF EXISTS incidents_calls; -- DATA LOSS - -CREATE TABLE IF NOT EXISTS incidents_calls( - incident_id UUID NOT NULL REFERENCES incidents(id) ON UPDATE CASCADE ON DELETE CASCADE, - call_id UUID NOT NULL, - calls_tbl_id UUID NULL, - swept_call_id UUID NULL REFERENCES swept_calls(id), - call_date TIMESTAMPTZ NULL, - notes JSONB, - FOREIGN KEY (calls_tbl_id, call_date) REFERENCES calls(id, call_date), - PRIMARY KEY (incident_id, call_id) -); - - --- ALTER TABLE incidents_calls ADD COLUMN call_date TIMESTAMPTZ NOT NULL; --- ALTER TABLE incidents_calls DROP CONSTRAINT incidents_calls_call_id_fkey; --- ALTER TABLE incidents_calls ADD CONSTRAINT incidents_calls_call_id_call_date_fkey FOREIGN KEY (call_id, call_date) REFERENCES calls(id, call_date) ON UPDATE CASCADE; - -COMMIT; diff --git a/sql/postgres/queries/calls.sql b/sql/postgres/queries/calls.sql index 4720797..1c581b1 100644 --- a/sql/postgres/queries/calls.sql +++ b/sql/postgres/queries/calls.sql @@ -59,7 +59,9 @@ SELECT pg_size_pretty(pg_database_size(current_database())); -- name: SweepCalls :execrows WITH to_sweep AS ( - SELECT * FROM calls + SELECT id, submitter, system, talkgroup, calls.call_date, audio_name, audio_blob, duration, audio_type, + audio_url, frequency, frequencies, patches, tg_label, tg_alpha_tag, tg_group, source, transcript + FROM calls JOIN incidents_calls ic ON ic.call_id = calls.id WHERE calls.call_date >= @range_start AND calls.call_date < @range_end ) INSERT INTO swept_calls SELECT * FROM to_sweep; -- 2.48.1 From 69ba62819a1077d2096a265d18398a1f123008b6 Mon Sep 17 00:00:00 2001 From: Daniel Ponte Date: Sun, 1 Dec 2024 02:59:47 -0500 Subject: [PATCH 15/15] -1 retains all --- pkg/database/partman/partman.go | 19 +++++---- pkg/database/partman/partman_test.go | 58 ++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/pkg/database/partman/partman.go b/pkg/database/partman/partman.go index e158a13..bb9608a 100644 --- a/pkg/database/partman/partman.go +++ b/pkg/database/partman/partman.go @@ -168,7 +168,7 @@ func (pm *partman) futurePartitions(cur partition) []partition { preProv = *pm.cfg.PreProvision } - partitions := make([]partition, 0, pm.cfg.Retain) + partitions := make([]partition, 0, preProv) for i := 1; i <= preProv; i++ { next := cur.Next(i) partitions = append(partitions, next) @@ -180,11 +180,14 @@ func (pm *partman) futurePartitions(cur partition) []partition { func (pm *partman) expectedPartitions(now time.Time) []partition { curPart := pm.newPartition(now) - retain := pm.retentionPartitions(curPart) + shouldExist := []partition{curPart} + if pm.cfg.Retain > -1 { + retain := pm.retentionPartitions(curPart) + shouldExist = append(shouldExist, retain...) + } future := pm.futurePartitions(curPart) - shouldExist := append(retain, curPart) shouldExist = append(shouldExist, future...) return shouldExist @@ -290,10 +293,12 @@ func (pm *partman) Check(ctx context.Context, now time.Time) error { unexpected, missing := pm.comparePartitions(existing, expected) - for _, p := range unexpected { - err := pm.prunePartition(ctx, db, p) - if err != nil { - return err + if pm.cfg.Retain > -1 { + for _, p := range unexpected { + err := pm.prunePartition(ctx, db, p) + if err != nil { + return err + } } } diff --git a/pkg/database/partman/partman_test.go b/pkg/database/partman/partman_test.go index c42b63f..90615d4 100644 --- a/pkg/database/partman/partman_test.go +++ b/pkg/database/partman/partman_test.go @@ -129,6 +129,34 @@ func TestPartman(t *testing.T) { "public.calls_p_2024_08", }, }, + { + name: "monthly retain all", + now: timeInUTC("2024-11-28 11:37:04"), + cfg: config.Partition{ + Enabled: true, + Schema: "public", + Interval: "monthly", + Retain: -1, + Drop: true, + PreProvision: common.PtrTo(2), + }, + extant: []database.PartitionResult{ + partResult("calls_p_2024_10", "2024-10-01", "2024-11-01"), + partResult("calls_p_2024_09", "2024-09-01", "2024-10-01"), + partResult("calls_p_2024_08", "2024-08-01", "2024-09-01"), + partResult("calls_p_2024_07", "2024-07-01", "2024-08-01"), + }, + expectCreate: []partSpec{ + {name: "calls_p_2024_11", timeRange: timeRange{start: dateInUTC("2024-11-01"), end: dateInUTC("2024-12-01")}}, + {name: "calls_p_2024_12", timeRange: timeRange{start: dateInUTC("2024-12-01"), end: dateInUTC("2025-01-01")}}, + {name: "calls_p_2025_01", timeRange: timeRange{start: dateInUTC("2025-01-01"), end: dateInUTC("2025-02-01")}}, + }, + expectDrop: []string{}, + expectSweep: []timeRange{}, + expectCleanup: []timeRange{}, + expectDetach: []string{}, + }, + { name: "weekly base", now: timeInUTC("2024-11-28 11:37:04"), // week 48 @@ -326,7 +354,7 @@ func TestPartman(t *testing.T) { detachedPartitions := make([]string, 0, len(tc.expectDetach)) sweepMap := make(map[timeRange]struct{}) - if tc.expectErr == nil { + if len(tc.expectCreate) > 0 { db.EXPECT(). CreatePartition( mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string"), mock.AnythingOfType("time.Time"), @@ -336,7 +364,9 @@ func TestPartman(t *testing.T) { ps := partSpec{name: partitionName, timeRange: timeRange{start: start, end: end}} createdPartitions = append(createdPartitions, ps) }).Return(nil) + } + if len(tc.expectSweep) > 0 { db.EXPECT(). SweepCalls( mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), @@ -345,8 +375,10 @@ func TestPartman(t *testing.T) { tr := timeRange{start: start.Time, end: end.Time} sweepMap[tr] = struct{}{} sweptRanges = append(sweptRanges, tr) - }).Return(nil) + }).Return(30, nil) + } + if len(tc.expectCleanup) > 0 { db.EXPECT(). CleanupSweptCalls( mctx, mock.AnythingOfType("pgtype.Timestamptz"), mock.AnythingOfType("pgtype.Timestamptz"), @@ -355,20 +387,22 @@ func TestPartman(t *testing.T) { require.Contains(t, sweepMap, tr) cleanupRanges = append(cleanupRanges, tr) - }).Return(nil) + }).Return(30, nil) + } - if tc.cfg.Drop { - db.EXPECT(). - DropPartition(mctx, mock.AnythingOfType("string")). - Run(func(ctx context.Context, partName string) { - droppedPartitions = append(droppedPartitions, partName) - }).Return(nil) - } + if tc.cfg.Drop && len(tc.expectDrop) > 0 { + db.EXPECT(). + DropPartition(mctx, mock.AnythingOfType("string")). + Run(func(ctx context.Context, partName string) { + droppedPartitions = append(droppedPartitions, partName) + }).Return(nil) + } + if len(tc.expectDetach) > 0 { db.EXPECT(). DetachPartition( - mctx, mock.AnythingOfType("string")). - Run(func(ctx context.Context, partName string) { + mctx, mock.AnythingOfType("string"), mock.AnythingOfType("string")). + Run(func(ctx context.Context, parentTable, partName string) { detachedPartitions = append(detachedPartitions, partName) }).Return(nil) } -- 2.48.1