From 24d24069497187b636049e1016254e6995211806 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 13:36:57 -0400 Subject: [PATCH 01/15] name db containers --- tools/test/internal/config/config.go | 16 +++++++ tools/test/internal/config/config_test.go | 55 +++++++++++++++++++++++ tools/test/internal/db/db.go | 1 + 3 files changed, 72 insertions(+) create mode 100644 tools/test/internal/config/config_test.go diff --git a/tools/test/internal/config/config.go b/tools/test/internal/config/config.go index a5a632d1788..90f2e33435c 100644 --- a/tools/test/internal/config/config.go +++ b/tools/test/internal/config/config.go @@ -2,6 +2,8 @@ // tools/test database helpers. package config +import "fmt" + // DefaultPostgresVersion is the Postgres major version used for ephemeral and // persistent test databases when none is specified. const DefaultPostgresVersion = "16" @@ -21,4 +23,18 @@ type App struct { // ParallelIterations records the requested diagnose worker count (used only // to reject external databases with parallel runs). ParallelIterations int + // DiagnoseMode is true when the app is running in diagnose mode. + DiagnoseMode bool + // WorkerIndex is the index of the diagnose worker. + WorkerIndex int + // PackageSlug is the slug of the package being tested. + PackageSlug string +} + +// PostgresContainerName returns the name of the Postgres container for the app. +func (a *App) PostgresContainerName() string { + if a.DiagnoseMode { + return fmt.Sprintf("iteration_%d_%s", a.WorkerIndex, a.PackageSlug) + } + return fmt.Sprintf("test_%s", a.PackageSlug) } diff --git a/tools/test/internal/config/config_test.go b/tools/test/internal/config/config_test.go new file mode 100644 index 00000000000..ec98b854a2d --- /dev/null +++ b/tools/test/internal/config/config_test.go @@ -0,0 +1,55 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPostgresContainerName(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + conf *App + want string + }{ + { + name: "run mode", + conf: &App{PackageSlug: "core_services"}, + want: "test_core_services", + }, + { + name: "diagnose worker", + conf: &App{DiagnoseMode: true, WorkerIndex: 1, PackageSlug: "core_services"}, + want: "iteration_1_core_services", + }, + { + name: "diagnose parallel worker", + conf: &App{DiagnoseMode: true, WorkerIndex: 3, PackageSlug: "core_services"}, + want: "iteration_3_core_services", + }, + { + name: "missing slug defaults", + conf: &App{}, + want: "test_pkgs", + }, + { + name: "diagnose defaults worker index", + conf: &App{DiagnoseMode: true, PackageSlug: "core_services"}, + want: "iteration_1_core_services", + }, + { + name: "nil config", + conf: nil, + want: "test_pkgs", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.conf.PostgresContainerName()) + }) + } +} diff --git a/tools/test/internal/db/db.go b/tools/test/internal/db/db.go index 26bb9d92c41..28715e74002 100644 --- a/tools/test/internal/db/db.go +++ b/tools/test/internal/db/db.go @@ -121,6 +121,7 @@ func ensure(ctx context.Context, conf *config.App, out *output.Printer, setGloba postgres.WithDatabase(postgresDBName), postgres.WithUsername(postgresUser), postgres.WithPassword(postgresPassword), + testcontainers.WithName(conf.PostgresContainerName()), testcontainers.WithCmdArgs( "-c", "max_connections=1000", "-c", "shared_buffers=128MB", From b4b2f9cdee459c4c364f0e392299b638e8b43412 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 15:07:12 -0400 Subject: [PATCH 02/15] convert to pgtestdb --- core/store/migrate/migrate.go | 12 ++-- core/store/store.go | 26 ++------- core/utils/testutils/heavyweight/orm.go | 57 ++++++++++++------- debug_url.go | 22 +++++++ go.mod | 2 + go.sum | 4 ++ internal/testdb/testdb.go | 36 +++++++----- tools/test/internal/config/config.go | 26 ++++++--- tools/test/internal/db/db.go | 6 +- tools/test/internal/dbdetect/dbdetect.go | 57 +++++++++++++++++++ tools/test/internal/dbdetect/dbdetect_test.go | 51 +++++++++++++++++ tools/test/main.go | 2 + 12 files changed, 232 insertions(+), 69 deletions(-) create mode 100644 debug_url.go diff --git a/core/store/migrate/migrate.go b/core/store/migrate/migrate.go index 0da4854f274..80c8e319859 100644 --- a/core/store/migrate/migrate.go +++ b/core/store/migrate/migrate.go @@ -23,9 +23,9 @@ import ( ) //go:embed migrations/*.sql migrations/*.go -var embedMigrations embed.FS +var EmbedMigrations embed.FS -const MIGRATIONS_DIR string = "migrations" +const MigrationsDir string = "migrations" func NewProvider(ctx context.Context, db *sql.DB) (*goose.Provider, error) { // Set this to override how Goose resolves the schema for the `goose_migrations` table. @@ -56,7 +56,7 @@ func NewProvider(ctx context.Context, db *sql.DB) (*goose.Provider, error) { logMigrations := os.Getenv("CL_LOG_SQL_MIGRATIONS") verbose, _ := strconv.ParseBool(logMigrations) - fys, err := fs.Sub(embedMigrations, MIGRATIONS_DIR) + fys, err := fs.Sub(EmbedMigrations, MigrationsDir) if err != nil { return nil, fmt.Errorf("failed to get sub filesystem for embedded migration dir: %w", err) } @@ -126,13 +126,13 @@ func ensureMigrated(ctx context.Context, db *sql.DB, p *goose.Provider, provider if name == "1611847145" { id = 1 } else { - idx := strings.Index(name, "_") - if idx < 0 { + before, _, ok := strings.Cut(name, "_") + if !ok { // old migration we don't care about continue } - id, err = strconv.ParseInt(name[:idx], 10, 64) + id, err = strconv.ParseInt(before, 10, 64) if err == nil && id <= 0 { return pkgerrors.New("migration IDs must be greater than zero") } diff --git a/core/store/store.go b/core/store/store.go index a1c68098dda..10df6cf1bb5 100644 --- a/core/store/store.go +++ b/core/store/store.go @@ -42,10 +42,7 @@ func PrepareTestDB(lggr logger.Logger, dbURL url.URL, userOnly bool) error { return err } defer db.Close() - templateDB := strings.Trim(dbURL.Path, "/") - if err = dropAndCreatePristineDB(db, templateDB); err != nil { - return err - } + // we no longer create chainlink_test_pristine since pgtestdb handles template caching fixturePath := "../store/fixtures/fixtures.sql" if userOnly { @@ -137,29 +134,13 @@ func dropAndCreateDB(parsed url.URL, _ bool) (err error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // PostgreSQL does not support bound parameters for database names; pq.QuoteIdentifier is the supported escape. - _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)") //nolint:gosec // G701 false positive: identifier from pq.QuoteIdentifier only - if err != nil { - return fmt.Errorf("unable to drop postgres database: %w", err) - } - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(dbname)) //nolint:gosec // G701 false positive: identifier from pq.QuoteIdentifier only - if err != nil { - return fmt.Errorf("unable to create postgres database: %w", err) - } - return nil -} - -func dropAndCreatePristineDB(db *sqlx.DB, template string) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(testdb.PristineDBName)+" WITH (FORCE)") + _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)") if err != nil { return fmt.Errorf("unable to drop postgres database: %w", err) } ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(testdb.PristineDBName)+" WITH TEMPLATE "+pq.QuoteIdentifier(template)) //nolint:gosec // G701 false positive: identifiers from pq.QuoteIdentifier only + _, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(dbname)) if err != nil { return fmt.Errorf("unable to create postgres database: %w", err) } @@ -229,6 +210,7 @@ func checkSchema(dbURL url.URL, prevSchema string, restrictKey string) error { } return nil } + func insertFixtures(dbURL url.URL, pathToFixtures string) (err error) { db, err := sql.Open(pgcommon.DriverPostgres, dbURL.String()) if err != nil { diff --git a/core/utils/testutils/heavyweight/orm.go b/core/utils/testutils/heavyweight/orm.go index 73467fe183d..dfaeee973c6 100644 --- a/core/utils/testutils/heavyweight/orm.go +++ b/core/utils/testutils/heavyweight/orm.go @@ -3,20 +3,21 @@ package heavyweight import ( + "net/url" "os" "strings" "testing" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - "github.com/jmoiron/sqlx" + "github.com/peterldowns/pgtestdb" + "github.com/stretchr/testify/require" commoncfg "github.com/smartcontractkit/chainlink-common/pkg/config" pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink/v2/core/store" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/pg" @@ -28,7 +29,8 @@ import ( // unit tests, so you can do things like use other Postgres connection types with it. func FullTestDBV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) { cfg, db := FullTestDBNoFixturesV2(t, overrideFn) - _, err := db.Exec(store.FixturesSQL()) + ctx := t.Context() + _, err := db.ExecContext(ctx, store.FixturesSQL()) require.NoError(t, err) return cfg, db } @@ -43,15 +45,42 @@ func FullTestDBEmptyV2(t testing.TB, overrideFn func(c *chainlink.Config, s *cha return prepareDB(t, false, overrideFn) } -func generateName() string { - return strings.ReplaceAll(uuid.NewString(), "-", "") -} - func prepareDB(t testing.TB, withTemplate bool, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) { tests.SkipShort(t, "FullTestDB") + // Read env.DatabaseURL directly to get the base connection + rawDBURL := string(env.DatabaseURL.Get()) + if rawDBURL == "" { + t.Fatalf("you must provide a CL_DATABASE_URL environment variable") + } + + dbURL, err := url.Parse(rawDBURL) + require.NoError(t, err) + + migrator := testdb.Migrator(withTemplate) + conf := pgtestdb.Config{ + DriverName: pgcommon.DriverPostgres, + User: dbURL.User.Username(), + Host: dbURL.Hostname(), + Port: dbURL.Port(), + Database: strings.TrimLeft(dbURL.Path, "/"), + Options: dbURL.RawQuery, + ForceTerminateConnections: true, + } + if pass, ok := dbURL.User.Password(); ok { + conf.Password = pass + } + newConf := pgtestdb.Custom(t, conf, migrator) + + migrationTestDBURL := *dbURL + migrationTestDBURL.Path = "/" + newConf.Database + dbStr := migrationTestDBURL.String() + gcfg := configtest.NewGeneralConfigSimulated(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.Database.DriverName = pgcommon.DriverPostgres + s.Database.URL = models.NewSecretURL((*commoncfg.URL)(&migrationTestDBURL)) + // Explicitly allow simple passwords since tests use `postgres` password + s.Database.AllowSimplePasswords = new(true) if overrideFn != nil { overrideFn(c, s) } @@ -60,19 +89,9 @@ func prepareDB(t testing.TB, withTemplate bool, overrideFn func(c *chainlink.Con require.NoError(t, os.MkdirAll(gcfg.RootDir(), 0700)) t.Cleanup(func() { os.RemoveAll(gcfg.RootDir()) }) - migrationTestDBURL := testdb.CreateOrReplace(t, gcfg.Database().URL(), generateName(), withTemplate) - db, err := pg.NewConnection(t.Context(), migrationTestDBURL.String(), pgcommon.DriverPostgres, gcfg.Database()) + db, err := pg.NewConnection(t.Context(), dbStr, pgcommon.DriverPostgres, gcfg.Database()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) // must close before dropping - // reset with new URL - gcfg = configtest.NewGeneralConfigSimulated(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.Database.DriverName = pgcommon.DriverPostgres - s.Database.URL = models.NewSecretURL((*commoncfg.URL)(&migrationTestDBURL)) - if overrideFn != nil { - overrideFn(c, s) - } - }) - return gcfg, db } diff --git a/debug_url.go b/debug_url.go new file mode 100644 index 00000000000..cbbb492a6db --- /dev/null +++ b/debug_url.go @@ -0,0 +1,22 @@ +package main + +import ( + "fmt" + "net/url" + "github.com/smartcontractkit/chainlink-common/pkg/config" +) + +func main() { + u, _ := url.Parse("postgres://postgres:postgres@localhost:5432/db") + cUrl := (*config.URL)(u) + sUrl := config.NewSecretURL(cUrl) + fmt.Printf("sUrl.String(): %s\n", sUrl.String()) + + // Now convert it back to url.URL + back := *sUrl.URL() + fmt.Printf("back.String(): %s\n", back.String()) + + // What about User.Password()? + pass, ok := back.User.Password() + fmt.Printf("Password: %s, ok: %v\n", pass, ok) +} diff --git a/go.mod b/go.mod index 613e54c77d7..e79cf9851b9 100644 --- a/go.mod +++ b/go.mod @@ -332,6 +332,8 @@ require ( github.com/onsi/ginkgo/v2 v2.27.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect + github.com/peterldowns/pgtestdb v0.1.1 // indirect + github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect diff --git a/go.sum b/go.sum index 62912287d64..3862ebfd427 100644 --- a/go.sum +++ b/go.sum @@ -1025,6 +1025,10 @@ github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7ol github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= +github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1 h1:f+e5A8elEb+5VJnrtlPI8GKq2LunCFJH+3by7ekJ3io= +github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1/go.mod h1:ELFSiezLH4Hp2W1XSCtq/bniC9ZkkdyFy06tLOeaAqw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index 64c04c01908..f298df74ee6 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -1,10 +1,11 @@ package testdb import ( - "net/url" - "testing" + "context" + "database/sql" - "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/sqltest" + "github.com/peterldowns/pgtestdb" + "github.com/smartcontractkit/chainlink/v2/core/store/migrate" ) const ( @@ -14,15 +15,24 @@ const ( TestDBNamePrefix = "chainlink_test_" ) -// CreateOrReplace creates a database named with a common prefix and the given suffix, and returns the URL. -// If the database already exists, it will be dropped and re-created. -// If withTemplate is true, the pristine DB will be used as a template. -func CreateOrReplace(t testing.TB, parsed url.URL, suffix string, withTemplate bool) url.URL { - // Match the naming schema that our dangling DB cleanup methods expect - dbname := TestDBNamePrefix + suffix - var template string - if withTemplate { - template = PristineDBName +type migrator struct { + withTemplate bool +} + +func (m *migrator) Hash() (string, error) { + if m.withTemplate { + return "withTemplate", nil } - return sqltest.CreateOrReplace(t, parsed, dbname, template) + return "empty", nil +} + +func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Config) error { + if !m.withTemplate { + return nil + } + return migrate.Migrate(ctx, db) +} + +func Migrator(withTemplate bool) pgtestdb.Migrator { + return &migrator{withTemplate: withTemplate} } diff --git a/tools/test/internal/config/config.go b/tools/test/internal/config/config.go index 90f2e33435c..cf4896bc5ab 100644 --- a/tools/test/internal/config/config.go +++ b/tools/test/internal/config/config.go @@ -23,18 +23,28 @@ type App struct { // ParallelIterations records the requested diagnose worker count (used only // to reject external databases with parallel runs). ParallelIterations int - // DiagnoseMode is true when the app is running in diagnose mode. + // DiagnoseMode is true when the harness is running testrig diagnose. DiagnoseMode bool - // WorkerIndex is the index of the diagnose worker. + // WorkerIndex is the 1-based diagnose worker slot when DiagnoseMode is set. WorkerIndex int - // PackageSlug is the slug of the package being tested. + // PackageSlug is a short, docker-safe token derived from the package patterns + // under test (e.g. core_services). PackageSlug string } -// PostgresContainerName returns the name of the Postgres container for the app. -func (a *App) PostgresContainerName() string { - if a.DiagnoseMode { - return fmt.Sprintf("iteration_%d_%s", a.WorkerIndex, a.PackageSlug) +// PostgresContainerName returns the docker container name for an ephemeral +// Postgres instance. Non-diagnose runs use test_; diagnose workers use +// iteration__. +func (c *App) PostgresContainerName() string { + if c == nil { + return "test_pkgs" } - return fmt.Sprintf("test_%s", a.PackageSlug) + slug := c.PackageSlug + if slug == "" { + slug = "pkgs" + } + if c.DiagnoseMode { + return fmt.Sprintf("iteration_%d_%s", max(c.WorkerIndex, 1), slug) + } + return "test_" + slug } diff --git a/tools/test/internal/db/db.go b/tools/test/internal/db/db.go index 28715e74002..fae9bc751c9 100644 --- a/tools/test/internal/db/db.go +++ b/tools/test/internal/db/db.go @@ -217,7 +217,11 @@ func EnsurePool(ctx context.Context, conf *config.App, out *output.Printer, size if size > 1 { workerOut = output.New(conf.AIOutput, io.Discard, io.Discard, output.SkipFD) } - h, err := ensure(poolCtx, conf, workerOut, size == 1) + workerConf := *conf + if conf.DiagnoseMode { + workerConf.WorkerIndex = i + 1 + } + h, err := ensure(poolCtx, &workerConf, workerOut, size == 1) if err != nil { cancel() errs <- err diff --git a/tools/test/internal/dbdetect/dbdetect.go b/tools/test/internal/dbdetect/dbdetect.go index 3bfffc1c9d3..87c09364f12 100644 --- a/tools/test/internal/dbdetect/dbdetect.go +++ b/tools/test/internal/dbdetect/dbdetect.go @@ -6,6 +6,7 @@ import ( "fmt" "os/exec" "regexp" + "slices" "strings" "time" ) @@ -46,6 +47,62 @@ var harnessRootValueFlags = map[string]bool{ "postgres-version": true, } +// IsDiagnoseCommand reports whether argv invokes the testrig diagnose subcommand. +func IsDiagnoseCommand(args []string) bool { + return slices.Contains(args, "diagnose") +} + +// PackageSlug returns a short docker-safe name for the package patterns in argv +// (e.g. ./core/services/... -> core_services). +func PackageSlug(args []string) string { + patterns := extractPackagePatterns(args) + switch len(patterns) { + case 0: + return "pkgs" + case 1: + return patternToSlug(patterns[0]) + default: + slugs := make([]string, len(patterns)) + for i, p := range patterns { + slugs[i] = patternToSlug(p) + } + return strings.Join(slugs, "__") + } +} + +func patternToSlug(pattern string) string { + t := strings.TrimPrefix(pattern, "./") + switch { + case t == "...": + return "pkgs" + case strings.HasSuffix(t, "/..."): + t = strings.TrimSuffix(t, "/...") + } + t = strings.ReplaceAll(t, "/", "_") + return sanitizeSlugToken(t) +} + +func sanitizeSlugToken(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', + r >= 'A' && r <= 'Z', + r >= '0' && r <= '9', + r == '_', r == '-', r == '.': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + out := b.String() + if out == "" { + return "pkgs" + } + return out +} + func extractPackagePatterns(args []string) []string { var patterns []string for i := 0; i < len(args); i++ { diff --git a/tools/test/internal/dbdetect/dbdetect_test.go b/tools/test/internal/dbdetect/dbdetect_test.go index 64fa3821705..e333851b5ab 100644 --- a/tools/test/internal/dbdetect/dbdetect_test.go +++ b/tools/test/internal/dbdetect/dbdetect_test.go @@ -28,6 +28,57 @@ func findRepoRoot(t *testing.T) string { return cwd } +func TestPackageSlug(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want string + }{ + { + name: "services tree", + args: []string{"run", "./core/services/..."}, + want: "core_services", + }, + { + name: "diagnose subcommand ignored for slug", + args: []string{"diagnose", "--iterations", "5", "./core/services/..."}, + want: "core_services", + }, + { + name: "specific package", + args: []string{"./core/services/gateway/connector"}, + want: "core_services_gateway_connector", + }, + { + name: "no patterns", + args: []string{"diagnose"}, + want: "pkgs", + }, + { + name: "multiple patterns", + args: []string{"./core/services/...", "./core/store/..."}, + want: "core_services__core_store", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, PackageSlug(tt.args)) + }) + } +} + +func TestIsDiagnoseCommand(t *testing.T) { + t.Parallel() + + assert.True(t, IsDiagnoseCommand([]string{"diagnose", "./core/..."})) + assert.False(t, IsDiagnoseCommand([]string{"run", "./core/..."})) + assert.False(t, IsDiagnoseCommand([]string{"gotestsum", "./core/..."})) +} + func TestNeedsPostgres(t *testing.T) { repoRoot := findRepoRoot(t) t.Logf("repoRoot: %q", repoRoot) diff --git a/tools/test/main.go b/tools/test/main.go index b55d3cf6727..8732e14f739 100644 --- a/tools/test/main.go +++ b/tools/test/main.go @@ -51,6 +51,8 @@ func dbProviderForArgs(ctx context.Context, count int, args []string) ([]testrig } conf.ParallelIterations = count + conf.DiagnoseMode = dbdetect.IsDiagnoseCommand(args) + conf.PackageSlug = dbdetect.PackageSlug(args) out := output.NewFromApp(conf) pool, err := db.EnsurePool(ctx, conf, out, count) From 2a8c2262db22fc5edeca7f9250d2ea31956d2110 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 15:08:36 -0400 Subject: [PATCH 03/15] lint --- internal/testdb/testdb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index f298df74ee6..a3013e43788 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -5,6 +5,7 @@ import ( "database/sql" "github.com/peterldowns/pgtestdb" + "github.com/smartcontractkit/chainlink/v2/core/store/migrate" ) From 23a2844e97e14dc792b6aa88aa733821d7d0a603 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 15:15:55 -0400 Subject: [PATCH 04/15] cleanup --- debug_url.go | 22 ---------------------- main_test.go | 28 +++++++++++++++++++++------- 2 files changed, 21 insertions(+), 29 deletions(-) delete mode 100644 debug_url.go diff --git a/debug_url.go b/debug_url.go deleted file mode 100644 index cbbb492a6db..00000000000 --- a/debug_url.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "fmt" - "net/url" - "github.com/smartcontractkit/chainlink-common/pkg/config" -) - -func main() { - u, _ := url.Parse("postgres://postgres:postgres@localhost:5432/db") - cUrl := (*config.URL)(u) - sUrl := config.NewSecretURL(cUrl) - fmt.Printf("sUrl.String(): %s\n", sUrl.String()) - - // Now convert it back to url.URL - back := *sUrl.URL() - fmt.Printf("back.String(): %s\n", back.String()) - - // What about User.Password()? - pass, ok := back.User.Password() - fmt.Printf("Password: %s, ok: %v\n", pass, ok) -} diff --git a/main_test.go b/main_test.go index 7735b7de53f..86361734679 100644 --- a/main_test.go +++ b/main_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/google/uuid" + "github.com/peterldowns/pgtestdb" "github.com/rogpeppe/go-internal/testscript" "github.com/stretchr/testify/require" @@ -40,9 +40,9 @@ const ( ) func TestMain(m *testing.M) { - os.Exit(testscript.RunMain(m, map[string]func() int{ - "chainlink": core.Main, - })) + testscript.Main(m, map[string]func(){ + "chainlink": func() { os.Exit(core.Main()) }, + }) } var ( @@ -145,7 +145,21 @@ func newDB(t testing.TB) string { t.Fatalf("failed to parse url: %v", err) } - name := strings.ReplaceAll(uuid.NewString(), "-", "_") + "_test" - u2 := testdb.CreateOrReplace(t, *u, name, true) - return u2.String() + migrator := testdb.Migrator(true) + conf := pgtestdb.Config{ + DriverName: "postgres", + User: u.User.Username(), + Host: u.Hostname(), + Port: u.Port(), + Database: strings.TrimLeft(u.Path, "/"), + Options: u.RawQuery, + ForceTerminateConnections: true, + } + if pass, ok := u.User.Password(); ok { + conf.Password = pass + } + newConf := pgtestdb.Custom(t, conf, migrator) + + u.Path = "/" + newConf.Database + return u.String() } From ecac07756ae4b43b157c579565137d4587562cbe Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 15:33:08 -0400 Subject: [PATCH 05/15] more conversion --- core/cmd/shell_local.go | 2 +- core/scripts/setup_testdb.sh | 3 -- core/store/store.go | 5 ++- core/utils/testutils/heavyweight/orm.go | 46 ++++---------------- go.mod | 3 +- go.sum | 4 +- internal/testdb/testdb.go | 57 +++++++++++++++++++++---- main_test.go | 33 +++----------- 8 files changed, 69 insertions(+), 84 deletions(-) diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 45d2068ff9a..a1b2ed62ad1 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -916,7 +916,7 @@ func (s *Shell) PrepareTestDatabase(c *cli.Context) error { } cfg := s.Config - // Creating pristine DB copy to speed up FullTestDB + dbUrl := cfg.Database().URL() userOnly := c.Bool("user-only") if err := store.PrepareTestDB(s.Logger, dbUrl, userOnly); err != nil { diff --git a/core/scripts/setup_testdb.sh b/core/scripts/setup_testdb.sh index 85aa5812e23..cdaf3547ece 100755 --- a/core/scripts/setup_testdb.sh +++ b/core/scripts/setup_testdb.sh @@ -29,9 +29,6 @@ ALTER DATABASE $database OWNER TO $username; GRANT ALL PRIVILEGES ON DATABASE "$database" TO "$username"; ALTER USER $username CREATEDB; --- Create a pristine database for testing -SELECT 'CREATE DATABASE chainlink_test_pristine WITH OWNER $username;' -WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'chainlink_test_pristine')\gexec EOF # Print the SQL commands diff --git a/core/store/store.go b/core/store/store.go index 10df6cf1bb5..42c4947bb09 100644 --- a/core/store/store.go +++ b/core/store/store.go @@ -28,9 +28,9 @@ import ( cutils "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink/v2/core/services/pg" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" - "github.com/smartcontractkit/chainlink/v2/internal/testdb" ) + //go:embed fixtures/fixtures.sql var fixturesSQL string @@ -264,7 +264,8 @@ func dropDanglingTestDBs(lggr logger.Logger, db *sqlx.DB) (err error) { }() } for _, dbname := range dbs { - if strings.HasPrefix(dbname, testdb.TestDBNamePrefix) && !strings.HasSuffix(dbname, "_pristine") { + if (strings.HasPrefix(dbname, "chainlink_test_") && !strings.HasSuffix(dbname, "_pristine")) || + (strings.HasPrefix(dbname, "testdb_") && !strings.HasPrefix(dbname, "testdb_tpl_")) { ch <- dbname } } diff --git a/core/utils/testutils/heavyweight/orm.go b/core/utils/testutils/heavyweight/orm.go index dfaeee973c6..b348582be8a 100644 --- a/core/utils/testutils/heavyweight/orm.go +++ b/core/utils/testutils/heavyweight/orm.go @@ -3,21 +3,18 @@ package heavyweight import ( - "net/url" "os" - "strings" "testing" - "github.com/jmoiron/sqlx" - "github.com/peterldowns/pgtestdb" "github.com/stretchr/testify/require" + "github.com/jmoiron/sqlx" + commoncfg "github.com/smartcontractkit/chainlink-common/pkg/config" pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink/v2/core/store" - "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/pg" @@ -29,8 +26,7 @@ import ( // unit tests, so you can do things like use other Postgres connection types with it. func FullTestDBV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) { cfg, db := FullTestDBNoFixturesV2(t, overrideFn) - ctx := t.Context() - _, err := db.ExecContext(ctx, store.FixturesSQL()) + _, err := db.Exec(store.FixturesSQL()) require.NoError(t, err) return cfg, db } @@ -48,39 +44,15 @@ func FullTestDBEmptyV2(t testing.TB, overrideFn func(c *chainlink.Config, s *cha func prepareDB(t testing.TB, withTemplate bool, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) { tests.SkipShort(t, "FullTestDB") - // Read env.DatabaseURL directly to get the base connection - rawDBURL := string(env.DatabaseURL.Get()) - if rawDBURL == "" { - t.Fatalf("you must provide a CL_DATABASE_URL environment variable") - } - - dbURL, err := url.Parse(rawDBURL) - require.NoError(t, err) - - migrator := testdb.Migrator(withTemplate) - conf := pgtestdb.Config{ - DriverName: pgcommon.DriverPostgres, - User: dbURL.User.Username(), - Host: dbURL.Hostname(), - Port: dbURL.Port(), - Database: strings.TrimLeft(dbURL.Path, "/"), - Options: dbURL.RawQuery, - ForceTerminateConnections: true, - } - if pass, ok := dbURL.User.Password(); ok { - conf.Password = pass - } - newConf := pgtestdb.Custom(t, conf, migrator) - - migrationTestDBURL := *dbURL - migrationTestDBURL.Path = "/" + newConf.Database - dbStr := migrationTestDBURL.String() + dbURL := testdb.New(t, withTemplate) + dbStr := dbURL.String() gcfg := configtest.NewGeneralConfigSimulated(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.Database.DriverName = pgcommon.DriverPostgres - s.Database.URL = models.NewSecretURL((*commoncfg.URL)(&migrationTestDBURL)) - // Explicitly allow simple passwords since tests use `postgres` password - s.Database.AllowSimplePasswords = new(true) + s.Database.URL = models.NewSecretURL((*commoncfg.URL)(dbURL)) + // Explicitly allow simple passwords since test DB users often have simple passwords like `pgtdbpass` + s.Database.AllowSimplePasswords = new(bool) + *s.Database.AllowSimplePasswords = true if overrideFn != nil { overrideFn(c, s) } diff --git a/go.mod b/go.mod index e79cf9851b9..6d17264c9ce 100644 --- a/go.mod +++ b/go.mod @@ -65,6 +65,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml/v2 v2.3.1 + github.com/peterldowns/pgtestdb v0.1.1 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.26.0 github.com/prometheus/client_golang v1.23.2 @@ -332,8 +333,6 @@ require ( github.com/onsi/ginkgo/v2 v2.27.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect - github.com/peterldowns/pgtestdb v0.1.1 // indirect - github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect diff --git a/go.sum b/go.sum index 3862ebfd427..924a5e3971a 100644 --- a/go.sum +++ b/go.sum @@ -1027,8 +1027,8 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQm github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= -github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1 h1:f+e5A8elEb+5VJnrtlPI8GKq2LunCFJH+3by7ekJ3io= -github.com/peterldowns/pgtestdb/migrators/goosemigrator v0.1.1/go.mod h1:ELFSiezLH4Hp2W1XSCtq/bniC9ZkkdyFy06tLOeaAqw= +github.com/peterldowns/testy v0.0.1 h1:9a6LzvnKcL52Crzud1z7jbsAojTntCh89ho6mgsr4KU= +github.com/peterldowns/testy v0.0.1/go.mod h1:J4sm75UEzbfBIcq0zbrshWWjsJQiJ5RrhTPYKVY2Ww8= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index a3013e43788..e1e8219433c 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -3,34 +3,73 @@ package testdb import ( "context" "database/sql" + "net/url" + "strings" + "testing" "github.com/peterldowns/pgtestdb" + "github.com/peterldowns/pgtestdb/migrators/common" + "github.com/stretchr/testify/require" + _ "github.com/jackc/pgx/v5/stdlib" + pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" ) -const ( - // PristineDBName is a clean copy of test DB with migrations. - PristineDBName = "chainlink_test_pristine" - // TestDBNamePrefix is a common prefix that will be auto-removed by the dangling DB cleanup process. - TestDBNamePrefix = "chainlink_test_" -) +// New provisions a new isolated test database (via pgtestdb) and returns the connected URL. +// The returned URL will have the provisioned DB role credentials (not superuser), so tests +// run exactly as they would in a real deployment environment without excess privileges. +func New(t testing.TB, withTemplate bool) *url.URL { + t.Helper() + + rawDBURL := string(env.DatabaseURL.Get()) + if rawDBURL == "" { + t.Fatalf("you must provide a CL_DATABASE_URL environment variable") + } + + dbURL, err := url.Parse(rawDBURL) + require.NoError(t, err) + + migrator := Migrator(withTemplate) + conf := pgtestdb.Config{ + DriverName: pgcommon.DriverPostgres, + User: dbURL.User.Username(), + Host: dbURL.Hostname(), + Port: dbURL.Port(), + Database: strings.TrimLeft(dbURL.Path, "/"), + Options: dbURL.RawQuery, + ForceTerminateConnections: true, + } + if pass, ok := dbURL.User.Password(); ok { + conf.Password = pass + } + + newConf := pgtestdb.Custom(t, conf, migrator) + newURLStr := newConf.URL() + newURL, err := url.Parse(newURLStr) + require.NoError(t, err) + return newURL +} type migrator struct { withTemplate bool } func (m *migrator) Hash() (string, error) { - if m.withTemplate { - return "withTemplate", nil + if !m.withTemplate { + return "empty", nil } - return "empty", nil + // Hash the embedded migration files so template databases rebuild if schemas change. + return common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) } func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Config) error { if !m.withTemplate { return nil } + // Note: We do not call SetMigrationENVVars because it is only strictly needed for goose up-to + // specific versions or custom seeder data. If needed, you can explicitly configure goose here. return migrate.Migrate(ctx, db) } diff --git a/main_test.go b/main_test.go index 86361734679..5da01d65536 100644 --- a/main_test.go +++ b/main_test.go @@ -2,21 +2,18 @@ package main import ( "fmt" - "net/url" "os" "path/filepath" "strconv" "strings" "testing" - "github.com/peterldowns/pgtestdb" "github.com/rogpeppe/go-internal/testscript" "github.com/stretchr/testify/require" "github.com/smartcontractkit/freeport" "github.com/smartcontractkit/chainlink/v2/core" - "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/static" "github.com/smartcontractkit/chainlink/v2/internal/testdb" "github.com/smartcontractkit/chainlink/v2/tools/txtar" @@ -40,9 +37,9 @@ const ( ) func TestMain(m *testing.M) { - testscript.Main(m, map[string]func(){ - "chainlink": func() { os.Exit(core.Main()) }, - }) + os.Exit(testscript.RunMain(m, map[string]func() int{ + "chainlink": core.Main, + })) } var ( @@ -140,26 +137,6 @@ func takeFreePort() (int, func(), error) { } func newDB(t testing.TB) string { - u, err := url.Parse(string(env.DatabaseURL.Get())) - if err != nil { - t.Fatalf("failed to parse url: %v", err) - } - - migrator := testdb.Migrator(true) - conf := pgtestdb.Config{ - DriverName: "postgres", - User: u.User.Username(), - Host: u.Hostname(), - Port: u.Port(), - Database: strings.TrimLeft(u.Path, "/"), - Options: u.RawQuery, - ForceTerminateConnections: true, - } - if pass, ok := u.User.Password(); ok { - conf.Password = pass - } - newConf := pgtestdb.Custom(t, conf, migrator) - - u.Path = "/" + newConf.Database - return u.String() + u2 := testdb.New(t, true) + return u2.String() } From 14fb76b5a68a65e673d03628bcaaf771c0270113 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 15:39:43 -0400 Subject: [PATCH 06/15] nits and tests --- core/store/store.go | 3 +-- core/utils/testutils/heavyweight/orm.go | 2 +- internal/testdb/testdb.go | 19 ++++++++++++++---- internal/testdb/testdb_test.go | 26 +++++++++++++++++++++++++ main_test.go | 6 +----- 5 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 internal/testdb/testdb_test.go diff --git a/core/store/store.go b/core/store/store.go index 42c4947bb09..8f419e86824 100644 --- a/core/store/store.go +++ b/core/store/store.go @@ -30,7 +30,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/store/migrate" ) - //go:embed fixtures/fixtures.sql var fixturesSQL string @@ -265,7 +264,7 @@ func dropDanglingTestDBs(lggr logger.Logger, db *sqlx.DB) (err error) { } for _, dbname := range dbs { if (strings.HasPrefix(dbname, "chainlink_test_") && !strings.HasSuffix(dbname, "_pristine")) || - (strings.HasPrefix(dbname, "testdb_") && !strings.HasPrefix(dbname, "testdb_tpl_")) { + (strings.HasPrefix(dbname, "testdb_") && (!strings.HasPrefix(dbname, "testdb_tpl_") || strings.Contains(dbname, "_inst_"))) { ch <- dbname } } diff --git a/core/utils/testutils/heavyweight/orm.go b/core/utils/testutils/heavyweight/orm.go index b348582be8a..a0fb76063af 100644 --- a/core/utils/testutils/heavyweight/orm.go +++ b/core/utils/testutils/heavyweight/orm.go @@ -26,7 +26,7 @@ import ( // unit tests, so you can do things like use other Postgres connection types with it. func FullTestDBV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) { cfg, db := FullTestDBNoFixturesV2(t, overrideFn) - _, err := db.Exec(store.FixturesSQL()) + _, err := db.ExecContext(t.Context(), store.FixturesSQL()) require.NoError(t, err) return cfg, db } diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index e1e8219433c..b1e381b7344 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -31,7 +31,7 @@ func New(t testing.TB, withTemplate bool) *url.URL { dbURL, err := url.Parse(rawDBURL) require.NoError(t, err) - migrator := Migrator(withTemplate) + migrator := migratorConfig(withTemplate) conf := pgtestdb.Config{ DriverName: pgcommon.DriverPostgres, User: dbURL.User.Username(), @@ -60,8 +60,19 @@ func (m *migrator) Hash() (string, error) { if !m.withTemplate { return "empty", nil } - // Hash the embedded migration files so template databases rebuild if schemas change. - return common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) + h1, err := common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) + if err != nil { + return "", err + } + h2, err := common.HashDirs(migrate.EmbedMigrations, "*.go", migrate.MigrationsDir) + if err != nil { + return "", err + } + hash := common.NewRecursiveHash( + common.Field("sql", h1), + common.Field("go", h2), + ) + return hash.String(), nil } func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Config) error { @@ -73,6 +84,6 @@ func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Conf return migrate.Migrate(ctx, db) } -func Migrator(withTemplate bool) pgtestdb.Migrator { +func migratorConfig(withTemplate bool) pgtestdb.Migrator { return &migrator{withTemplate: withTemplate} } diff --git a/internal/testdb/testdb_test.go b/internal/testdb/testdb_test.go new file mode 100644 index 00000000000..fa04262e561 --- /dev/null +++ b/internal/testdb/testdb_test.go @@ -0,0 +1,26 @@ +package testdb + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigrator_Hash(t *testing.T) { + t.Parallel() + + t.Run("empty returns empty", func(t *testing.T) { + m := migratorConfig(false) + hash, err := m.Hash() + require.NoError(t, err) + require.Equal(t, "empty", hash) + }) + + t.Run("withTemplate hashes successfully", func(t *testing.T) { + m := migratorConfig(true) + hash, err := m.Hash() + require.NoError(t, err) + require.NotEmpty(t, hash) + require.NotEqual(t, "empty", hash) + }) +} diff --git a/main_test.go b/main_test.go index 5da01d65536..7d296cf765c 100644 --- a/main_test.go +++ b/main_test.go @@ -120,7 +120,7 @@ func commonEnv(t testing.TB) func(*testscript.Env) error { envVarName := strings.TrimSpace(string(b)) te.T().Log("test database requested:", envVarName) - u2 := newDB(t) + u2 := testdb.New(t, true).String() te.Setenv(envVarName, u2) } @@ -136,7 +136,3 @@ func takeFreePort() (int, func(), error) { return ports[0], func() { freeport.Return(ports) }, nil } -func newDB(t testing.TB) string { - u2 := testdb.New(t, true) - return u2.String() -} From 8e4f92a033200abf82d58b157182b8fa62194f92 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 16:16:24 -0400 Subject: [PATCH 07/15] Benchmarks --- core/internal/testutils/pgtest/bench_test.go | 35 +++++++++++++ .../utils/testutils/heavyweight/bench_test.go | 50 +++++++++++++++++++ tools/test/internal/dbdetect/dbdetect.go | 18 +++++-- tools/test/internal/dbdetect/dbdetect_test.go | 10 ++++ 4 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 core/internal/testutils/pgtest/bench_test.go create mode 100644 core/utils/testutils/heavyweight/bench_test.go diff --git a/core/internal/testutils/pgtest/bench_test.go b/core/internal/testutils/pgtest/bench_test.go new file mode 100644 index 00000000000..51f0175af47 --- /dev/null +++ b/core/internal/testutils/pgtest/bench_test.go @@ -0,0 +1,35 @@ +package pgtest + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/core/config/env" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" +) + +func requireBenchDB(b *testing.B) { + b.Helper() + testutils.SkipShortDB(b) + if string(env.DatabaseURL.Get()) == "" { + b.Skip("CL_DATABASE_URL required") + } +} + +// BenchmarkNewSqlxDB measures txdb connection provisioning (open transaction + session +// timeouts). Each iteration closes the connection explicitly so the cost reflects a +// single test's DB setup, not accumulated transactions. +// +// Run: CL_DATABASE_URL='postgres://...' go test -bench=BenchmarkNewSqlxDB -benchmem ./core/internal/testutils/pgtest/ +func BenchmarkNewSqlxDB(b *testing.B) { + requireBenchDB(b) + b.ReportAllocs() + + for b.Loop() { + db := NewSqlxDB(b) + _, err := db.Exec("SELECT 1") + require.NoError(b, err) + require.NoError(b, db.Close()) + } +} diff --git a/core/utils/testutils/heavyweight/bench_test.go b/core/utils/testutils/heavyweight/bench_test.go new file mode 100644 index 00000000000..7f0a0b4320d --- /dev/null +++ b/core/utils/testutils/heavyweight/bench_test.go @@ -0,0 +1,50 @@ +package heavyweight + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/core/config/env" +) + +func requireBenchDB(b *testing.B) { + b.Helper() + if testing.Short() { + b.Skip("skipping DB benchmark with -short") + } + if string(env.DatabaseURL.Get()) == "" { + b.Skip("CL_DATABASE_URL required; run via make test or set CL_DATABASE_URL") + } +} + +// BenchmarkFullTestDBNoFixturesV2 measures isolated DB provisioning via +// CreateOrReplace (DROP + CREATE DATABASE … WITH TEMPLATE chainlink_test_pristine). +// Database drops are deferred to benchmark teardown via t.Cleanup; use a modest +// benchtime to avoid accumulating hundreds of databases locally. +// +// Run: CL_DATABASE_URL='postgres://...' go test -bench=BenchmarkFullTestDB -benchtime=5x -benchmem ./core/utils/testutils/heavyweight/ +func BenchmarkFullTestDBNoFixturesV2(b *testing.B) { + requireBenchDB(b) + b.ReportAllocs() + + for b.Loop() { + _, db := FullTestDBNoFixturesV2(b, nil) + _, err := db.Exec("SELECT 1") + require.NoError(b, err) + require.NoError(b, db.Close()) + } +} + +// BenchmarkFullTestDBEmptyV2 measures empty DB provisioning (no migrations template). +func BenchmarkFullTestDBEmptyV2(b *testing.B) { + requireBenchDB(b) + b.ReportAllocs() + + for b.Loop() { + _, db := FullTestDBEmptyV2(b, nil) + _, err := db.Exec("SELECT 1") + require.NoError(b, err) + require.NoError(b, db.Close()) + } +} diff --git a/tools/test/internal/dbdetect/dbdetect.go b/tools/test/internal/dbdetect/dbdetect.go index 87c09364f12..0b5e0583cfb 100644 --- a/tools/test/internal/dbdetect/dbdetect.go +++ b/tools/test/internal/dbdetect/dbdetect.go @@ -207,8 +207,20 @@ func NeedsPostgres(repoRoot string, args []string) (bool, error) { return true, fmt.Errorf("go list: %w: %s", err, strings.TrimSpace(stderr.String())) } - targetDep := "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - needsDB := strings.Contains(stdout.String(), targetDep) + deps := stdout.String() + for _, targetDep := range postgresTestDeps { + if strings.Contains(deps, targetDep) { + return true, nil + } + } + + return false, nil +} - return needsDB, nil +// postgresTestDeps lists packages that imply a real Postgres server (CL_DATABASE_URL). +// go list -deps -test must match at least one for the testrig harness to start Postgres. +var postgresTestDeps = []string{ + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest", + "github.com/smartcontractkit/chainlink/v2/core/utils/testutils/heavyweight", + "github.com/smartcontractkit/chainlink/v2/internal/testdb", } diff --git a/tools/test/internal/dbdetect/dbdetect_test.go b/tools/test/internal/dbdetect/dbdetect_test.go index e333851b5ab..f0529c7d662 100644 --- a/tools/test/internal/dbdetect/dbdetect_test.go +++ b/tools/test/internal/dbdetect/dbdetect_test.go @@ -141,6 +141,16 @@ func TestNeedsPostgres(t *testing.T) { args: []string{"./core/internal/testutils/dbdetectfixture"}, want: false, }, + { + name: "heavyweight benchmarks need DB", + args: []string{ + "-bench=BenchmarkFullTestDB", + "-benchtime=5x", + "-benchmem", + "./core/utils/testutils/heavyweight/", + }, + want: true, + }, } for _, tt := range tests { From 9f60e3b3cb24f267eb4bc8c2b753dae286f06f21 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 16:27:45 -0400 Subject: [PATCH 08/15] require --- internal/testdb/testdb.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index b1e381b7344..1f2ad74a410 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -24,12 +24,10 @@ func New(t testing.TB, withTemplate bool) *url.URL { t.Helper() rawDBURL := string(env.DatabaseURL.Get()) - if rawDBURL == "" { - t.Fatalf("you must provide a CL_DATABASE_URL environment variable") - } - + require.NotEmpty(t, rawDBURL, "CL_DATABASE_URL environment variable is required") dbURL, err := url.Parse(rawDBURL) require.NoError(t, err) + require.NotEmpty(t, dbURL.String(), "CL_DATABASE_URL environment variable is required") migrator := migratorConfig(withTemplate) conf := pgtestdb.Config{ From 89508e9f910f9140e2df8d615565956005afbae5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 16:48:38 -0400 Subject: [PATCH 09/15] fix bench --- .../utils/testutils/heavyweight/bench_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/utils/testutils/heavyweight/bench_test.go b/core/utils/testutils/heavyweight/bench_test.go index 7f0a0b4320d..c810cf242aa 100644 --- a/core/utils/testutils/heavyweight/bench_test.go +++ b/core/utils/testutils/heavyweight/bench_test.go @@ -26,11 +26,18 @@ func requireBenchDB(b *testing.B) { // Run: CL_DATABASE_URL='postgres://...' go test -bench=BenchmarkFullTestDB -benchtime=5x -benchmem ./core/utils/testutils/heavyweight/ func BenchmarkFullTestDBNoFixturesV2(b *testing.B) { requireBenchDB(b) - b.ReportAllocs() + // Warm up template; cost excluded from timer. + _, db := FullTestDBNoFixturesV2(b, nil) + _, err := db.ExecContext(b.Context(), "SELECT 1") + require.NoError(b, err) + require.NoError(b, db.Close()) + + b.ReportAllocs() + b.ResetTimer() for b.Loop() { _, db := FullTestDBNoFixturesV2(b, nil) - _, err := db.Exec("SELECT 1") + _, err := db.ExecContext(b.Context(), "SELECT 1") require.NoError(b, err) require.NoError(b, db.Close()) } @@ -41,9 +48,15 @@ func BenchmarkFullTestDBEmptyV2(b *testing.B) { requireBenchDB(b) b.ReportAllocs() + // Warm up template; cost excluded from timer. + _, db := FullTestDBEmptyV2(b, nil) + _, err := db.ExecContext(b.Context(), "SELECT 1") + require.NoError(b, err) + require.NoError(b, db.Close()) + for b.Loop() { _, db := FullTestDBEmptyV2(b, nil) - _, err := db.Exec("SELECT 1") + _, err := db.ExecContext(b.Context(), "SELECT 1") require.NoError(b, err) require.NoError(b, db.Close()) } From 26a3bc8904215bb985844679499871ec35f60aa5 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 16:58:38 -0400 Subject: [PATCH 10/15] cache hashes --- internal/testdb/testdb.go | 39 ++++++++++++++++++++++------------ internal/testdb/testdb_test.go | 21 ++++++++++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index 1f2ad74a410..772b8f096b7 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -5,6 +5,7 @@ import ( "database/sql" "net/url" "strings" + "sync" "testing" "github.com/peterldowns/pgtestdb" @@ -54,23 +55,35 @@ type migrator struct { withTemplate bool } +// templateHash is computed once per process from embedded migration files. +// Migrations are compile-time constants (go:embed), so the hash is safe to cache. +var ( + templateHashOnce sync.Once + templateHash string + templateHashErr error +) + func (m *migrator) Hash() (string, error) { if !m.withTemplate { return "empty", nil } - h1, err := common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) - if err != nil { - return "", err - } - h2, err := common.HashDirs(migrate.EmbedMigrations, "*.go", migrate.MigrationsDir) - if err != nil { - return "", err - } - hash := common.NewRecursiveHash( - common.Field("sql", h1), - common.Field("go", h2), - ) - return hash.String(), nil + templateHashOnce.Do(func() { + h1, err := common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) + if err != nil { + templateHashErr = err + return + } + h2, err := common.HashDirs(migrate.EmbedMigrations, "*.go", migrate.MigrationsDir) + if err != nil { + templateHashErr = err + return + } + templateHash = common.NewRecursiveHash( + common.Field("sql", h1), + common.Field("go", h2), + ).String() + }) + return templateHash, templateHashErr } func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Config) error { diff --git a/internal/testdb/testdb_test.go b/internal/testdb/testdb_test.go index fa04262e561..27f856700bd 100644 --- a/internal/testdb/testdb_test.go +++ b/internal/testdb/testdb_test.go @@ -23,4 +23,25 @@ func TestMigrator_Hash(t *testing.T) { require.NotEmpty(t, hash) require.NotEqual(t, "empty", hash) }) + + t.Run("withTemplate returns same hash from new migrator instances", func(t *testing.T) { + hash1, err := migratorConfig(true).Hash() + require.NoError(t, err) + + hash2, err := migratorConfig(true).Hash() + require.NoError(t, err) + require.Equal(t, hash1, hash2) + }) +} + +func TestMigrator_HashCachedNoAllocs(t *testing.T) { + m := migratorConfig(true) + _, err := m.Hash() + require.NoError(t, err) + + allocs := testing.AllocsPerRun(5, func() { + _, err := m.Hash() + require.NoError(t, err) + }) + require.Zero(t, allocs) } From 74cc959c5c698926a0d9dffc9cd5ecf1b2f38e3e Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 5 Jun 2026 19:42:50 -0400 Subject: [PATCH 11/15] lint --- core/services/chainlink/config.go | 5 ++--- core/utils/testutils/heavyweight/orm.go | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/core/services/chainlink/config.go b/core/services/chainlink/config.go index e65da56a227..27ce964731c 100644 --- a/core/services/chainlink/config.go +++ b/core/services/chainlink/config.go @@ -434,7 +434,7 @@ func (s *Secrets) SetFrom(f *Secrets) (err error) { func (s *Secrets) setDefaults() { if nil == s.Database.AllowSimplePasswords { - s.Database.AllowSimplePasswords = new(bool) + s.Database.AllowSimplePasswords = new(false) } } @@ -509,8 +509,7 @@ func (s *Secrets) setEnv() error { } } if env.DatabaseAllowSimplePasswords.IsTrue() { - s.Database.AllowSimplePasswords = new(bool) - *s.Database.AllowSimplePasswords = true + s.Database.AllowSimplePasswords = new(true) } if keystorePassword := env.PasswordKeystore.Get(); keystorePassword != "" { s.Password.Keystore = &keystorePassword diff --git a/core/utils/testutils/heavyweight/orm.go b/core/utils/testutils/heavyweight/orm.go index a0fb76063af..31362407efd 100644 --- a/core/utils/testutils/heavyweight/orm.go +++ b/core/utils/testutils/heavyweight/orm.go @@ -51,8 +51,7 @@ func prepareDB(t testing.TB, withTemplate bool, overrideFn func(c *chainlink.Con c.Database.DriverName = pgcommon.DriverPostgres s.Database.URL = models.NewSecretURL((*commoncfg.URL)(dbURL)) // Explicitly allow simple passwords since test DB users often have simple passwords like `pgtdbpass` - s.Database.AllowSimplePasswords = new(bool) - *s.Database.AllowSimplePasswords = true + s.Database.AllowSimplePasswords = new(false) if overrideFn != nil { overrideFn(c, s) } From 0c86d92e58756ca6ed0c03879793bce95d73ba48 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Sat, 6 Jun 2026 10:36:14 -0400 Subject: [PATCH 12/15] tidy --- core/scripts/go.sum | 2 ++ deployment/go.mod | 1 + deployment/go.sum | 4 ++++ integration-tests/go.mod | 1 + integration-tests/go.sum | 4 ++++ integration-tests/load/go.mod | 1 + integration-tests/load/go.sum | 4 ++++ system-tests/lib/go.sum | 2 ++ system-tests/tests/go.sum | 2 ++ 9 files changed, 21 insertions(+) diff --git a/core/scripts/go.sum b/core/scripts/go.sum index d28cc07b49f..9cd791ad7e8 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1386,6 +1386,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/deployment/go.mod b/deployment/go.mod index 3a4c0fc72a2..fc15d6ef43e 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -382,6 +382,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/peterldowns/pgtestdb v0.1.1 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect diff --git a/deployment/go.sum b/deployment/go.sum index bc9ed3b38f0..a6d1d3901b0 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1208,6 +1208,10 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= +github.com/peterldowns/testy v0.0.1 h1:9a6LzvnKcL52Crzud1z7jbsAojTntCh89ho6mgsr4KU= +github.com/peterldowns/testy v0.0.1/go.mod h1:J4sm75UEzbfBIcq0zbrshWWjsJQiJ5RrhTPYKVY2Ww8= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index a509ae879ac..e786d000d92 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -355,6 +355,7 @@ require ( github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/peterldowns/pgtestdb v0.1.1 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index af8eed195c4..b4a96a1a83d 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1191,6 +1191,10 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= +github.com/peterldowns/testy v0.0.1 h1:9a6LzvnKcL52Crzud1z7jbsAojTntCh89ho6mgsr4KU= +github.com/peterldowns/testy v0.0.1/go.mod h1:J4sm75UEzbfBIcq0zbrshWWjsJQiJ5RrhTPYKVY2Ww8= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index b44960f6256..b08ebda6f84 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -420,6 +420,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.3.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/peterldowns/pgtestdb v0.1.1 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index e61398308bc..093bb96aacb 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1426,6 +1426,10 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= +github.com/peterldowns/testy v0.0.1 h1:9a6LzvnKcL52Crzud1z7jbsAojTntCh89ho6mgsr4KU= +github.com/peterldowns/testy v0.0.1/go.mod h1:J4sm75UEzbfBIcq0zbrshWWjsJQiJ5RrhTPYKVY2Ww8= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 4e4e6f79145..be5c6602530 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1355,6 +1355,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index e2f9ea77ca3..b709c870055 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1365,6 +1365,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterldowns/pgtestdb v0.1.1 h1:+hBCD1DcbKeg5Sfg0G+5WNIy/Cm0ORgwMkF4ygihrmU= +github.com/peterldowns/pgtestdb v0.1.1/go.mod h1:yVWInWV0dxvmLdL2ao3nXDzWZ9+G6EhJ4gRwvI1Ozeg= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= From cd010940c18e29fa77ef649cb825678519524644 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Sat, 6 Jun 2026 15:43:53 -0400 Subject: [PATCH 13/15] lint --- core/cmd/shell_local.go | 2 -- core/store/database_ddl.go | 37 +++++++++++++++++++++++++++++++++++++ core/store/store.go | 10 ++++------ internal/testdb/testdb.go | 10 +++++----- 4 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 core/store/database_ddl.go diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index a1b2ed62ad1..1b2e46e29d6 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -915,8 +915,6 @@ func (s *Shell) PrepareTestDatabase(c *cli.Context) error { return s.errorOut(err) } cfg := s.Config - - dbUrl := cfg.Database().URL() userOnly := c.Bool("user-only") if err := store.PrepareTestDB(s.Logger, dbUrl, userOnly); err != nil { diff --git a/core/store/database_ddl.go b/core/store/database_ddl.go new file mode 100644 index 00000000000..878f00bff77 --- /dev/null +++ b/core/store/database_ddl.go @@ -0,0 +1,37 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "regexp" + + "github.com/lib/pq" +) + +var postgresDBNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +func quotePostgresDBName(name string) (string, error) { + if name == "" || len(name) > 63 || !postgresDBNamePattern.MatchString(name) { + return "", fmt.Errorf("invalid postgres database name: %q", name) + } + return pq.QuoteIdentifier(name), nil +} + +func execDropDatabase(ctx context.Context, db *sql.DB, name string) error { + quoted, err := quotePostgresDBName(name) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+quoted+" WITH (FORCE)") + return err +} + +func execCreateDatabase(ctx context.Context, db *sql.DB, name string) error { + quoted, err := quotePostgresDBName(name) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, "CREATE DATABASE "+quoted) + return err +} diff --git a/core/store/store.go b/core/store/store.go index 8f419e86824..77d0cbeb404 100644 --- a/core/store/store.go +++ b/core/store/store.go @@ -132,15 +132,13 @@ func dropAndCreateDB(parsed url.URL, _ bool) (err error) { // Second parameter kept for ResetDatabase API compatibility (preparetest --force). ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - // PostgreSQL does not support bound parameters for database names; pq.QuoteIdentifier is the supported escape. - _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)") - if err != nil { + // PostgreSQL does not support bound parameters for database names; quotePostgresDBName validates and escapes. + if err = execDropDatabase(ctx, db, dbname); err != nil { return fmt.Errorf("unable to drop postgres database: %w", err) } ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(dbname)) - if err != nil { + if err = execCreateDatabase(ctx, db, dbname); err != nil { return fmt.Errorf("unable to create postgres database: %w", err) } return nil @@ -257,7 +255,7 @@ func dropDanglingTestDBs(lggr logger.Logger, db *sqlx.DB) (err error) { errCh <- func() error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - return cutils.JustError(db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)")) + return execDropDatabase(ctx, db.DB, dbname) }() } }() diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index 772b8f096b7..12f9a30919e 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -12,7 +12,7 @@ import ( "github.com/peterldowns/pgtestdb/migrators/common" "github.com/stretchr/testify/require" - _ "github.com/jackc/pgx/v5/stdlib" + _ "github.com/jackc/pgx/v5/stdlib" // registers pgx driver for pgcommon.DriverPostgres pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" @@ -60,7 +60,7 @@ type migrator struct { var ( templateHashOnce sync.Once templateHash string - templateHashErr error + errTemplateHash error ) func (m *migrator) Hash() (string, error) { @@ -70,12 +70,12 @@ func (m *migrator) Hash() (string, error) { templateHashOnce.Do(func() { h1, err := common.HashDirs(migrate.EmbedMigrations, "*.sql", migrate.MigrationsDir) if err != nil { - templateHashErr = err + errTemplateHash = err return } h2, err := common.HashDirs(migrate.EmbedMigrations, "*.go", migrate.MigrationsDir) if err != nil { - templateHashErr = err + errTemplateHash = err return } templateHash = common.NewRecursiveHash( @@ -83,7 +83,7 @@ func (m *migrator) Hash() (string, error) { common.Field("go", h2), ).String() }) - return templateHash, templateHashErr + return templateHash, errTemplateHash } func (m *migrator) Migrate(ctx context.Context, db *sql.DB, config pgtestdb.Config) error { From b89f7dc9b7136c29d9bdaef100d79cf42d6982b4 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Sat, 6 Jun 2026 15:56:51 -0400 Subject: [PATCH 14/15] fix is test db url --- core/config/toml/types.go | 14 +++++++++++++- core/config/toml/types_test.go | 2 ++ core/utils/testutils/heavyweight/orm.go | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/config/toml/types.go b/core/config/toml/types.go index f1e9d9c6d14..a7f8aececa6 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -390,8 +390,20 @@ type DatabaseSecrets struct { AllowSimplePasswords *bool } +func isTestDatabaseURL(dbURI url.URL) bool { + dbname := strings.TrimPrefix(dbURI.Path, "/") + if strings.Contains(dbname, "_test") { + return true + } + // pgtestdb template/instance databases (testdb_tpl_*_inst_*). + if strings.HasPrefix(dbname, "testdb_") { + return true + } + return false +} + func validateDBURL(dbURI url.URL) error { - if strings.Contains(dbURI.Redacted(), "_test") { + if isTestDatabaseURL(dbURI) { return nil } diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index 04db63ed194..bae3d4b1be6 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -99,6 +99,8 @@ func Test_validateDBURL(t *testing.T) { {"with user and password of insufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=shortpw&user=myuser", fmt.Sprintf("%s %s\n", utils.ErrMsgHeader, "password is less than 16 characters long")}, {"with no user and password of sufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=thisisareallylongpassword", ""}, {"with user and password of sufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=thisisareallylongpassword&user=myuser", ""}, + {"pgtestdb instance with short password", "postgresql://pgtdbuser:short@localhost:5432/testdb_tpl_abc_inst_def?sslmode=disable", ""}, + {"chainlink_test with short password", "postgresql://postgres:short@localhost:5432/chainlink_test?sslmode=disable", ""}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/core/utils/testutils/heavyweight/orm.go b/core/utils/testutils/heavyweight/orm.go index 31362407efd..5aaba41aeca 100644 --- a/core/utils/testutils/heavyweight/orm.go +++ b/core/utils/testutils/heavyweight/orm.go @@ -50,7 +50,7 @@ func prepareDB(t testing.TB, withTemplate bool, overrideFn func(c *chainlink.Con gcfg := configtest.NewGeneralConfigSimulated(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.Database.DriverName = pgcommon.DriverPostgres s.Database.URL = models.NewSecretURL((*commoncfg.URL)(dbURL)) - // Explicitly allow simple passwords since test DB users often have simple passwords like `pgtdbpass` + // pgtestdb URLs use short passwords; validateDBURL exempts testdb_* database names. s.Database.AllowSimplePasswords = new(false) if overrideFn != nil { overrideFn(c, s) From 21b2392a44e4fed4eb78879d5cd93a8e4bdf59ab Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Sat, 6 Jun 2026 16:07:21 -0400 Subject: [PATCH 15/15] lint --- core/store/database_ddl.go | 13 +++++++++++-- internal/testdb/testdb.go | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/store/database_ddl.go b/core/store/database_ddl.go index 878f00bff77..51b82ffb102 100644 --- a/core/store/database_ddl.go +++ b/core/store/database_ddl.go @@ -9,6 +9,11 @@ import ( "github.com/lib/pq" ) +const ( + dropDatabaseSQLFmt = "DROP DATABASE IF EXISTS %s WITH (FORCE)" + createDatabaseSQLFmt = "CREATE DATABASE %s" +) + var postgresDBNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) func quotePostgresDBName(name string) (string, error) { @@ -23,7 +28,9 @@ func execDropDatabase(ctx context.Context, db *sql.DB, name string) error { if err != nil { return err } - _, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+quoted+" WITH (FORCE)") + // PostgreSQL does not support bound parameters for database identifiers. + //nolint:gosec // G701 -- name validated by quotePostgresDBName; identifier escaped with pq.QuoteIdentifier + _, err = db.ExecContext(ctx, fmt.Sprintf(dropDatabaseSQLFmt, quoted)) return err } @@ -32,6 +39,8 @@ func execCreateDatabase(ctx context.Context, db *sql.DB, name string) error { if err != nil { return err } - _, err = db.ExecContext(ctx, "CREATE DATABASE "+quoted) + // PostgreSQL does not support bound parameters for database identifiers. + //nolint:gosec // G701 -- name validated by quotePostgresDBName; identifier escaped with pq.QuoteIdentifier + _, err = db.ExecContext(ctx, fmt.Sprintf(createDatabaseSQLFmt, quoted)) return err } diff --git a/internal/testdb/testdb.go b/internal/testdb/testdb.go index 12f9a30919e..c723e7d32d7 100644 --- a/internal/testdb/testdb.go +++ b/internal/testdb/testdb.go @@ -8,11 +8,11 @@ import ( "sync" "testing" + _ "github.com/jackc/pgx/v5/stdlib" // registers pgx driver for pgcommon.DriverPostgres "github.com/peterldowns/pgtestdb" "github.com/peterldowns/pgtestdb/migrators/common" "github.com/stretchr/testify/require" - _ "github.com/jackc/pgx/v5/stdlib" // registers pgx driver for pgcommon.DriverPostgres pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg" "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/store/migrate"