[DX-4370] Convert FullTestDB to pgtestdb

core/cmd/shell_local.go

@@ -915,8 +915,6 @@ func (s *Shell) PrepareTestDatabase(c *cli.Context) error {
 		return s.errorOut(err)
 	}
 	cfg := s.Config
-
-	// Creating pristine DB copy to speed up FullTestDB
 	dbUrl := cfg.Database().URL()
 	userOnly := c.Bool("user-only")
 	if err := store.PrepareTestDB(s.Logger, dbUrl, userOnly); err != nil {

core/config/toml/types.go

@@ -390,8 +390,20 @@ type DatabaseSecrets struct {
 	AllowSimplePasswords *bool
 }
 
+func isTestDatabaseURL(dbURI url.URL) bool {
+	dbname := strings.TrimPrefix(dbURI.Path, "/")
+	if strings.Contains(dbname, "_test") {
+		return true
+	}
+	// pgtestdb template/instance databases (testdb_tpl_*_inst_*).
+	if strings.HasPrefix(dbname, "testdb_") {
+		return true
+	}
+	return false
+}
+
 func validateDBURL(dbURI url.URL) error {
-	if strings.Contains(dbURI.Redacted(), "_test") {
+	if isTestDatabaseURL(dbURI) {
 		return nil
 	}
 

core/config/toml/types_test.go

@@ -99,6 +99,8 @@ func Test_validateDBURL(t *testing.T) {
 		{"with user and password of insufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=shortpw&user=myuser", fmt.Sprintf("%s	%s\n", utils.ErrMsgHeader, "password is less than 16 characters long")},
 		{"with no user and password of sufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=thisisareallylongpassword", ""},
 		{"with user and password of sufficient length as params", "postgresql://foo.example.com:5432/chainlink?application_name=Test+Application&password=thisisareallylongpassword&user=myuser", ""},
+		{"pgtestdb instance with short password", "postgresql://pgtdbuser:short@localhost:5432/testdb_tpl_abc_inst_def?sslmode=disable", ""},
+		{"chainlink_test with short password", "postgresql://postgres:short@localhost:5432/chainlink_test?sslmode=disable", ""},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

core/internal/testutils/pgtest/bench_test.go

@@ -0,0 +1,35 @@
+package pgtest
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink/v2/core/config/env"
+	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
+)
+
+func requireBenchDB(b *testing.B) {
+	b.Helper()
+	testutils.SkipShortDB(b)
+	if string(env.DatabaseURL.Get()) == "" {
+		b.Skip("CL_DATABASE_URL required")
+	}
+}
+
+// BenchmarkNewSqlxDB measures txdb connection provisioning (open transaction + session
+// timeouts). Each iteration closes the connection explicitly so the cost reflects a
+// single test's DB setup, not accumulated transactions.
+//
+// Run: CL_DATABASE_URL='postgres://...' go test -bench=BenchmarkNewSqlxDB -benchmem ./core/internal/testutils/pgtest/
+func BenchmarkNewSqlxDB(b *testing.B) {
+	requireBenchDB(b)
+	b.ReportAllocs()
+
+	for b.Loop() {
+		db := NewSqlxDB(b)
+		_, err := db.Exec("SELECT 1")
+		require.NoError(b, err)
+		require.NoError(b, db.Close())
+	}
+}

core/scripts/setup_testdb.sh

@@ -29,9 +29,6 @@ ALTER DATABASE $database OWNER TO $username;
 GRANT ALL PRIVILEGES ON DATABASE "$database" TO "$username";
 ALTER USER $username CREATEDB;
 
--- Create a pristine database for testing
-SELECT 'CREATE DATABASE chainlink_test_pristine WITH OWNER $username;' 
-WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'chainlink_test_pristine')\gexec
 EOF
 
 # Print the SQL commands

core/services/chainlink/config.go

@@ -434,7 +434,7 @@ func (s *Secrets) SetFrom(f *Secrets) (err error) {
 
 func (s *Secrets) setDefaults() {
 	if nil == s.Database.AllowSimplePasswords {
-		s.Database.AllowSimplePasswords = new(bool)
+		s.Database.AllowSimplePasswords = new(false)
 	}
 }
 
@@ -509,8 +509,7 @@ func (s *Secrets) setEnv() error {
 		}
 	}
 	if env.DatabaseAllowSimplePasswords.IsTrue() {
-		s.Database.AllowSimplePasswords = new(bool)
-		*s.Database.AllowSimplePasswords = true
+		s.Database.AllowSimplePasswords = new(true)
 	}
 	if keystorePassword := env.PasswordKeystore.Get(); keystorePassword != "" {
 		s.Password.Keystore = &keystorePassword

core/store/database_ddl.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"regexp"
+
+	"github.com/lib/pq"
+)
+
+const (
+	dropDatabaseSQLFmt   = "DROP DATABASE IF EXISTS %s WITH (FORCE)"
+	createDatabaseSQLFmt = "CREATE DATABASE %s"
+)
+
+var postgresDBNamePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
+
+func quotePostgresDBName(name string) (string, error) {
+	if name == "" || len(name) > 63 || !postgresDBNamePattern.MatchString(name) {
+		return "", fmt.Errorf("invalid postgres database name: %q", name)
+	}
+	return pq.QuoteIdentifier(name), nil
+}
+
+func execDropDatabase(ctx context.Context, db *sql.DB, name string) error {
+	quoted, err := quotePostgresDBName(name)
+	if err != nil {
+		return err
+	}
+	// PostgreSQL does not support bound parameters for database identifiers.
+	//nolint:gosec // G701 -- name validated by quotePostgresDBName; identifier escaped with pq.QuoteIdentifier
+	_, err = db.ExecContext(ctx, fmt.Sprintf(dropDatabaseSQLFmt, quoted))
+	return err
+}
+
+func execCreateDatabase(ctx context.Context, db *sql.DB, name string) error {
+	quoted, err := quotePostgresDBName(name)
+	if err != nil {
+		return err
+	}
+	// PostgreSQL does not support bound parameters for database identifiers.
+	//nolint:gosec // G701 -- name validated by quotePostgresDBName; identifier escaped with pq.QuoteIdentifier
+	_, err = db.ExecContext(ctx, fmt.Sprintf(createDatabaseSQLFmt, quoted))
+	return err
+}

core/store/migrate/migrate.go

@@ -23,9 +23,9 @@ import (
 )
 
 //go:embed migrations/*.sql migrations/*.go
-var embedMigrations embed.FS
+var EmbedMigrations embed.FS
 
-const MIGRATIONS_DIR string = "migrations"
+const MigrationsDir string = "migrations"
 
 func NewProvider(ctx context.Context, db *sql.DB) (*goose.Provider, error) {
 	// Set this to override how Goose resolves the schema for the `goose_migrations` table.
@@ -56,7 +56,7 @@ func NewProvider(ctx context.Context, db *sql.DB) (*goose.Provider, error) {
 	logMigrations := os.Getenv("CL_LOG_SQL_MIGRATIONS")
 	verbose, _ := strconv.ParseBool(logMigrations)
 
-	fys, err := fs.Sub(embedMigrations, MIGRATIONS_DIR)
+	fys, err := fs.Sub(EmbedMigrations, MigrationsDir)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get sub filesystem for embedded migration dir: %w", err)
 	}
@@ -126,13 +126,13 @@ func ensureMigrated(ctx context.Context, db *sql.DB, p *goose.Provider, provider
 			if name == "1611847145" {
 				id = 1
 			} else {
-				idx := strings.Index(name, "_")
-				if idx < 0 {
+				before, _, ok := strings.Cut(name, "_")
+				if !ok {
 					// old migration we don't care about
 					continue
 				}
 
-				id, err = strconv.ParseInt(name[:idx], 10, 64)
+				id, err = strconv.ParseInt(before, 10, 64)
 				if err == nil && id <= 0 {
 					return pkgerrors.New("migration IDs must be greater than zero")
 				}

core/store/store.go

@@ -28,7 +28,6 @@ import (
 	cutils "github.com/smartcontractkit/chainlink-common/pkg/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	"github.com/smartcontractkit/chainlink/v2/core/store/migrate"
-	"github.com/smartcontractkit/chainlink/v2/internal/testdb"
 )
 
 //go:embed fixtures/fixtures.sql
@@ -42,10 +41,7 @@ func PrepareTestDB(lggr logger.Logger, dbURL url.URL, userOnly bool) error {
 		return err
 	}
 	defer db.Close()
-	templateDB := strings.Trim(dbURL.Path, "/")
-	if err = dropAndCreatePristineDB(db, templateDB); err != nil {
-		return err
-	}
+	// we no longer create chainlink_test_pristine since pgtestdb handles template caching
 
 	fixturePath := "../store/fixtures/fixtures.sql"
 	if userOnly {
@@ -136,31 +132,13 @@ func dropAndCreateDB(parsed url.URL, _ bool) (err error) {
 	// Second parameter kept for ResetDatabase API compatibility (preparetest --force).
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	// PostgreSQL does not support bound parameters for database names; pq.QuoteIdentifier is the supported escape.
-	_, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)") //nolint:gosec // G701 false positive: identifier from pq.QuoteIdentifier only
-	if err != nil {
-		return fmt.Errorf("unable to drop postgres database: %w", err)
-	}
-	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	_, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(dbname)) //nolint:gosec // G701 false positive: identifier from pq.QuoteIdentifier only
-	if err != nil {
-		return fmt.Errorf("unable to create postgres database: %w", err)
-	}
-	return nil
-}
-
-func dropAndCreatePristineDB(db *sqlx.DB, template string) (err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	_, err = db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(testdb.PristineDBName)+" WITH (FORCE)")
-	if err != nil {
+	// PostgreSQL does not support bound parameters for database names; quotePostgresDBName validates and escapes.
+	if err = execDropDatabase(ctx, db, dbname); err != nil {
 		return fmt.Errorf("unable to drop postgres database: %w", err)
 	}
 	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	_, err = db.ExecContext(ctx, "CREATE DATABASE "+pq.QuoteIdentifier(testdb.PristineDBName)+" WITH TEMPLATE "+pq.QuoteIdentifier(template)) //nolint:gosec // G701 false positive: identifiers from pq.QuoteIdentifier only
-	if err != nil {
+	if err = execCreateDatabase(ctx, db, dbname); err != nil {
 		return fmt.Errorf("unable to create postgres database: %w", err)
 	}
 	return nil
@@ -229,6 +207,7 @@ func checkSchema(dbURL url.URL, prevSchema string, restrictKey string) error {
 	}
 	return nil
 }
+
 func insertFixtures(dbURL url.URL, pathToFixtures string) (err error) {
 	db, err := sql.Open(pgcommon.DriverPostgres, dbURL.String())
 	if err != nil {
@@ -276,13 +255,14 @@ func dropDanglingTestDBs(lggr logger.Logger, db *sqlx.DB) (err error) {
 				errCh <- func() error {
 					ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 					defer cancel()
-					return cutils.JustError(db.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pq.QuoteIdentifier(dbname)+" WITH (FORCE)"))
+					return execDropDatabase(ctx, db.DB, dbname)
 				}()
 			}
 		}()
 	}
 	for _, dbname := range dbs {
-		if strings.HasPrefix(dbname, testdb.TestDBNamePrefix) && !strings.HasSuffix(dbname, "_pristine") {
+		if (strings.HasPrefix(dbname, "chainlink_test_") && !strings.HasSuffix(dbname, "_pristine")) ||
+			(strings.HasPrefix(dbname, "testdb_") && (!strings.HasPrefix(dbname, "testdb_tpl_") || strings.Contains(dbname, "_inst_"))) {
 			ch <- dbname
 		}
 	}

core/utils/testutils/heavyweight/bench_test.go

@@ -0,0 +1,63 @@
+package heavyweight
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink/v2/core/config/env"
+)
+
+func requireBenchDB(b *testing.B) {
+	b.Helper()
+	if testing.Short() {
+		b.Skip("skipping DB benchmark with -short")
+	}
+	if string(env.DatabaseURL.Get()) == "" {
+		b.Skip("CL_DATABASE_URL required; run via make test or set CL_DATABASE_URL")
+	}
+}
+
+// BenchmarkFullTestDBNoFixturesV2 measures isolated DB provisioning via
+// CreateOrReplace (DROP + CREATE DATABASE … WITH TEMPLATE chainlink_test_pristine).
+// Database drops are deferred to benchmark teardown via t.Cleanup; use a modest
+// benchtime to avoid accumulating hundreds of databases locally.
+//
+// Run: CL_DATABASE_URL='postgres://...' go test -bench=BenchmarkFullTestDB -benchtime=5x -benchmem ./core/utils/testutils/heavyweight/
+func BenchmarkFullTestDBNoFixturesV2(b *testing.B) {
+	requireBenchDB(b)
+
+	// Warm up template; cost excluded from timer.
+	_, db := FullTestDBNoFixturesV2(b, nil)
+	_, err := db.ExecContext(b.Context(), "SELECT 1")
+	require.NoError(b, err)
+	require.NoError(b, db.Close())
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		_, db := FullTestDBNoFixturesV2(b, nil)
+		_, err := db.ExecContext(b.Context(), "SELECT 1")
+		require.NoError(b, err)
+		require.NoError(b, db.Close())
+	}
+}
+
+// BenchmarkFullTestDBEmptyV2 measures empty DB provisioning (no migrations template).
+func BenchmarkFullTestDBEmptyV2(b *testing.B) {
+	requireBenchDB(b)
+	b.ReportAllocs()
+
+	// Warm up template; cost excluded from timer.
+	_, db := FullTestDBEmptyV2(b, nil)
+	_, err := db.ExecContext(b.Context(), "SELECT 1")
+	require.NoError(b, err)
+	require.NoError(b, db.Close())
+
+	for b.Loop() {
+		_, db := FullTestDBEmptyV2(b, nil)
+		_, err := db.ExecContext(b.Context(), "SELECT 1")
+		require.NoError(b, err)
+		require.NoError(b, db.Close())
+	}
+}