[DOP-39630] Hide secrets from settings repr

mddocs/docs/changelog/next_release/413.improvement.md

@@ -0,0 +1 @@
+Hide auth provider secrets and database password from logs.

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
   "asgi-correlation-id~=5.0.0",
   "uuid6~=2025.0.1",
   "celery~=5.6.0",
+  "devtools~=0.12.2",
   "psycopg2-binary~=2.9.11",
 ]
 dynamic = ["version"]

syncmaster/db/factory.py

@@ -3,15 +3,15 @@
 
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
-    async_engine_from_config,
     async_sessionmaker,
+    create_async_engine,
 )
 
 from syncmaster.server.settings import DatabaseSettings
 
 
 def create_session_factory(settings: DatabaseSettings) -> async_sessionmaker[AsyncSession]:
-    engine = async_engine_from_config(settings.model_dump(), prefix="")
+    engine = create_async_engine(url=str(settings.url), **settings.model_dump(exclude={"url"}))
     return async_sessionmaker(
         bind=engine,
         class_=AsyncSession,

syncmaster/db/migrations/env.py

@@ -34,7 +34,7 @@ class MigrationAppSettings(BaseSettings):
 if not config.get_main_option("sqlalchemy.url"):
     # read application settings only if sqlalchemy.url is not being passed via cli arguments
     # TODO: remove settings object creating during import
-    config.set_main_option("sqlalchemy.url", MigrationAppSettings().database.url)
+    config.set_main_option("sqlalchemy.url", str(MigrationAppSettings().database.url))
 
 target_metadata = (
     Base.metadata,

syncmaster/scheduler/__init__.py

@@ -8,6 +8,6 @@
 def celery_factory(settings: SchedulerAppSettings) -> Celery:
     return Celery(
         __name__,
-        broker=settings.broker.url,
+        broker=str(settings.broker.url),
         backend="db+" + settings.database.sync_url,
     )

syncmaster/scheduler/utils.py

@@ -15,7 +15,7 @@
 
 @contextlib.asynccontextmanager
 async def get_async_engine(settings: Settings) -> AsyncGenerator[AsyncEngine, None]:
-    engine = create_async_engine(url=settings.database.url)
+    engine = create_async_engine(url=str(settings.database.url), **settings.database.model_dump(exclude={"url"}))
     try:
         yield engine
     finally:

syncmaster/server/__init__.py

@@ -30,7 +30,7 @@
 def celery_factory(settings: Settings) -> Celery:
     return Celery(
         __name__,
-        broker=settings.broker.url,
+        broker=str(settings.broker.url),
         backend="db+" + settings.database.sync_url,
     )
 

syncmaster/server/providers/auth/dummy_provider.py

@@ -2,10 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
-from pprint import pformat
 from time import time
 from typing import Any
 
+from devtools import pformat
 from fastapi import FastAPI, Request
 
 from syncmaster.db.models import User

syncmaster/server/providers/auth/keycloak_provider.py

@@ -4,6 +4,7 @@
 import time
 from typing import Any, NoReturn
 
+from devtools import pformat
 from fastapi import FastAPI, Request
 from jwcrypto import jwk
 from jwcrypto.common import JWException
@@ -37,7 +38,7 @@ def __init__(self, settings: KeycloakAuthProviderSettings) -> None:
     @classmethod
     def setup(cls, app: FastAPI) -> FastAPI:
         settings = KeycloakAuthProviderSettings.model_validate(app.state.settings.auth.model_dump(exclude={"provider"}))
-        log.info("Using %s provider with settings:\n%s", cls.__name__, settings)
+        log.info("Using %s provider with settings:\n%s", cls.__name__, pformat(settings))
 
         app.state.auth_provider = cls(settings=settings)
         app.add_middleware(

syncmaster/server/providers/auth/oauth2_gateway_provider.py

@@ -3,6 +3,7 @@
 import logging
 from typing import Any
 
+from devtools import pformat
 from fastapi import FastAPI, Request
 from keycloak import KeycloakOpenID, KeycloakOperationError
 
@@ -32,7 +33,7 @@ def setup(cls, app: FastAPI) -> FastAPI:
         settings = OAuth2GatewayProviderSettings.model_validate(
             app.state.settings.auth.model_dump(exclude={"provider"}),
         )
-        log.info("Using %s provider with settings:\n%s", cls.__name__, settings)
+        log.info("Using %s provider with settings:\n%s", cls.__name__, pformat(settings))
 
         app.state.auth_provider = cls(settings=settings)
         return app

syncmaster/server/scripts/manage_superusers.py

@@ -119,7 +119,7 @@ async def main(args: argparse.Namespace, session: AsyncSession) -> None:
     settings = SuperuserAppSettings()
     setup_logging(settings.logging)
 
-    engine = create_async_engine(settings.database.url)
+    engine = create_async_engine(*settings.database.model_dump(mode="json"))
     SessionLocal = async_sessionmaker(autocommit=False, autoflush=False, bind=engine, class_=AsyncSession)
     parser = create_parser()
     args = parser.parse_args()

syncmaster/server/settings/auth/__init__.py

@@ -1,7 +1,12 @@
 # SPDX-FileCopyrightText: 2023-present MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 
-from pydantic import BaseModel, ConfigDict, Field, ImportString
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict, Field, ImportString, field_validator
+
+if TYPE_CHECKING:
+    from syncmaster.server.providers.auth.base_provider import AuthProvider
 
 
 class AuthSettings(BaseModel):
@@ -28,3 +33,17 @@ class AuthSettings(BaseModel):
     )
 
     model_config = ConfigDict(extra="allow")
+
+    @field_validator("provider", mode="after")
+    @classmethod
+    def _validate_provider(cls, value: type) -> "type[AuthProvider]":
+        from syncmaster.server.providers.auth.base_provider import AuthProvider  # noqa: PLC0415
+
+        if not issubclass(value, AuthProvider):
+            msg = f"Class {value} is not a subclass of {AuthProvider}"
+            raise TypeError(msg)
+        return value
+
+    # prevent leaking provider secrets
+    def __repr_args__(self):
+        return [("provider", self.provider)]

syncmaster/settings/broker.py

@@ -1,6 +1,26 @@
 # SPDX-FileCopyrightText: 2023-present MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
-from pydantic import BaseModel, ConfigDict, Field
+from typing import Annotated
+from urllib.parse import urlsplit
+
+from pydantic import AfterValidator, AnyUrl, BaseModel, ConfigDict, Field, UrlConstraints
+from sqlalchemy import make_url
+
+
+def validate_url(value: AnyUrl):
+    split = urlsplit(str(value))
+    if not split.username or not split.password:
+        msg = "RabbitMQ URL must contain username and password"
+        raise ValueError(msg)
+
+    return value
+
+
+RabbitMQURL = Annotated[
+    AnyUrl,
+    UrlConstraints(allowed_schemes=["amqp"], host_required=True, preserve_empty_path=False),
+    AfterValidator(validate_url),
+]
 
 
 class RabbitMQSettings(BaseModel):
@@ -20,10 +40,20 @@ class RabbitMQSettings(BaseModel):
     ```
     """
 
-    url: str = Field(
+    url: RabbitMQURL = Field(
         description=(
             "RabbitMQ connection URL.\n\nSee the [RabbitMQ documentation](https://www.rabbitmq.com/uri-spec.html) "
         ),
     )
 
     model_config = ConfigDict(extra="allow")
+
+    def __repr_args__(self):
+        safe_url = make_url(str(self.url)).render_as_string(
+            hide_password=True,
+        )
+        extra = super().__repr_args__()
+        return [
+            ("url", safe_url),
+            *[item for item in extra if item[0] != "url"],
+        ]

syncmaster/settings/database.py

@@ -1,9 +1,31 @@
 # SPDX-FileCopyrightText: 2023-present MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 import textwrap
-from urllib.parse import urlparse, urlunparse
+from typing import Annotated
+from urllib.parse import urlparse, urlsplit
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import AfterValidator, BaseModel, ConfigDict, Field, PostgresDsn, UrlConstraints
+from sqlalchemy import make_url
+
+
+def validate_url(value: PostgresDsn):
+    if not value.path or len(value.path) <= 1:
+        msg = "Database URL must contain database name"
+        raise ValueError(msg)
+
+    split = urlsplit(str(value))
+    if not split.username or not split.password:
+        msg = "Database URL must contain username and password"
+        raise ValueError(msg)
+
+    return value
+
+
+PostgresURL = Annotated[
+    PostgresDsn,
+    UrlConstraints(allowed_schemes=["postgresql+asyncpg"], host_required=True),
+    AfterValidator(validate_url),
+]
 
 
 class DatabaseSettings(BaseModel):
@@ -27,7 +49,7 @@ class DatabaseSettings(BaseModel):
     ```
     """
 
-    url: str = Field(
+    url: PostgresDsn = Field(
         description=textwrap.dedent(
             """
             Database connection URL.
@@ -43,10 +65,17 @@ class DatabaseSettings(BaseModel):
 
     @property
     def sync_url(self) -> str:
-        parsed_url = urlparse(self.url)
-        # replace '+asyncpg' with '+psycopg2' in the scheme - used by celery
-        scheme = parsed_url.scheme.replace("+asyncpg", "+psycopg2")
-        sync_parsed_url = parsed_url._replace(scheme=scheme)
-        return urlunparse(sync_parsed_url)
+        schema = urlparse(str(self.url)).scheme
+        return str(self.url).replace(schema, "postgresql+psycopg2")
 
     model_config = ConfigDict(extra="allow")
+
+    def __repr_args__(self):
+        safe_url = make_url(str(self.url)).render_as_string(
+            hide_password=True,
+        )
+        extra = super().__repr_args__()
+        return [
+            ("url", safe_url),
+            *[item for item in extra if item[0] != "url"],
+        ]

syncmaster/worker/__init__.py

@@ -9,7 +9,7 @@
 def celery_factory(settings: WorkerAppSettings) -> Celery:
     return Celery(
         __name__,
-        broker=settings.broker.url,
+        broker=str(settings.broker.url),
         backend="db+" + settings.database.sync_url,
         task_cls=WorkerTask,
         imports=[

tests/conftest.py

@@ -87,7 +87,7 @@ def test_settings():
 def alembic_config(settings: Settings) -> AlembicConfig:
     alembic_cfg = AlembicConfig(PROJECT_PATH / "syncmaster" / "db" / "alembic.ini")
     alembic_cfg.set_main_option("script_location", os.fspath(PROJECT_PATH / "syncmaster/db/migrations"))
-    alembic_cfg.set_main_option("sqlalchemy.url", settings.database.url)
+    alembic_cfg.set_main_option("sqlalchemy.url", str(settings.database.url))
     return alembic_cfg
 
 
@@ -98,7 +98,7 @@ async def async_engine(settings: Settings, alembic_config: AlembicConfig):
         await run_async_migrations(alembic_config, Base.metadata, "-1", "down")
 
     await run_async_migrations(alembic_config, Base.metadata, "head")
-    engine = create_async_engine(settings.database.url)
+    engine = create_async_engine(url=str(settings.database.url), **settings.database.model_dump(exclude={"url"}))
     yield engine
     await engine.dispose()
 

tests/test_unit/test_settings/test_scheduler_settings.py

@@ -26,7 +26,7 @@ def test_scheduler_settings_are_loaded_from_default_yaml_file(
         dedent(
             """\
             database:
-              url: "postgresql+asyncpg://user:password'#[value]@localhost:5432/syncmaster"
+              url: "postgresql+asyncpg://user:password@localhost:5432/syncmaster"
             broker:
               url: amqp://user:password@localhost:5672/
             encryption:
@@ -40,8 +40,8 @@ def test_scheduler_settings_are_loaded_from_default_yaml_file(
 
     settings = SchedulerAppSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://user:password'#[value]@localhost:5432/syncmaster"
-    assert settings.broker.url == "amqp://user:password@localhost:5672/"
+    assert str(settings.database.url) == "postgresql+asyncpg://user:password@localhost:5432/syncmaster"
+    assert str(settings.broker.url) == "amqp://user:password@localhost:5672/"
     assert settings.encryption.secret_key == "secret_key"
     assert settings.scheduler.transfer_fetching_timeout_seconds == 200
 
@@ -56,9 +56,9 @@ def test_scheduler_settings_yaml_file_overrides_environment(
         dedent(
             """\
             database:
-              url: postgresql+asyncpg://yaml@localhost:5432/syncmaster
+              url: postgresql+asyncpg://yaml:yaml@localhost:5432/syncmaster
             broker:
-              url: amqp://yaml@localhost:5672/
+              url: amqp://yaml:yaml@localhost:5672/
             encryption:
               secret_key: "yaml_secret_key"
             scheduler:
@@ -70,19 +70,19 @@ def test_scheduler_settings_yaml_file_overrides_environment(
     monkeypatch.setenv("SYNCMASTER_CONFIG_FILE", str(config_path))
     monkeypatch.setenv(
         "SYNCMASTER__DATABASE__URL",
-        "postgresql+asyncpg://env@localhost:5432/syncmaster",
+        "postgresql+asyncpg://env:env@localhost:5432/syncmaster",
     )
     monkeypatch.setenv(
         "SYNCMASTER__BROKER__URL",
-        "amqp://env@localhost:5672/",
+        "amqp://env:env@localhost:5672/",
     )
     monkeypatch.setenv("SYNCMASTER__ENCRYPTION__SECRET_KEY", "env_secret_key")
     monkeypatch.setenv("SYNCMASTER__SCHEDULER__transfer_fetching_timeout_seconds", "300")
 
     settings = SchedulerAppSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/syncmaster"
-    assert settings.broker.url == "amqp://yaml@localhost:5672/"
+    assert str(settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/syncmaster"
+    assert str(settings.broker.url) == "amqp://yaml:yaml@localhost:5672/"
     assert settings.encryption.secret_key == "yaml_secret_key"
     assert settings.scheduler.transfer_fetching_timeout_seconds == 200
 
@@ -95,18 +95,18 @@ def test_scheduler_settings_can_be_loaded_from_environment_without_yaml_file(
     monkeypatch.setenv("SYNCMASTER_CONFIG_FILE", str(tmp_path / "missing.yml"))
     monkeypatch.setenv(
         "SYNCMASTER__DATABASE__URL",
-        "postgresql+asyncpg://env@localhost:5432/syncmaster",
+        "postgresql+asyncpg://env:env@localhost:5432/syncmaster",
     )
     monkeypatch.setenv(
         "SYNCMASTER__BROKER__URL",
-        "amqp://env@localhost:5672/",
+        "amqp://env:env@localhost:5672/",
     )
     monkeypatch.setenv("SYNCMASTER__ENCRYPTION__SECRET_KEY", "env_secret_key")
     monkeypatch.setenv("SYNCMASTER__SCHEDULER__transfer_fetching_timeout_seconds", "300")
 
     settings = SchedulerAppSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://env@localhost:5432/syncmaster"
-    assert settings.broker.url == "amqp://env@localhost:5672/"
+    assert str(settings.database.url) == "postgresql+asyncpg://env:env@localhost:5432/syncmaster"
+    assert str(settings.broker.url) == "amqp://env:env@localhost:5672/"
     assert settings.encryption.secret_key == "env_secret_key"
     assert settings.scheduler.transfer_fetching_timeout_seconds == 300