Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/reference/benchling-connection.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ connection = BenchlingConnection(

Whether your Benchling tenant has access to fieldsets.

- **registry_id: Optional[str] = None**

The API id of the registry to scope this connection to, for example `src_abc123`. Only needed when the tenant has more than one registry. When set, Liminal reads and writes only the entity schemas and dropdowns in that registry, and the results schemas owned by the registry's organization. Without it, Liminal works as before on single-registry tenants. On a tenant with several registries, Liminal raises an error rather than guess when a lookup matches objects in more than one registry, or when it would need to create a schema or dropdown.

!!! tip
For a tenant with several registries, give each registry its own Liminal directory (`env.py` and `versions/`), with one `BenchlingConnection` per tenant that sets that registry's `registry_id`. Each registry then keeps its own revision history in its own `liminal_remote` schema.

- **config_flags: TenantConfigFlags = TenantConfigFlags()**

Set of config flags that are configured on the tenant level, that defines specific features of the tenant. These can be updated on Benchling's end by contacting their support team.
Expand Down
4 changes: 4 additions & 0 deletions liminal/connection/benchling_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ class BenchlingConnection(BaseModel):
DEPRECATED: Liminal no longer uses Playwright to log into Benchling's internal API, so this is ignored. It will be removed in v6.
fieldsets: bool = False
Whether your Benchling tenant has access to fieldsets.
registry_id: str | None = None
The API id of the registry to scope this connection to, e.g. src_abc123. Only needed when the tenant has more than one registry.
When set, Liminal reads and writes only the schemas and dropdowns in this registry, and the result schemas owned by its organization.
config_flags: TenantConfigFlags = TenantConfigFlags()
Set of config flags that are configured on the tenant level. These can be updated on Benchling's end by contacting their support team.
"""
Expand All @@ -59,6 +62,7 @@ class BenchlingConnection(BaseModel):
internal_api_admin_password: str | None = None
playwright_data_dir: str | None = None
fieldsets: bool = False
registry_id: str | None = None
config_flags: TenantConfigFlags = TenantConfigFlags()

@model_validator(mode="before")
Expand Down
98 changes: 89 additions & 9 deletions liminal/connection/benchling_service.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import logging
import warnings
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, NamedTuple

from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2
from benchling_sdk.benchling import Benchling, BenchlingApiClientDecorator
from benchling_sdk.errors import BenchlingError
from benchling_sdk.helpers.retry_helpers import RetryStrategy
from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
Expand All @@ -28,6 +29,17 @@
REMOTE_REVISION_ID_FIELD_WH_NAME = "revision_id"


class RegistryRef(NamedTuple):
"""The registry a connection writes to, and the organization that owns it."""

id: str
owner_id: str


class AmbiguousRegistryError(ValueError):
"""A lookup matched objects that may sit in different registries, and the connection doesn't name one."""


class BenchlingService(Benchling):
"""
Class that creates a connection object that can be used to connect to Benchling's API and/or Postgres warehouse.
Expand Down Expand Up @@ -64,6 +76,8 @@ def __init__(
)
self.connection = connection
self._session: Session | None = None
self._registry: RegistryRef | None = None
self._registry_count: int | None = None
self.use_api = use_api
self.benchling_tenant = connection.tenant_name
if use_api:
Expand Down Expand Up @@ -103,17 +117,83 @@ def session(self) -> Session:

@property
def registry_id(self) -> str:
# This assumes there is only one registry (which has always been the case at DynoTx)
registries = self.registry.registries()
assert len(registries) == 1
return registries[0].id
"""The id of the registry this connection writes to. Raises if it can't be determined."""
return self._get_registry().id

@property
def organization_id(self) -> str:
# This assumes there is only one registry (which has always been the case at DynoTx)
registries = self.registry.registries()
assert len(registries) == 1
return registries[0].owner.id
"""The id of the organization that owns this connection's registry. Raises if it can't be determined."""
return self._get_registry().owner_id

@property
def _is_registry_scoped(self) -> bool:
"""Whether this connection is scoped to one registry through `registry_id`."""
return self.connection.registry_id is not None

@property
def _has_unresolved_registry(self) -> bool:
"""Whether the tenant has more than one registry and this connection doesn't name one.
Lookups by name can then match objects in several registries."""
return self._find_registry() is None

def _is_in_registry_scope(self, record: dict[str, Any]) -> bool:
"""Whether a v3 API record belongs to this connection's registry or its organization.
Always True when the connection isn't scoped to a registry."""
if not self._is_registry_scoped:
return True
registry_id = (record.get("registry") or {}).get("id")
if registry_id:
return registry_id == self.registry_id
organization_id = (record.get("organization") or {}).get("id")
if organization_id:
return organization_id == self.organization_id
return False

def _ensure_unambiguous(self, matches: list[Any], description: str) -> None:
"""Raises when a lookup matches more than one object on a tenant whose registry isn't set,
because the matches may sit in different registries and Liminal would have to guess."""
if len(matches) > 1 and self._has_unresolved_registry:
raise AmbiguousRegistryError(
f"Tenant {self.benchling_tenant}: found {len(matches)} {description}. "
"The tenant has more than one registry, so set registry_id on the BenchlingConnection to choose one."
)

def _find_registry(self) -> RegistryRef | None:
"""Resolves the connection's registry once. Returns None when the tenant has several registries
and the connection doesn't set registry_id."""
if self._registry_count is None:
if self.connection.registry_id is not None:
self._registry = self._get_registry_by_id(self.connection.registry_id)
self._registry_count = 1
else:
registries = self.registry.registries()
self._registry_count = len(registries)
if len(registries) == 1:
self._registry = RegistryRef(
registries[0].id, registries[0].owner.id
)
return self._registry

def _get_registry_by_id(self, registry_id: str) -> RegistryRef:
from liminal.utils import rate_limited_get_v3

try:
response = rate_limited_get_v3(self, f"/api/v3/registry/{registry_id}")
except BenchlingError as e:
raise ValueError(
f"Tenant {self.benchling_tenant}: registry {registry_id} not found ({e.status_code})."
) from e
registry = response.parsed
return RegistryRef(registry["id"], registry["owner"]["id"])

def _get_registry(self) -> RegistryRef:
registry = self._find_registry()
if registry is None:
raise ValueError(
f"Tenant {self.benchling_tenant} has {self._registry_count} registries. "
"Set registry_id on the BenchlingConnection to choose one."
)
return registry

def __enter__(self) -> Session:
self._session = self.get_session()
Expand Down
3 changes: 2 additions & 1 deletion liminal/dropdowns/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ def list_dropdowns(
benchling_service: BenchlingService, include_archived: bool = True
) -> list[dict[str, Any]]:
"""Fetch dropdowns from the v3 API."""
return list_all_items_v3(
dropdowns = list_all_items_v3(
benchling_service, f"{_DROPDOWN_API_PATH}/items", include_archived
)
return [d for d in dropdowns if benchling_service._is_in_registry_scope(d)]


def list_dropdown_options(
Expand Down
10 changes: 10 additions & 0 deletions liminal/dropdowns/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@ def execute(self, benchling_service: BenchlingService) -> dict[str, Dropdown]:
f"Dropdown {self.dropdown_name} is different in code versus Benchling."
)

def validate(self, benchling_service: BenchlingService) -> None:
if (
benchling_service._has_unresolved_registry
and not dropdown_exists_in_benchling(benchling_service, self.dropdown_name)
):
raise ValueError(
f"{self.__class__.__name__} {self.dropdown_name}: Tenant {benchling_service.benchling_tenant} has more than one registry. "
"Set registry_id on the BenchlingConnection to choose where to create this dropdown."
)

def describe_operation(self) -> str:
return f"{self.dropdown_name}: Creating dropdown with options {self.dropdown_options}."

Expand Down
33 changes: 25 additions & 8 deletions liminal/dropdowns/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,29 +30,40 @@ def get_benchling_dropdown_name_to_classname_map(
def get_benchling_dropdown_summaries(
benchling_service: BenchlingService,
) -> list[DropdownSummary]:
return [
summaries = [
dropdown
for sublist in benchling_service.dropdowns.list()
for dropdown in sublist
]
if not benchling_service._is_registry_scoped:
return summaries
# The v2 dropdown summary has no registry, so scope it by the ids the v3 list returns.
scoped_ids = {
d["id"] for d in list_dropdowns(benchling_service, include_archived=True)
}
return [d for d in summaries if d.id in scoped_ids]


def get_benchling_dropdown_summary_by_name(
benchling_service: BenchlingService, name: str
) -> DropdownSummary:
for dropdown in get_benchling_dropdown_summaries(benchling_service):
if dropdown.name == name:
return dropdown
matches = [
d for d in get_benchling_dropdown_summaries(benchling_service) if d.name == name
]
benchling_service._ensure_unambiguous(matches, f"dropdowns named {name}")
if matches:
return matches[0]
raise Exception(f"Dropdown {name} not found in given list.")


def get_benchling_dropdown_by_name(
benchling_service: BenchlingService, name: str
) -> Dropdown:
dropdown = None
for d in get_benchling_dropdown_summaries(benchling_service):
if d.name == name:
dropdown = d
matches = [
d for d in get_benchling_dropdown_summaries(benchling_service) if d.name == name
]
benchling_service._ensure_unambiguous(matches, f"dropdowns named {name}")
dropdown = matches[-1] if matches else None
if dropdown is None:
raise Exception(
f"Dropdown {name} not found in Benchling {benchling_service.benchling_tenant}."
Expand Down Expand Up @@ -96,6 +107,12 @@ def get_benchling_dropdowns_dict(
include_archived=include_archived,
dropdown_names_for_options=dropdown_names_for_options,
)
names = [dropdown["name"] for dropdown in dropdowns]
duplicate_names = sorted({n for n in names if names.count(n) > 1})
benchling_service._ensure_unambiguous(
[d for d in dropdowns if d["name"] in duplicate_names],
f"dropdowns sharing the names {duplicate_names}",
)
return {
dropdown["name"]: _convert_dropdown_from_v3_to_v2_sdk(
dropdown, include_archived
Expand Down
4 changes: 3 additions & 1 deletion liminal/entity_schemas/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def list_entity_schemas(
for future in as_completed(futures):
entity_schemas.extend(future.result())

return entity_schemas
return [s for s in entity_schemas if benchling_service._is_in_registry_scope(s)]


@lru_cache
Expand All @@ -61,6 +61,8 @@ def list_entity_schemas_cached(
s
for schemas in benchling_service.schemas.list_entity_schemas()
for s in schemas
if not benchling_service._is_registry_scoped
or s.registry_id == benchling_service.registry_id
]


Expand Down
11 changes: 4 additions & 7 deletions liminal/entity_schemas/entity_schema_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -531,14 +531,11 @@ def get_one(
) -> EntitySchemaModel:
if schemas_data is None:
schemas_data = list_entity_schemas(benchling_service, include_archived=True)
schema = next(
(
schema
for schema in schemas_data
if schema["systemName"] == wh_schema_name
),
None,
matches = [s for s in schemas_data if s["systemName"] == wh_schema_name]
benchling_service._ensure_unambiguous(
matches, f"entity schemas with warehouse name {wh_schema_name}"
)
schema = matches[0] if matches else None
if schema is None:
raise ValueError(
f"Schema {wh_schema_name} not found in Benchling {benchling_service.benchling_tenant}."
Expand Down
16 changes: 16 additions & 0 deletions liminal/entity_schemas/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,22 @@ def validate(self, benchling_service: BenchlingService) -> None:
f"{self.__class__.__name__} {self._validated_schema_properties.warehouse_name}: On field {field.warehouse_name}, unit {field.unit_name} not found in Benchling Unit Dictionary as a valid unit. Please check the field definition or your Unit Dictionary."
)

if benchling_service._has_unresolved_registry:
self._validate_registry_for_create(benchling_service)

def _validate_registry_for_create(
self, benchling_service: BenchlingService
) -> None:
"""Fails before any write when creating this schema would need a registry the connection doesn't name."""
wh_name = self._validated_schema_properties.warehouse_name
try:
EntitySchemaModel.get_one(benchling_service, wh_name)
except ValueError:
raise ValueError(
f"{self.__class__.__name__} {wh_name}: Tenant {benchling_service.benchling_tenant} has more than one registry. "
"Set registry_id on the BenchlingConnection to choose where to create this schema."
)

def _validate_create(self, benchling_service: BenchlingService) -> None:
all_schemas = list_entity_schemas(benchling_service, include_archived=True)
if self._validated_schema_properties.name in [
Expand Down
5 changes: 4 additions & 1 deletion liminal/results_schemas/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ def list_results_schemas(
benchling_service: BenchlingService,
) -> list[dict[str, Any]]:
"""Fetch result schemas from the v3 API."""
return list_all_items_v3(benchling_service, f"{_RESULT_SCHEMA_API_PATH}/items")
result_schemas = list_all_items_v3(
benchling_service, f"{_RESULT_SCHEMA_API_PATH}/items"
)
return [s for s in result_schemas if benchling_service._is_in_registry_scope(s)]


def list_result_schema_field_definitions(
Expand Down
12 changes: 4 additions & 8 deletions liminal/results_schemas/models/results_schema_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,11 @@ def get_one(
"""
if schemas_data is None:
schemas_data = cls.get_all_json(benchling_service)
schema = next(
(
schema
for schema in schemas_data
if schema["systemName"] == wh_schema_name
# and schema["registryId"] == benchling_service.registry_id
),
None,
matches = [s for s in schemas_data if s["systemName"] == wh_schema_name]
benchling_service._ensure_unambiguous(
matches, f"results schemas with warehouse name {wh_schema_name}"
)
schema = matches[0] if matches else None
if schema is None:
raise ValueError(
f"Schema {wh_schema_name} not found in Benchling {benchling_service.benchling_tenant}."
Expand Down
6 changes: 5 additions & 1 deletion liminal/results_schemas/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,5 +101,9 @@ def get_benchling_results_schemas(
benchling_service: BenchlingService,
) -> list[AssayResultSchema]:
return [
s for loe in benchling_service.schemas.list_assay_result_schemas() for s in loe
s
for loe in benchling_service.schemas.list_assay_result_schemas()
for s in loe
if not benchling_service._is_registry_scoped
or s.organization.id == benchling_service.organization_id
]
Loading