From d5443e1c2d93f504ad3b3d4a13ab79e8a6b2d8c4 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:07:15 +0000 Subject: [PATCH 01/11] feat(evals): Add security-review vs wrdn-authz coverage matrix Run each wrdn-authz scenario under builtin security-review and under wrdn-authz loaded from a pinned getsentry/warden-skills ref, so the results show which bugs only the wrdn skill catches. Seed four scenarios from real getsentry/sentry PR findings that only wrdn-authz posted, and document how the matrix and the vulnerability corpus fit together. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/evals/INTERNAL.md | 39 + .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/core/endpoints/scim/utils.py | 274 +++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../organization_seer_workflow_configs.py | 627 +++++++ .../src/sentry/seer/workflows/workflow.py | 430 +++++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/src/sentry/auth/access.py | 1533 ++++++++++++++++ .../organization_member_team_details.py | 593 +++++++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/src/sentry/auth/access.py | 1537 +++++++++++++++++ .../core/endpoints/organization_projects.py | 483 ++++++ packages/evals/src/index.test.ts | 1 + packages/evals/src/security-matrix.eval.ts | 92 + .../sentry-scim-restriction-revert.json | 30 + ...kflow-put-drops-inaccessible-projects.json | 36 + ...rvice-account-all-projects-team-write.json | 36 + ...-service-account-org-projects-listing.json | 36 + 18 files changed, 6167 insertions(+) create mode 100644 packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py create mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py create mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py create mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py create mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py create mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py create mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py create mode 100644 packages/evals/src/security-matrix.eval.ts create mode 100644 packages/evals/wrdn-authz/sentry-scim-restriction-revert.json create mode 100644 packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json create mode 100644 packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json create mode 100644 packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json diff --git a/packages/evals/INTERNAL.md b/packages/evals/INTERNAL.md index 31999ee1e..4fb11b473 100644 --- a/packages/evals/INTERNAL.md +++ b/packages/evals/INTERNAL.md @@ -59,11 +59,50 @@ source finding is not actually a reachable bug under the target skill. - `packages/evals/src/code-review.eval.ts`: code-review correctness benchmark scenarios. - `packages/evals/src/security-review.eval.ts`: security-review benchmark scenarios. - `packages/evals/src/verify.eval.ts`: verifier-only scenarios. +- `packages/evals/wrdn-*/*.json`: one scenario per file for a remote + `getsentry/warden-skills` security skill. +- `packages/evals/src/security-matrix.eval.ts`: runs each `wrdn-*` scenario + under both builtin security-review and the owning `wrdn-*` skill. Eval names should read as `/`. Runtime and model belong to the suite configuration, not the case identity. Avoid category names that hide the real skill under test. +## Security Coverage + +Security accuracy has two layers. Neither replaces the other. + +**PR-fixture matrix (CI).** `security-matrix.eval.ts` answers one question: +does a `wrdn-*` skill catch bugs that security-review misses? Each scenario in +`packages/evals//` runs in two lanes, `security-matrix//security-review` +and `security-matrix//`. A case only the `wrdn-*` lane passes is +coverage that skill adds. A skill whose cases security-review also passes is +redundant on that evidence. + +- Remote skills load from the pinned `WARDEN_SKILLS_REMOTE` ref through + Warden's normal remote skill cache. Bump the SHA on purpose and rerun. +- Seed scenarios from real findings that only the `wrdn-*` skill posted on a + PR. Write `should_find` for the real bug even when a lane misses it today. +- These cases are expected to fail in the security-review lane. That lowers the + aggregate eval score by design; read the per-lane results, not the average. +- Roach records them like any other eval. Add `run-evals` to record new cases. + +**Vulnerability corpus (repeated benchmark).** The +[Sentry vulnerability corpus](../docs/src/content/docs/benchmarking/running.mdx) +is the regression bar for security-review model, effort, and harness changes. +It is not a CI eval: runs cost real money and are scored by hand. + +- One run is noise. Two identical Grok 4.5 runs agreed on only 32 of the 46 + entries either one found. Compare configurations on at least 3 runs each. +- Report per-entry hit rate across runs, not one found count. An entry found + 3 of 3 times that drops to 0 of 3 is a regression; a 1 of 3 entry moving is + not evidence. +- The corpus is mostly authorization bugs, with a few webhook, crypto, + injection, and secret-logging entries. It has close to nothing for DoS, data + exfiltration, or code execution, so `wrdn-dos-review`, `wrdn-data-exfil`, + and `wrdn-code-execution` need their own fixtures in the matrix before any + claim about their coverage holds. + ## Adding Full-Pipeline Evals 1. Add or scaffold a scenario JSON file under `packages/evals//`. diff --git a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py new file mode 100644 index 000000000..396c2cece --- /dev/null +++ b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import NoReturn, NotRequired, TypedDict + +from django.http import HttpRequest +from drf_spectacular.utils import extend_schema +from rest_framework import serializers +from rest_framework.exceptions import APIException, ParseError +from rest_framework.negotiation import BaseContentNegotiation +from rest_framework.parsers import BaseParser +from rest_framework.renderers import BaseRenderer +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView +from sentry_sdk import traces + +from sentry.api.api_owners import ApiOwner +from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission +from sentry.auth.services.auth import auth_service +from sentry.models.organization import Organization +from sentry.organizations.services.organization import RpcOrganization, RpcUserOrganizationContext + +from .constants import SCIM_400_INVALID_FILTER, SCIM_API_ERROR, SCIM_API_LIST + +SCIM_CONTENT_TYPES = ["application/json", "application/json+scim"] +ACCEPTED_FILTERED_KEYS = ["userName", "value", "displayName"] + + +class SCIMApiError(APIException): + def __init__(self, detail: str, status_code: int = 400, scim_type: str | None = None) -> None: + span = traces.get_current_span() + if span is not None: + span.set_attribute("http.status_code", status_code) + error_body = { + "schemas": [SCIM_API_ERROR], + "status": str(status_code), + "detail": detail, + } + if scim_type is not None: + error_body["scimType"] = scim_type + super().__init__(error_body) + self.status_code = status_code + + +class SCIMFilterError(ValueError): + pass + + +class SCIMClientNegotiation(BaseContentNegotiation): + # SCIM uses the content type "application/json+scim" + # which is just json for our purposes. + def select_parser(self, request: Request, parsers: Iterable[BaseParser]) -> BaseParser | None: + """ + Select the first parser in the `.parser_classes` list. + """ + for parser in parsers: + if parser.media_type in SCIM_CONTENT_TYPES: + return parser + return None + + def select_renderer( + self, request: Request, renderers: Iterable[BaseRenderer], format_suffix: str | None = None + ) -> tuple[BaseRenderer, str] | None: + """ + Select the first renderer in the `.renderer_classes` list. + """ + for renderer in renderers: + if renderer.media_type in SCIM_CONTENT_TYPES: + return (renderer, renderer.media_type) + return None + + +class SCIMQueryParamSerializer(serializers.Serializer): + # SCIM API parameters are standardized camelCase by RFC7644. + # We convert them to snake_case using the source field + + startIndex = serializers.IntegerField( + min_value=1, + required=False, + default=1, + source="start_index", + help_text="SCIM 1-offset based index for pagination.", + ) + count = serializers.IntegerField( + min_value=0, + required=False, + default=100, + help_text="The maximum number of results the query should return, maximum of 100.", + ) + filter = serializers.CharField( + required=False, + default=None, + help_text="A SCIM filter expression. The only operator currently supported is `eq`.", + ) + excludedAttributes = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + source="excluded_attributes", + help_text="Fields that should be left off of return values. Right now the only supported field for this query is members.", + ) + + def validate_filter(self, filter: str) -> str | int | None: + try: + return parse_filter_conditions(filter) + except SCIMFilterError: + raise serializers.ValidationError("invalid filter") + + +class SCIMQueryParams(TypedDict): + start_index: int + count: int + filter: NotRequired[str | None] + excluded_attributes: NotRequired[list[str]] + + +class OrganizationSCIMPermission(OrganizationPermission): + def has_object_permission( + self, + request: Request, + view: APIView, + organization: Organization | RpcOrganization | RpcUserOrganizationContext, + ) -> bool: + result = super().has_object_permission(request, view, organization) + # The scim endpoints should only be used in conjunction with a SAML2 integration + if not result: + return result + if isinstance(organization, RpcUserOrganizationContext): + organization = organization.organization + provider = auth_service.get_auth_provider(organization_id=organization.id) + return provider is not None and provider.flags.scim_enabled + + +class OrganizationSCIMMemberPermission(OrganizationSCIMPermission): + scope_map = { + "GET": ["member:read", "member:write", "member:admin"], + "POST": ["member:write", "member:admin"], + "PATCH": ["member:write", "member:admin"], + "PUT": ["member:write", "member:admin"], + "DELETE": ["member:admin"], + } + + +class OrganizationSCIMTeamPermission(OrganizationSCIMPermission): + scope_map = { + "GET": ["team:read", "team:write", "team:admin"], + "POST": ["team:write", "team:admin"], + "PATCH": ["team:write", "team:admin"], + "DELETE": ["team:admin"], + } + + +class SCIMListBaseResponse(TypedDict): + schemas: list[str] + totalResults: int + startIndex: int + itemsPerPage: int + + +class SCIMMetaResponse(TypedDict): + resourceType: str + location: str + + +@extend_schema(tags=["SCIM"]) +class SCIMEndpoint(OrganizationEndpoint): + owner = ApiOwner.FOUNDATIONS + requires_user = True + content_negotiation_class = SCIMClientNegotiation + cursor_name = "startIndex" + + def add_cursor_headers( + self, request: Request, response: Response, cursor_result: object + ) -> None: + pass + + def list_api_format( + self, results: Sequence[object], total_results: int, start_index: int + ) -> dict[str, object]: + return { + "schemas": [SCIM_API_LIST], + "totalResults": total_results, # TODO: audit perf of queryset.count() + "startIndex": start_index, + "itemsPerPage": len(results), # what's max? + "Resources": results, + } + + def get_query_parameters(self, request: Request) -> SCIMQueryParams: + serializer = SCIMQueryParamSerializer(data=request.GET) + if not serializer.is_valid(): + if "filter" in serializer.errors: + # invalid filter needs to return a specific formatted + # error response + raise ParseError(detail=SCIM_400_INVALID_FILTER) + raise ParseError(serializer.errors) + + return serializer.validated_data + + +class SCIMDiscoveryEndpoint(SCIMEndpoint): + """Base for the RFC 7644 §4 discovery endpoints (ServiceProviderConfig, + ResourceTypes, Schemas). Discovery resources are static and GET-only: + query parameters are ignored rather than validated, a supplied filter is + rejected, and other HTTP methods get SCIM-format 405 bodies. + """ + + permission_classes = (OrganizationSCIMMemberPermission,) + + def reject_filter_param(self, request: Request) -> None: + # RFC 7644 §4: a filter on a discovery endpoint SHOULD be rejected + # with 403 so clients can't assume its conditions were applied. + # Handlers must call this first, rather than us hooking initial(): + # initial() runs before convert_args resolves the org, and the org + # access + scim_enabled checks must fail before the filter check. + if "filter" in request.GET: + raise SCIMApiError( + detail="Filtering is not supported on this endpoint.", status_code=403 + ) + + def http_method_not_allowed( + self, request: HttpRequest, *args: object, **kwargs: object + ) -> NoReturn: + raise SCIMApiError(detail=f'Method "{request.method}" not allowed.', status_code=405) + + +def parse_filter_conditions(raw_filters: str | None) -> str | int | None: + """ + this function parses a scim filter, see: + https://datatracker.ietf.org/doc/html/rfc7644#section-3.4.2.2 + + right now the only subset of that filtering we support is the simple "eq" + operator. the input would look like so: + userName eq "test.user@okta.local" + + We may want to support further SCIM grammar for other IDPs and may use + a package to replace this functionality. + """ + # TODO: support "and" operator + # TODO: support email querying/filtering + # TODO: graceful error handling when unsupported operators are used + + if raw_filters is None: + return None + + try: + conditions = raw_filters.split(",") + except ValueError: + raise SCIMFilterError + + # we don't support multiple filters right now. + if len(conditions) > 1: + raise SCIMFilterError + + condition = conditions[0] + try: + [key, value] = condition.split(" eq ") + except ValueError: + raise SCIMFilterError + + if not key or not value: + raise SCIMFilterError + key = key.strip() + value = value.strip() + + # remove encasing quotes around the value + value = value[1:-1] + + if key not in ACCEPTED_FILTERED_KEYS: + raise SCIMFilterError + if key == "value": + return int(value) + + return value diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py new file mode 100644 index 000000000..4f6ae12a5 --- /dev/null +++ b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py @@ -0,0 +1,627 @@ +import logging +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from functools import partial +from typing import Any, NotRequired, TypedDict + +from django.db.models import OuterRef, Subquery +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import serializers +from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError +from rest_framework.request import Request +from rest_framework.response import Response +from sentry_conventions.attributes import ATTRIBUTE_NAMES + +from sentry import features +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import cell_silo_endpoint +from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission +from sentry.api.paginator import OffsetPaginator +from sentry.api.serializers.rest_framework.base import convert_dict_key_case, snake_to_camel_case +from sentry.api.utils import to_valid_int_id, to_valid_int_id_list +from sentry.apidocs.parameters import CursorQueryParam, GlobalParams +from sentry.apidocs.utils import inline_sentry_response_serializer +from sentry.constants import ALL_ACCESS_PROJECT_ID, ALL_ACCESS_PROJECTS +from sentry.models.organization import Organization +from sentry.ratelimits.config import RateLimitConfig +from sentry.seer.models.workflow import ( + SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, + SeerWorkflowConfig, + SeerWorkflowConfigExtras, + SeerWorkflowConfigProject, + SeerWorkflowFilters, + SeerWorkflowRun, + SeerWorkflowSchedule, + SeerWorkflowStrategy, +) +from sentry.seer.workflows.breached_metrics import ( + BreachedMetricWorkflow, + get_config_monitors_bulk, + get_monitor_project_ids, +) +from sentry.seer.workflows.registry import get_seer_workflow +from sentry.seer.workflows.workflow import ( + AllProjects, + SeerWorkflow, + SelectedProjects, + WorkflowUpdateError, +) +from sentry.types.ratelimit import RateLimit, RateLimitCategory + +logger = logging.getLogger(__name__) + + +class SeerWorkflowConfigResponse(TypedDict): + id: str + strategy: str + enabled: bool + schedule: str + version: int + projects: list[int] + monitorIds: list[int] + environments: list[str] + filters: dict[str, Any] + outputPolicy: dict[str, Any] + seededDefault: str | None + dateAdded: datetime + dateUpdated: datetime + dateLastRun: NotRequired[datetime] + lastRunStatus: NotRequired[str | None] + name: NotRequired[str] + + +class SeerWorkflowConfigsResponse(TypedDict): + configs: list[SeerWorkflowConfigResponse] + + +class WorkflowConfigListErrorResponse(TypedDict): + detail: dict[str, str] + + +class SeerWorkflowConfigPermission(OrganizationPermission): + # POST and PUT are further limited to the projects the caller can access. + scope_map = { + "GET": ["org:read", "org:write", "org:admin"], + "POST": ["project:read", "project:write", "project:admin"], + "PUT": ["project:read", "project:write", "project:admin"], + } + + +class WorkflowConfigCreateSerializer(serializers.Serializer): + strategy = serializers.ChoiceField( + choices=sorted(s.value for s in SEER_CONFIGURABLE_WORKFLOW_STRATEGIES) + ) + enabled = serializers.BooleanField(default=False) + schedule = serializers.ChoiceField(choices=SeerWorkflowSchedule.choices, required=False) + environments = serializers.ListField( + child=serializers.CharField(max_length=64), default=list, allow_empty=True + ) + # [-1] selects all projects; omitted means no projects. + projectIds = serializers.ListField( + child=serializers.IntegerField(min_value=ALL_ACCESS_PROJECT_ID), + required=False, + allow_empty=True, + ) + name = serializers.CharField(default=None, allow_null=True, max_length=64) + # Metric automations only; their projects come from the connected monitors. + monitorIds = serializers.ListField( + child=serializers.IntegerField(min_value=1), + required=False, + allow_empty=True, + ) + + def validate_projectIds(self, value: list[int]) -> list[int]: + if ALL_ACCESS_PROJECT_ID in value and value != [ALL_ACCESS_PROJECT_ID]: + raise serializers.ValidationError("[-1] cannot be combined with project IDs.") + return value + + def validate(self, data: dict[str, Any]) -> dict[str, Any]: + if data["strategy"] == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + if "projectIds" in data: + raise serializers.ValidationError( + { + "projectIds": "Metric automation projects are selected by connecting monitors." + } + ) + if "schedule" in data: + raise serializers.ValidationError( + { + "schedule": "Metric automations run when a monitor breaches, not on a schedule." + } + ) + elif "monitorIds" in data: + raise serializers.ValidationError( + {"monitorIds": "Only metric automations support monitor selection."} + ) + return data + + +class WorkflowConfigUpdateSerializer(serializers.Serializer): + projectIds = serializers.ListField( + child=serializers.IntegerField(min_value=ALL_ACCESS_PROJECT_ID), + required=False, + allow_empty=True, + ) + + monitorIds = serializers.ListField( + child=serializers.IntegerField(min_value=1), + required=False, + allow_empty=True, + ) + + def validate_projectIds(self, value: list[int]) -> list[int]: + if ALL_ACCESS_PROJECT_ID in value and value != [ALL_ACCESS_PROJECT_ID]: + raise serializers.ValidationError("[-1] cannot be combined with project IDs.") + return value + + def validate(self, data: dict[str, Any]) -> dict[str, Any]: + if ("projectIds" in data) == ("monitorIds" in data): + raise serializers.ValidationError("Send exactly one of projectIds or monitorIds.") + return data + + +@dataclass(frozen=True) +class WorkflowConfigLastRun: + config_id: int + last_run_at: datetime | None + last_run_status: str | None + + +def _check_project_access( + project_ids: Sequence[int], + *, + request: Request, + organization: Organization, + endpoint: OrganizationEndpoint, +) -> str | None: + """ + Check the caller can read every project in a list. Returns a user-facing denial + detail, or None when access is allowed. + + - Every explicit project in ``projectIds`` must be active in the organization, + and the caller must have read access to it. + - Requesting all projects with ``[-1]`` requires global access. + """ + if set(project_ids) == ALL_ACCESS_PROJECTS: + if not request.access.has_global_access: + return "Requesting all projects requires global access" + elif project_ids: + # Raises PermissionDenied unless every project is active and accessible. + requested = endpoint.get_projects(request, organization, project_ids=set(project_ids)) + for project in requested: + if not request.access.has_project_scope(project, "project:read"): + # TODO(aliu): unreachable unless the endpoint permission class is looser + return f"You do not have access to project {project.id}" + + return None + + +@cell_silo_endpoint +@extend_schema(tags=["Seer"]) +class OrganizationSeerWorkflowConfigsEndpoint(OrganizationEndpoint): + owner = ApiOwner.ML_AI + publish_status = {"GET": ApiPublishStatus.PRIVATE, "POST": ApiPublishStatus.PRIVATE} + permission_classes = (SeerWorkflowConfigPermission,) + enforce_rate_limit = True + rate_limits = RateLimitConfig( + limit_overrides={ + "POST": { + RateLimitCategory.USER: RateLimit(limit=5, window=60), + RateLimitCategory.ORGANIZATION: RateLimit(limit=20, window=60 * 60), + }, + } + ) + + @extend_schema( + operation_id="List Seer automation workflows", + parameters=[ + GlobalParams.ORG_ID_OR_SLUG, + CursorQueryParam, + OpenApiParameter( + name="id", + type=int, + location=OpenApiParameter.QUERY, + many=True, + explode=True, + description="Filter by config IDs. Repeat the parameter for multiple IDs.", + ), + ], + responses={ + 200: inline_sentry_response_serializer( + "SeerWorkflowConfigs", SeerWorkflowConfigsResponse + ), + 400: inline_sentry_response_serializer( + "SeerWorkflowConfigListError", WorkflowConfigListErrorResponse + ), + }, + ) + def get(self, request: Request, organization: Organization) -> Response: + """List saved Seer automation workflows. + + The ``configs`` list contains the current page; pagination links are in headers. + Filter by config IDs with ``?id=123&id=456``; omit ``id`` to list all configs. + Invalid IDs return 400; no visible matches return ``{"configs": []}``. + ``projects`` is ``[-1]`` for all projects, ``[]`` for no projects, or a list + of selected active project IDs accessible to the caller. An empty + ``environments`` list means all environments; otherwise it contains names. + ``filters`` contains strategy-specific filters. ``seededDefault`` records + the seeded strategy even after edits. + ``outputPolicy`` is the saved, opaque policy document. + ``name`` is the user-supplied name, omitted if unset. + ``dateLastRun`` is the latest visible run's creation time and ``lastRunStatus`` + is its status (null for older runs that predate status tracking). Both are + omitted if no runs are visible under the runs endpoint's project-scope rules. + """ + if not features.has("organizations:seer-automations", organization, actor=request.user): + raise NotFound + + try: + config_ids = to_valid_int_id_list("id", request.query_params.getlist("id")) + except ValidationError as error: + return Response({"detail": error.detail}, status=400) + + projects = self.get_projects(request, organization, include_all_accessible=True) + configs = SeerWorkflowConfig.objects.filter( + organization=organization, strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES + ) + if config_ids: + configs = configs.filter(id__in=config_ids) + return self.paginate( + request=request, + queryset=configs, + order_by="id", + paginator_cls=OffsetPaginator, + on_results=partial( + serialize_workflow_configs, + accessible_project_ids=[p.id for p in projects], + has_global_access=request.access.has_global_access, + ), + ) + + @extend_schema( + operation_id="Create a Seer automation workflow", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request=WorkflowConfigCreateSerializer, + responses={ + 201: inline_sentry_response_serializer( + "SeerWorkflowConfig", SeerWorkflowConfigResponse + ), + }, + ) + def post(self, request: Request, organization: Organization) -> Response: + """Create a Seer automation workflow. + + Only ``strategy`` is required. ``enabled`` defaults to false, ``schedule`` to + daily, and ``environments`` to all. ``name`` is optional. Omitting + ``projectIds`` selects no projects. Use ``[-1]`` for all projects. + + Metric automations (``breached_metric_investigation``) take ``monitorIds`` + instead of ``projectIds``; their projects come from the connected monitors, and + the caller needs read access to each monitor's project. + + The number of automations per organization is capped, and creation is rate + limited. Failures past request validation, such as reaching the cap, return an + empty 400. Returns the new config in the same shape as an item in ``configs``. + """ + if not features.has("organizations:seer-automations", organization, actor=request.user): + raise NotFound + + serializer = WorkflowConfigCreateSerializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=400) + data = serializer.validated_data + strategy = SeerWorkflowStrategy(data["strategy"]) + + project_ids: list[int] = data.get("projectIds") or [] + monitor_ids: list[int] = data.get("monitorIds") or [] + if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + denied = _check_project_access( + get_monitor_project_ids(monitor_ids, organization), + request=request, + organization=organization, + endpoint=self, + ) + else: + denied = _check_project_access( + project_ids, request=request, organization=organization, endpoint=self + ) + if denied is not None: + raise PermissionDenied(detail=denied) + + try: + workflow: SeerWorkflow + if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + workflow = BreachedMetricWorkflow.create_with_monitors( + organization.id, + enabled=data["enabled"], + environments=data["environments"], + monitor_ids=monitor_ids, + name=data.get("name"), + ) + else: + workflow = SeerWorkflow.create( + organization_id=organization.id, + strategy=strategy, + enabled=data["enabled"], + schedule=SeerWorkflowSchedule(data.get("schedule", SeerWorkflowSchedule.DAILY)), + environments=data["environments"], + project_ids=project_ids, + name=data.get("name"), + ) + except ValueError as e: + logger.exception("bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400}) + if isinstance(e, WorkflowUpdateError): + return Response({"detail": str(e)}, status=400) + return Response(status=400) + + projects = self.get_projects(request, organization, include_all_accessible=True) + result = serialize_workflow_configs( + [workflow.get_config()], + accessible_project_ids=[p.id for p in projects], + has_global_access=request.access.has_global_access, + ) + return Response(result["configs"][0], status=201) + + +@cell_silo_endpoint +@extend_schema(tags=["Seer"]) +class OrganizationSeerWorkflowConfigDetailsEndpoint(OrganizationEndpoint): + owner = ApiOwner.ML_AI + publish_status = {"GET": ApiPublishStatus.PRIVATE, "PUT": ApiPublishStatus.PRIVATE} + permission_classes = (SeerWorkflowConfigPermission,) + + def serialize( + self, request: Request, organization: Organization, config: SeerWorkflowConfig + ) -> SeerWorkflowConfigResponse: + projects = self.get_projects(request, organization, include_all_accessible=True) + return serialize_workflow_configs( + [config], + accessible_project_ids=[p.id for p in projects], + has_global_access=request.access.has_global_access, + )["configs"][0] + + @extend_schema( + operation_id="Retrieve a Seer automation workflow", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + responses={ + 200: inline_sentry_response_serializer( + "SeerWorkflowConfig", SeerWorkflowConfigResponse + ), + }, + ) + def get(self, request: Request, organization: Organization, config_id: str) -> Response: + """Retrieve a saved Seer automation workflow by ID. + + The response has the same shape as an item in the list endpoint's ``configs``. + """ + if not features.has("organizations:seer-automations", organization, actor=request.user): + raise NotFound + + try: + config = SeerWorkflowConfig.objects.get( + id=to_valid_int_id("config_id", config_id), + organization=organization, + strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, + ) + except (ValueError, SeerWorkflowConfig.DoesNotExist) as e: + raise NotFound from e + + return Response(self.serialize(request, organization, config)) + + @extend_schema( + operation_id="Update a Seer automation workflow", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request=WorkflowConfigUpdateSerializer, + responses={ + 200: inline_sentry_response_serializer( + "SeerWorkflowConfig", SeerWorkflowConfigResponse + ), + }, + ) + def put(self, request: Request, organization: Organization, config_id: str) -> Response: + """Update a saved Seer automation workflow. + + Only the fields sent are changed. ``projectIds`` replaces the projects the + caller can access; selected projects the caller can't access are kept. + + - The same project-access rules apply here AND to the associated projects of ``monitorIds``. + - Changing the projects of a config that covers all projects requires global + access. + - ``monitorIds`` replaces the monitors of a metric automation and is rejected + for other strategies. + + Failures past request validation return an empty 400. Returns the updated config + in the same shape as an item in the list endpoint's ``configs``. + """ + if not features.has("organizations:seer-automations", organization, actor=request.user): + raise NotFound + + serializer = WorkflowConfigUpdateSerializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=400) + + try: + config = SeerWorkflowConfig.objects.get( + id=to_valid_int_id("config_id", config_id), + organization=organization, + strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, + ) + except (ValueError, SeerWorkflowConfig.DoesNotExist) as e: + raise NotFound from e + workflow = get_seer_workflow(config) + + # Validate and update projects + project_ids: list[int] | None = serializer.validated_data.get("projectIds") + all_projects = project_ids == [ALL_ACCESS_PROJECT_ID] + if project_ids is not None: + denied = _check_project_access( + project_ids, + request=request, + organization=organization, + endpoint=self, + ) + if denied is not None: + raise PermissionDenied(detail=denied) + + try: + if all_projects: + workflow.set_project_selection(AllProjects()) + else: + workflow.set_project_selection(SelectedProjects(project_ids=tuple(project_ids))) + except ValueError as e: + logger.exception( + "bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400} + ) + if isinstance(e, WorkflowUpdateError): + return Response({"detail": str(e)}, status=400) + return Response(status=400) + config = workflow.get_config() + + # Validate and update monitors + monitor_ids: list[int] | None = serializer.validated_data.get("monitorIds") + if monitor_ids is not None: + if not isinstance(workflow, BreachedMetricWorkflow): + return Response( + {"detail": "Only metric automations support monitor selection."}, status=400 + ) + denied = _check_project_access( + get_monitor_project_ids(monitor_ids, organization), + request=request, + organization=organization, + endpoint=self, + ) + if denied is not None: + raise PermissionDenied(detail=denied) + try: + workflow.set_monitor_selection(monitor_ids) + except ValueError as e: + logger.exception( + "bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400} + ) + if isinstance(e, WorkflowUpdateError): + return Response({"detail": str(e)}, status=400) + return Response(status=400) + config = workflow.get_config() + + return Response(self.serialize(request, organization, config)) + + +def serialize_workflow_configs( + configs: Sequence[SeerWorkflowConfig], + *, + accessible_project_ids: Sequence[int], + has_global_access: bool, +) -> SeerWorkflowConfigsResponse: + selected_projects = get_selected_project_ids( + configs, accessible_project_ids=accessible_project_ids + ) + connected_monitors = get_config_monitors_bulk( + configs, accessible_project_ids=accessible_project_ids + ) + last_runs = get_config_last_runs( + configs, + accessible_project_ids=accessible_project_ids, + has_global_access=has_global_access, + ) + last_runs_by_config_id = {run.config_id: run for run in last_runs} + return { + "configs": [ + serialize_workflow_config( + config, + selected_project_ids=selected_projects[config.id], + connected_monitors=connected_monitors.get(config.id, []), + last_run=last_runs_by_config_id.get(config.id), + ) + for config in configs + ] + } + + +def get_selected_project_ids( + configs: Sequence[SeerWorkflowConfig], *, accessible_project_ids: Sequence[int] +) -> defaultdict[int, list[int]]: + selected_projects: defaultdict[int, list[int]] = defaultdict(list) + links = SeerWorkflowConfigProject.objects.filter( + config__in=configs, project_id__in=accessible_project_ids + ).order_by("project_id") + for config_id, project_id in links.values_list("config_id", "project_id"): + selected_projects[config_id].append(project_id) + return selected_projects + + +def get_config_last_runs( + configs: Sequence[SeerWorkflowConfig], + *, + accessible_project_ids: Sequence[int], + has_global_access: bool, +) -> list[WorkflowConfigLastRun]: + runs = SeerWorkflowRun.objects.filter( + workflow_config_id=OuterRef("id"), organization_id=OuterRef("organization_id") + ) + if not has_global_access: + runs = runs.filter(extras__snapshot__project_ids__contained_by=list(accessible_project_ids)) + latest_run = runs.order_by("-date_added", "-id") + last_runs = ( + SeerWorkflowConfig.objects.filter(id__in=[config.id for config in configs]) + .annotate( + last_run_at=Subquery(latest_run.values("date_added")[:1]), + last_run_status=Subquery(latest_run.values("status")[:1]), + ) + .values_list("id", "last_run_at", "last_run_status") + ) + return [ + WorkflowConfigLastRun( + config_id=config_id, last_run_at=last_run_at, last_run_status=last_run_status + ) + for config_id, last_run_at, last_run_status in last_runs + ] + + +def serialize_workflow_config( + config: SeerWorkflowConfig, + *, + selected_project_ids: list[int], + connected_monitors: list[tuple[int, int]], + last_run: WorkflowConfigLastRun | None, +) -> SeerWorkflowConfigResponse: + filters = SeerWorkflowFilters.parse_obj(config.filters) + monitor_ids = [monitor_id for monitor_id, _ in connected_monitors] + if config.strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + # Metric automation projects come from their connected monitors. + project_ids = sorted({project_id for _, project_id in connected_monitors}) + elif filters.all_projects: + project_ids = [ALL_ACCESS_PROJECT_ID] + else: + project_ids = selected_project_ids + result: SeerWorkflowConfigResponse = { + "id": str(config.id), + "strategy": config.strategy, + "enabled": config.enabled, + "schedule": config.schedule, + "version": config.version, + "projects": project_ids, + "monitorIds": monitor_ids, + "environments": filters.environments, + "filters": convert_dict_key_case( + { + key: value + for key, value in config.filters.items() + if key not in {"all_projects", "environments"} + }, + snake_to_camel_case, + ), + "outputPolicy": config.output_policy, + "seededDefault": SeerWorkflowConfigExtras.parse_obj(config.extras).seeded_default, + "dateAdded": config.date_added, + "dateUpdated": config.date_updated, + } + if last_run is not None and last_run.last_run_at is not None: + result["dateLastRun"] = last_run.last_run_at + result["lastRunStatus"] = last_run.last_run_status + name = SeerWorkflowConfigExtras.parse_obj(config.extras).name + if name is not None: + result["name"] = name + return result diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py new file mode 100644 index 000000000..06e91926c --- /dev/null +++ b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import logging +from collections.abc import Collection, Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from functools import partial +from typing import Any, assert_never + +from django.db import router, transaction +from django.db.models import Count, Q, QuerySet +from pydantic import BaseModel + +from sentry.constants import ALL_ACCESS_PROJECTS, ObjectStatus +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.seer.models.workflow import ( + MAX_WORKFLOW_CONFIGS_PER_ORG, + SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, + SeerWorkflowConfig, + SeerWorkflowConfigExtras, + SeerWorkflowConfigProject, + SeerWorkflowFilters, + SeerWorkflowRun, + SeerWorkflowRunExtras, + SeerWorkflowRunStatus, + SeerWorkflowSchedule, + SeerWorkflowStrategy, + WorkflowRunSnapshot, +) +from sentry.seer.workflows.runs import SeerWorkflowRunLifecycle +from sentry.service_accounts.models import ServiceAccountReplica +from sentry.utils import json + +logger = logging.getLogger(__name__) + +SEER_DEFAULT_WORKFLOW_STRATEGIES = (SeerWorkflowStrategy.AGENTIC_TRIAGE_V2,) + + +class WorkflowUpdateError(ValueError): + """A rejected workflow update whose message is safe to show users.""" + + +@dataclass(frozen=True) +class AllProjects: + pass + + +@dataclass(frozen=True) +class SelectedProjects: + project_ids: tuple[int, ...] + + +type ProjectSelection = AllProjects | SelectedProjects + + +@dataclass(frozen=True, kw_only=True) +class SeerWorkflow: + """Apply each change to fresh state under the workflow config's row lock. + + Returned models do not carry a lock or refresh automatically. Configuration + changes, including project selection, should go through these methods. + """ + + config_id: int + organization_id: int + + @classmethod + def ensure_defaults(cls, organization_id: int) -> None: + """Seed defaults for strategies with no config; disabled configs count as existing.""" + database = router.db_for_write(SeerWorkflowConfig) + configs = SeerWorkflowConfig.objects.using(database).filter( + organization_id=organization_id, + strategy__in=[s.value for s in SEER_DEFAULT_WORKFLOW_STRATEGIES], + ) + existing = set(configs.values_list("strategy", flat=True)) + if all(s.value in existing for s in SEER_DEFAULT_WORKFLOW_STRATEGIES): + return + + with transaction.atomic(database): + # Prevent duplicate seeds without a uniqueness constraint on configurable strategies. + list( + Organization.objects.using(database).select_for_update().filter(id=organization_id) + ) + existing = set(configs.values_list("strategy", flat=True)) + SeerWorkflowConfig.objects.using(database).bulk_create( + [ + SeerWorkflowConfig( + organization_id=organization_id, + strategy=strategy.value, + filters=SeerWorkflowFilters(all_projects=True).dict(), + extras=SeerWorkflowConfigExtras(seeded_default=strategy).dict( + exclude_none=True + ), + ) + for strategy in SEER_DEFAULT_WORKFLOW_STRATEGIES + if strategy.value not in existing + ] + ) + + @classmethod + def _create_config( + cls, + *, + database: str, + organization_id: int, + strategy: SeerWorkflowStrategy, + enabled: bool, + filters: SeerWorkflowFilters, + name: str | None = None, + ) -> SeerWorkflowConfig: + """ + Create a config within the org's automation limit. Call inside a transaction on + ``database`` so the limit check and any related writes are atomic. + """ + existing = ( + SeerWorkflowConfig.objects.using(database) + .filter( + organization_id=organization_id, + strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, + ) + .count() + ) + if existing >= MAX_WORKFLOW_CONFIGS_PER_ORG: + raise WorkflowUpdateError( + f"Organizations can have at most {MAX_WORKFLOW_CONFIGS_PER_ORG} automations" + ) + + return SeerWorkflowConfig.objects.using(database).create( + organization_id=organization_id, + strategy=strategy, + enabled=enabled, + filters=filters.dict(), + extras=SeerWorkflowConfigExtras(name=name).dict(exclude_none=True), + ) + + @classmethod + def create( + cls, + *, + organization_id: int, + strategy: SeerWorkflowStrategy, + enabled: bool = False, + schedule: SeerWorkflowSchedule, + project_ids: Collection[int], + environments: Sequence[str] = (), + name: str | None = None, + ) -> SeerWorkflow: + """ + Create a user-defined automation. Use ``project_ids=[-1]`` to select all + projects and ``environments=[]`` for all environments. + """ + if strategy not in SEER_CONFIGURABLE_WORKFLOW_STRATEGIES: + raise WorkflowUpdateError(f"{strategy} automations cannot be configured") + if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + raise ValueError("Use create_with_monitors for metric workflows") + + project_set = set(project_ids) + all_projects = project_set == ALL_ACCESS_PROJECTS + if all_projects: + # [-1] is not stored as project links, only in config.filters + project_set = set() + cls._validate_project_ids(organization_id, project_set) + + database = router.db_for_write(SeerWorkflowConfig) + with transaction.atomic(database): + config = cls._create_config( + database=database, + organization_id=organization_id, + strategy=strategy, + enabled=enabled, + filters=SeerWorkflowFilters( + all_projects=all_projects, + environments=list(environments), + ), + name=name, + ) + config.schedule = schedule + config.save(using=database, update_fields=["schedule"]) + if project_set: + SeerWorkflowConfigProject.objects.using(database).bulk_create( + [ + SeerWorkflowConfigProject(config=config, project_id=project_id) + for project_id in sorted(project_set) + ] + ) + transaction.on_commit( + partial( + logger.info, + "seer.workflow.created", + extra={ + "organization_id": organization_id, + "config_id": config.id, + "config_version": config.version, + "strategy": config.strategy, + "all_projects": all_projects, + "project_ids": list(project_set), + "project_count": len(project_set), + }, + ), + using=database, + ) + return cls(config_id=config.id, organization_id=organization_id) + + def get_config(self) -> SeerWorkflowConfig: + """Fetch the config without acquiring a lock.""" + return self._config_queryset().get() + + @property + def extras(self) -> SeerWorkflowConfigExtras: + return SeerWorkflowConfigExtras.parse_obj(self.get_config().extras) + + @property + def filters(self) -> SeerWorkflowFilters: + return SeerWorkflowFilters.parse_obj(self.get_config().filters) + + def get_or_create_run( + self, + *, + trigger_key: str, + inputs: BaseModel, + extras: Mapping[str, Any] | None = None, + ) -> WorkflowRunCreationResult: + """Retries preserve the original configuration and inputs.""" + if not trigger_key or len(trigger_key) > 256: + raise ValueError("Trigger keys must contain between 1 and 256 characters") + + with self._lock_config() as config: + existing = self.get_runs().filter(trigger_key=trigger_key).first() + if existing is not None: + return WorkflowRunCreationResult( + lifecycle=SeerWorkflowRunLifecycle( + run_id=existing.id, organization_id=existing.organization_id + ), + created=False, + ) + + snapshot = WorkflowRunSnapshot( + config_version=config.version, + strategy=SeerWorkflowStrategy(config.strategy), + filters=config.filters, + project_ids=tuple(self._get_project_ids(config)), + output_policy=config.output_policy, + inputs=json.loads(inputs.json()), + ) + run_extras = SeerWorkflowRunExtras.parse_obj({**(extras or {}), "snapshot": snapshot}) + run = SeerWorkflowRun.objects.create( + organization_id=config.organization_id, + workflow_config=config, + trigger_key=trigger_key, + status=SeerWorkflowRunStatus.QUEUED, + extras=json.loads(run_extras.json()), + ) + return WorkflowRunCreationResult( + lifecycle=SeerWorkflowRunLifecycle( + run_id=run.id, organization_id=run.organization_id + ), + created=True, + ) + + def get_runs(self) -> QuerySet[SeerWorkflowRun]: + return ( + SeerWorkflowRun.objects.using(router.db_for_write(SeerWorkflowRun)) + .filter(organization_id=self.organization_id, workflow_config_id=self.config_id) + .order_by("-date_added", "-id") + ) + + def matches_scope(self, project_id: int, environment: str | None) -> bool: + """Check the automation's scope before recording an admission decision.""" + return self._matches_scope(self.get_config(), project_id, environment) + + def has_quota_available(self) -> bool: + """Allow runs until automation quota enforcement is implemented.""" + # TODO(telkins): Enforce automation quota before accepting a new run. + return True + + def get_project_ids(self) -> list[int]: + """Return active project IDs matching this workflow's filters, in ascending order.""" + return self._get_project_ids(self.get_config()) + + def has_all_projects(self) -> bool: + """ + Return whether this workflow's scope covers every active project in its organization. + Edge case: always false for empty selections, even if the org has no active projects. + """ + config = self.get_config() + selected_projects = self._get_projects(config).values("id") + counts = Project.objects.filter( + organization_id=config.organization_id, status=ObjectStatus.ACTIVE + ).aggregate( + total=Count("id"), + selected=Count("id", filter=Q(id__in=selected_projects)), + ) + return counts["selected"] > 0 and counts["selected"] == counts["total"] + + def set_service_account(self, service_account_id: int | None) -> None: + """Store the control account ID after validating its current cell replica.""" + with self._lock_config() as config: + if service_account_id is not None: + try: + replica = ServiceAccountReplica.objects.get( + service_account_id=service_account_id, + organization_id=config.organization_id, + ) + except ServiceAccountReplica.DoesNotExist as exc: + raise ValueError( + "Service account is unavailable in the workflow's organization" + ) from exc + if not replica.is_active: + raise ValueError("Service account is inactive") + + if config.service_account_id != service_account_id: + config.service_account_id = service_account_id + config.version += 1 + config.save(update_fields=["service_account_id", "version", "date_updated"]) + + def resolve_service_account(self) -> ServiceAccountReplica: + """Resolve authoritative identity against current cell state on every call.""" + config = self.get_config() + if config.service_account_id is None: + raise ValueError("Workflow has no service account") + try: + replica = ServiceAccountReplica.objects.get( + service_account_id=config.service_account_id, + organization_id=config.organization_id, + ) + except ServiceAccountReplica.DoesNotExist as exc: + raise ValueError( + "Service account is unavailable in the workflow's organization" + ) from exc + if not replica.is_active: + raise ValueError("Service account is inactive") + return replica + + def set_project_selection(self, selection: ProjectSelection) -> None: + """Replace the selected projects, bumping the version only when the selection changes. + + Callers must authorize the change; this only checks that the projects belong to the + workflow's organization. + """ + match selection: + case AllProjects(): + all_projects = True + project_ids: set[int] = set() + case SelectedProjects(project_ids=selected): + all_projects = False + project_ids = set(selected) + case _: + assert_never(selection) + + with self._lock_config() as config: + if config.strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: + raise WorkflowUpdateError( + "Breached metric projects are selected by connecting monitors" + ) + self._validate_project_ids(config.organization_id, project_ids) + + existing = set( + SeerWorkflowConfigProject.objects.filter(config=config).values_list( + "project_id", flat=True + ) + ) + removed = existing - project_ids + added = project_ids - existing + filters = SeerWorkflowFilters.parse_obj(config.filters) + if not removed and not added and filters.all_projects == all_projects: + return + + SeerWorkflowConfigProject.objects.filter(config=config, project_id__in=removed).delete() + SeerWorkflowConfigProject.objects.bulk_create( + [ + SeerWorkflowConfigProject(config=config, project_id=project_id) + for project_id in sorted(added) + ] + ) + filters.all_projects = all_projects + config.filters = config.filters | filters.dict() + config.version += 1 + config.save(update_fields=["filters", "version", "date_updated"]) + + @contextmanager + def _lock_config(self) -> Iterator[SeerWorkflowConfig]: + with transaction.atomic(router.db_for_write(SeerWorkflowConfig)): + yield self._config_queryset().select_for_update().get() + + def _config_queryset(self) -> QuerySet[SeerWorkflowConfig]: + return SeerWorkflowConfig.objects.using(router.db_for_write(SeerWorkflowConfig)).filter( + id=self.config_id, organization_id=self.organization_id + ) + + def _matches_scope( + self, config: SeerWorkflowConfig, project_id: int, environment: str | None + ) -> bool: + """Check scope using a config snapshot without modifying it or acquiring a lock.""" + filters = SeerWorkflowFilters.parse_obj(config.filters) + return ( + config.enabled + and (not filters.environments or environment in filters.environments) + and self._get_projects(config).filter(id=project_id).exists() + ) + + def _get_project_ids(self, config: SeerWorkflowConfig) -> list[int]: + return list(self._get_projects(config).order_by("id").values_list("id", flat=True)) + + @classmethod + def _validate_project_ids(cls, organization_id: int, project_ids: set[int]) -> None: + valid_project_ids = set( + Project.objects.filter(organization_id=organization_id, id__in=project_ids).values_list( + "id", flat=True + ) + ) + if project_ids != valid_project_ids: + raise WorkflowUpdateError("Selected projects must exist in the workflow's organization") + + def _get_projects(self, config: SeerWorkflowConfig) -> QuerySet[Project]: + """Return active projects in the config's scope; strategies may derive it differently.""" + projects = Project.objects.filter( + organization_id=config.organization_id, status=ObjectStatus.ACTIVE + ) + if not SeerWorkflowFilters.parse_obj(config.filters).all_projects: + projects = projects.filter( + id__in=SeerWorkflowConfigProject.objects.filter(config=config).values("project_id") + ) + return projects + + +@dataclass(frozen=True) +class WorkflowRunCreationResult: + lifecycle: SeerWorkflowRunLifecycle + created: bool diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py new file mode 100644 index 000000000..d9ca1f347 --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py @@ -0,0 +1,1533 @@ +from __future__ import annotations + +import abc +from collections.abc import Collection, Iterable, Mapping +from dataclasses import dataclass +from functools import cached_property +from typing import Any + +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.http.request import HttpRequest +from rest_framework.request import Request +from sentry_sdk import traces + +from sentry import features, roles +from sentry.api.exceptions import DataSecrecyError +from sentry.auth.deprecated_scope_translation import translate_deprecated_scopes +from sentry.auth.principal import has_user +from sentry.auth.scope_declaration import check_scope_declaration, check_scope_declarations +from sentry.auth.services.access.service import access_service +from sentry.auth.services.auth import AuthenticatedToken, RpcAuthState, RpcMemberSsoState +from sentry.auth.staff import is_active_staff +from sentry.auth.superuser import ( + Superuser, + get_superuser_access_expiry, + get_superuser_scopes, + is_active_superuser, + resolve_superuser_access, +) +from sentry.auth.system import is_system_auth +from sentry.constants import ObjectStatus +from sentry.data_secrecy.logic import should_allow_superuser_access +from sentry.models.apiscopes import add_scope_hierarchy +from sentry.models.organization import Organization +from sentry.models.organizationmember import OrganizationMember +from sentry.models.organizationmemberteam import OrganizationMemberTeam +from sentry.models.project import Project +from sentry.models.team import Team, TeamStatus +from sentry.organizations.services.organization import ( + RpcTeamMember, + RpcUserOrganizationContext, + organization_service, +) +from sentry.organizations.services.organization.serial import summarize_member +from sentry.roles import organization_roles +from sentry.roles.manager import OrganizationRole, TeamRole +from sentry.seer.agent_token import is_agent_auth +from sentry.sentry_apps.models.sentry_app import SentryApp +from sentry.service_accounts.auth import is_service_account_auth +from sentry.service_accounts.models import ServiceAccountProjectReplica, ServiceAccountReplica +from sentry.types.request import _HttpRequestWithUser +from sentry.users.models.user import User +from sentry.users.services.user import RpcUser +from sentry.users.services.user.service import user_service +from sentry.utils import metrics +from sentry.viewer_context import SuperuserAccess, set_viewer_context_superuser + +__all__ = ( + "from_user", + "from_member", + "DEFAULT", + "from_user_and_rpc_user_org_context", + "from_rpc_member", +) + + +def has_role_in_organization(role: str, organization: Organization, user_id: int) -> bool: + query = OrganizationMember.objects.filter( + user_is_active=True, + user_id=user_id, + organization_id=organization.id, + role=role, + ) + return query.exists() + + +class Access(abc.ABC): + @property + @abc.abstractmethod + def sso_is_valid(self) -> bool: + pass + + @property + @abc.abstractmethod + def requires_sso(self) -> bool: + pass + + @property + @abc.abstractmethod + def has_open_membership(self) -> bool: + pass + + @property + @abc.abstractmethod + def has_global_access(self) -> bool: + pass + + @property + @abc.abstractmethod + def scopes(self) -> frozenset[str]: + pass + + @property + @abc.abstractmethod + def permissions(self) -> frozenset[str]: + pass + + # TODO(cathy): remove this + @property + @abc.abstractmethod + def role(self) -> str | None: + pass + + @property + @abc.abstractmethod + def team_ids_with_membership(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def accessible_team_ids(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def project_ids_with_team_membership(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def accessible_project_ids(self) -> frozenset[int]: + pass + + @property + def is_integration_token(self) -> bool: + return False + + def has_permission(self, permission: str) -> bool: + """ + Return bool representing if the user has the given permission. + >>> access.has_permission('broadcasts.admin') + """ + return permission in self.permissions + + def has_scope(self, scope: str) -> bool: + """ + Return bool representing if the user has the given scope. + >>> access.has_project('org:read') + """ + check_scope_declaration(scope) + return scope in self.scopes + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + """Whether adding ``scope`` to the current auth scope cap would grant it.""" + check_scope_declaration(scope) + return False + + def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: + """Whether adding ``scope`` to the current auth scope cap would grant it for a project.""" + check_scope_declaration(scope) + return False + + def get_organization_role(self) -> OrganizationRole | None: + if self.role is not None: + return organization_roles.get(self.role) + return None + + @abc.abstractmethod + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + pass + + @abc.abstractmethod + def has_team_access(self, team: Team) -> bool: + """ + Return bool representing if a user should have access to information for the given team. + >>> access.has_team_access(team) + """ + + @abc.abstractmethod + def has_team_scope(self, team: Team, scope: str) -> bool: + pass + + def has_team_membership(self, team: Team) -> bool: + return team.id in self.team_ids_with_membership + + @abc.abstractmethod + def get_team_role(self, team: Team) -> TeamRole | None: + pass + + @abc.abstractmethod + def has_project_access(self, project: Project) -> bool: + """ + Return bool representing if a user should have access to information for the given project. + >>> access.has_project_access(project) + """ + raise NotImplementedError + + def has_projects_access(self, projects: Iterable[Project]) -> bool: + """ + Returns bool representing if a user should have access to every requested project + """ + return all([self.has_project_access(project) for project in projects]) + + def has_project_membership(self, project: Project) -> bool: + """ + Return bool representing if a user has explicit membership for the given project. + >>> access.has_project_membership(project) + """ + return project.id in self.project_ids_with_team_membership + + def has_project_scope(self, project: Project, scope: str) -> bool: + """ + Return bool representing if a user should have access with the given scope to information + for the given project. + + >>> access.has_project_scope(project, 'project:read') + """ + return self.has_any_project_scope(project, [scope]) + + @abc.abstractmethod + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + pass + + +def _intersect_member_and_token_scopes( + member_scopes: Collection[str], token_scopes: Iterable[str] | None +) -> frozenset[str]: + member_scopes = frozenset(member_scopes) + if token_scopes is None: + return member_scopes + + token_scopes = frozenset(token_scopes) + token_only_scopes = token_scopes & settings.SENTRY_TOKEN_ONLY_SCOPES + + return (token_scopes & member_scopes) | token_only_scopes + + +@dataclass +class DbAccess(Access): + # TODO(dcramer): this is still a little gross, and ideally backend access + # would be based on the same scopes as API access so there's clarity in + # what things mean + + sso_is_valid: bool = False + requires_sso: bool = False + has_open_membership: bool = False + + # if has_global_access is True, then any project + # matching organization_id is valid. This is used for + # both `organization.allow_joinleave` and to indicate + # that the role is global / a user is an active superuser + has_global_access: bool = False + + scopes: frozenset[str] = frozenset() + scopes_upper_bound: frozenset[str] | None = None + permissions: frozenset[str] = frozenset() + + _member: OrganizationMember | None = None + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + return self._member.role if self._member else None + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + if self._member is None or self.scopes_upper_bound is None: + return False + candidate_scopes = _intersect_member_and_token_scopes( + self._member.get_scopes(), self.scopes_upper_bound | {scope} + ) + return scope in add_scope_hierarchy(list(candidate_scopes)) + + @cached_property + def _team_memberships(self) -> Mapping[Team, OrganizationMemberTeam]: + if self._member is None: + return {} + return { + omt.team: omt + for omt in OrganizationMemberTeam.objects.filter( + organizationmember=self._member, is_active=True, team__status=TeamStatus.ACTIVE + ).select_related("team") + } + + @cached_property + def team_ids_with_membership(self) -> frozenset[int]: + """Return the IDs of teams in which the user has actual membership. + + This represents the set of all teams for which `has_team_membership` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + + Compare to accessible_team_ids, which is equal to this property in the + typical case but represents a superset of IDs in case of superuser access. + """ + return frozenset(team.id for team in self._team_memberships.keys()) + + @property + def accessible_team_ids(self) -> frozenset[int]: + """Return the IDs of teams to which the user has access. + + This represents the set of all teams for which `has_team_access` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + """ + return self.team_ids_with_membership + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + """Return the IDs of projects to which the user has access via actual team membership. + + This represents the set of all projects for which `has_project_membership` + returns true. Use that method where possible and use this property only when + you need to iterate or query for all such teams. + + Compare to accessible_project_ids, which is equal to this property in the + typical case but represents a superset of IDs in case of superuser access. + """ + teams = self._team_memberships.keys() + if not teams: + return frozenset() + + with traces.start_span( + name="get_project_access_in_teams", + attributes={"sentry.op": "get_project_access_in_teams"}, + ) as span: + projects = frozenset( + Project.objects.filter(status=ObjectStatus.ACTIVE, teams__in=teams) + .distinct() + .values_list("id", flat=True) + ) + span.set_attribute("Project Count", len(projects)) + span.set_attribute("Team Count", len(teams)) + + return projects + + @property + def accessible_project_ids(self) -> frozenset[int]: + """Return the IDs of projects to which the user has access. + + This represents the set of all teams for which `has_project_access` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + """ + return self.project_ids_with_team_membership + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + if self._member and self._member.user_id is not None: + return has_role_in_organization( + role=role, organization=organization, user_id=self._member.user_id + ) + return False + + def has_team_scope(self, team: Team, scope: str) -> bool: + """ + Return bool representing if a user should have access with the given scope to information + for the given team. + + >>> access.has_team_scope(team, 'team:read') + """ + check_scope_declaration(scope) + if not self.has_team_access(team): + return False + if self.has_scope(scope): + return True + + membership = self._team_memberships.get(team) + if not membership: + return False + + team_scopes = membership.get_scopes() + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + if membership and scope in team_scopes: + metrics.incr( + "team_roles.pass_by_team_scope", + tags={"team_role": membership.role, "scope": scope}, + ) + return True + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + team_member = self._team_memberships.get(team) + if team_member: + return team_member.get_team_role() + else: + return None + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + """ + Represent if a user should have access with any one of the given scopes to + information for the given project. + + For performance's sake, prefer this over multiple calls to `has_project_scope`. + """ + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + if any(self.has_scope(scope) for scope in scopes): + return True + + if self._member and features.has("organizations:team-roles", self._member.organization): + with traces.start_span( + name="check_access_for_all_project_teams", + attributes={"sentry.op": "check_access_for_all_project_teams"}, + ) as span: + memberships = [ + self._team_memberships[team] + for team in project.teams.all() + if team in self._team_memberships + ] + span.set_attribute("organization", self._member.organization.id) + span.set_attribute("organization.slug", self._member.organization.slug) + span.set_attribute("membership_count", len(memberships)) + + for membership in memberships: + team_scopes = membership.get_scopes() + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + for scope in scopes: + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_project_scope", + tags={"team_role": membership.role, "scope": scope}, + ) + return True + return False + + +@dataclass +class SingularRpcAccessOrgOptimization: + access: RpcBackedAccess + + +def maybe_singular_rpc_access_org_context( + access: Access, org_ids: set[int] +) -> SingularRpcAccessOrgOptimization | None: + if ( + isinstance(access, RpcBackedAccess) + and len(org_ids) == 1 + and access.rpc_user_organization_context.organization.id in org_ids + ): + return SingularRpcAccessOrgOptimization(access) + return None + + +maybe_singular_api_access_org_context = maybe_singular_rpc_access_org_context + + +@dataclass +class RpcBackedAccess(Access): + rpc_user_organization_context: RpcUserOrganizationContext + scopes_upper_bound: frozenset[str] | None + auth_state: RpcAuthState + + # TODO: remove once getsentry has updated to use the new names. + @property + def api_user_organization_context(self) -> RpcUserOrganizationContext: + return self.rpc_user_organization_context + + @cached_property + def permissions(self) -> frozenset[str]: + return frozenset(self.auth_state.permissions) + + @property + def sso_is_valid(self) -> bool: + return self.auth_state.sso_state.is_valid + + @property + def requires_sso(self) -> bool: + return self.auth_state.sso_state.is_required + + @property + def has_open_membership(self) -> bool: + return self.rpc_user_organization_context.organization.flags.allow_joinleave + + @property + def has_global_access(self) -> bool: + if self.has_open_membership: + return True + + if ( + self.rpc_user_organization_context.member + and roles.get(self.rpc_user_organization_context.member.role).is_global + ): + return True + + return False + + @cached_property + def scopes(self) -> frozenset[str]: + if self.rpc_user_organization_context.member is None: + return frozenset(self.scopes_upper_bound or []) + + if self.scopes_upper_bound is None: + return frozenset(self.rpc_user_organization_context.member.scopes) + + return _intersect_member_and_token_scopes( + self.rpc_user_organization_context.member.scopes, + self.scopes_upper_bound, + ) + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + member = self.rpc_user_organization_context.member + if member is None or self.scopes_upper_bound is None: + return False + candidate_scopes = _intersect_member_and_token_scopes( + member.scopes, self.scopes_upper_bound | {scope} + ) + return scope in add_scope_hierarchy(list(candidate_scopes)) + + def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: + """ + Return whether adding a scope to the token would grant access for this project. + + Check both organization-level scopes and scopes from the member's role on + a project team. This does not modify the token or access instance. + """ + if not self.has_project_access(project): + return False + if self.would_have_scope_with_added_auth_scope(scope): + return True + + # Team roles can only grant project scopes for member-backed token access. + member = self.rpc_user_organization_context.member + if ( + member is None + or self.scopes_upper_bound is None + or not features.has( + "organizations:team-roles", self.rpc_user_organization_context.organization + ) + ): + return False + + # A role on any team linked to the project may grant the requested scope... + project_team_ids = set(project.teams.values_list("id", flat=True)) + return any( + member_team.role + and member_team.team_id in project_team_ids + and scope in member_team.role.scopes + for member_team in member.member_teams + ) + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + if self.rpc_user_organization_context.member is None: + return None + return self.rpc_user_organization_context.member.role + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + member = self.rpc_user_organization_context.member + if member and member.user_id: + return has_role_in_organization( + role=role, organization=organization, user_id=member.user_id + ) + return False + + @cached_property + def team_ids_with_membership(self) -> frozenset[int]: + if self.rpc_user_organization_context.member is None: + return frozenset() + return frozenset( + mt.team_id for mt in self.rpc_user_organization_context.member.member_teams + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return self.team_ids_with_membership + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + if self.rpc_user_organization_context.member is None: + return frozenset() + return frozenset(self.rpc_user_organization_context.member.project_ids) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return self.project_ids_with_team_membership + + def has_team_access(self, team: Team) -> bool: + if team.status != TeamStatus.ACTIVE: + return False + if ( + self.has_global_access + and self.rpc_user_organization_context.organization.id == team.organization_id + ): + return True + return team.id in self.team_ids_with_membership + + def get_team_membership(self, team_id: int) -> RpcTeamMember | None: + if self.rpc_user_organization_context.member is None: + return None + + for team_membership in self.rpc_user_organization_context.member.member_teams: + if team_membership.team_id == team_id: + return team_membership + return None + + def has_team_scope(self, team: Team, scope: str) -> bool: + check_scope_declaration(scope) + if not self.has_team_access(team): + return False + if self.has_scope(scope): + return True + + if self.rpc_user_organization_context.member is None: + return False + + team_membership = self.get_team_membership(team.id) + if not team_membership: + return False + + team_scopes = frozenset(team_membership.scopes) + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_team_scope", + tags={"team_role": f"{team_membership.role}", "scope": scope}, + ) + return True + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + team_member = self.get_team_membership(team.id) + if team_member: + return team_member.role + return None + + def has_project_access(self, project: Project) -> bool: + if project.status != ObjectStatus.ACTIVE: + return False + if ( + self.has_global_access + and self.rpc_user_organization_context.member + and self.rpc_user_organization_context.organization.id == project.organization_id + ): + return True + return project.id in self.project_ids_with_team_membership + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + """ + Represent if a user should have access with any one of the given scopes to + information for the given project. + + For performance's sake, prefer this over multiple calls to `has_project_scope`. + """ + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + if any(self.has_scope(scope) for scope in scopes): + return True + + if self.rpc_user_organization_context.member and features.has( + "organizations:team-roles", self.rpc_user_organization_context.organization + ): + with traces.start_span( + name="check_access_for_all_project_teams", + attributes={"sentry.op": "check_access_for_all_project_teams"}, + ) as span: + project_teams_id = set(project.teams.values_list("id", flat=True)) + orgmember_teams = self.rpc_user_organization_context.member.member_teams + span.set_attribute( + "organization", self.rpc_user_organization_context.organization.id + ) + span.set_attribute( + "organization.slug", self.rpc_user_organization_context.organization.slug + ) + span.set_attribute("membership_count", len(orgmember_teams)) + + for member_team in orgmember_teams: + if not member_team.role: + continue + if member_team.team_id not in project_teams_id: + continue + + team_scopes = member_team.role.scopes + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + for scope in scopes: + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_project_scope", + tags={"team_role": f"{member_team.role.id}", "scope": scope}, + ) + return True + return False + + +def _wrap_scopes(scopes_upper_bound: Iterable[str] | None) -> frozenset[str] | None: + if scopes_upper_bound is not None: + return frozenset(scopes_upper_bound) + return None + + +class OrganizationMemberAccess(DbAccess): + def __init__( + self, + member: OrganizationMember, + scopes: Iterable[str], + permissions: Iterable[str], + scopes_upper_bound: Iterable[str] | None, + ) -> None: + auth_state = access_service.get_user_auth_state( + organization_id=member.organization_id, + is_superuser=False, + is_staff=False, + org_member=summarize_member(member), + user_id=member.user_id, + ) + sso_state = auth_state.sso_state + has_global_access = ( + bool(member.organization.flags.allow_joinleave) or roles.get(member.role).is_global + ) + + super().__init__( + _member=member, + sso_is_valid=sso_state.is_valid, + requires_sso=sso_state.is_required, + has_global_access=has_global_access, + scopes=frozenset(scopes), + permissions=frozenset(permissions), + scopes_upper_bound=_wrap_scopes(scopes_upper_bound), + ) + + def has_team_access(self, team: Team) -> bool: + assert self._member is not None + if team.status != TeamStatus.ACTIVE: + return False + if self.has_global_access and self._member.organization.id == team.organization_id: + return True + return team.id in self.team_ids_with_membership + + def has_project_access(self, project: Project) -> bool: + assert self._member is not None + if project.status != ObjectStatus.ACTIVE: + return False + if self.has_global_access and self._member.organization.id == project.organization_id: + return True + return project.id in self.project_ids_with_team_membership + + +class OrganizationGlobalAccess(DbAccess): + """Access to all an organization's teams and projects.""" + + def __init__( + self, organization: Organization | int, scopes: Iterable[str], **kwargs: Any + ) -> None: + self._organization_id = ( + organization.id if isinstance(organization, Organization) else organization + ) + super().__init__(has_global_access=True, scopes=frozenset(scopes), **kwargs) + + def has_team_access(self, team: Team) -> bool: + return bool( + team.organization_id == self._organization_id and team.status == TeamStatus.ACTIVE + ) + + def has_project_access(self, project: Project) -> bool: + return bool( + project.organization_id == self._organization_id + and project.status == ObjectStatus.ACTIVE + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset( + Team.objects.filter( + organization_id=self._organization_id, status=TeamStatus.ACTIVE + ).values_list("id", flat=True) + ) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset( + Project.objects.filter( + organization_id=self._organization_id, status=ObjectStatus.ACTIVE + ).values_list("id", flat=True) + ) + + +class ServiceAccountAccess(DbAccess): + """ + A service account's access to its organization. It has no membership, so its + scopes are the account's own. With `all_projects` it reaches every active project + in the org. Without it, it reaches only the active projects linked to the + account. It never reaches teams. Open membership is a setting for people, so + it doesn't apply. + """ + + def __init__( + self, + *, + service_account_id: int, + organization_id: int, + scopes: Iterable[str], + all_projects: bool, + ) -> None: + self._service_account_id = service_account_id + self._organization_id = organization_id + super().__init__( + sso_is_valid=True, + has_global_access=all_projects, + scopes=frozenset(scopes), + ) + + def has_team_access(self, team: Team) -> bool: + return False + + def has_project_access(self, project: Project) -> bool: + if ( + project.organization_id != self._organization_id + or project.status != ObjectStatus.ACTIVE + ): + return False + return self.has_global_access or project.id in self.accessible_project_ids + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + # With no teams, the account's project links stand in for membership, so + # endpoints that default to "my projects" see the projects it can reach. + return self.accessible_project_ids + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset() + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + if self.has_global_access: + return frozenset( + Project.objects.filter( + organization_id=self._organization_id, status=ObjectStatus.ACTIVE + ).values_list("id", flat=True) + ) + # A replica link can briefly outlive a transfer of its project to another org. + return frozenset( + ServiceAccountProjectReplica.objects.filter( + service_account__service_account_id=self._service_account_id, + project__organization_id=self._organization_id, + project__status=ObjectStatus.ACTIVE, + ).values_list("project_id", flat=True) + ) + + +class ApiBackedOrganizationGlobalAccess(RpcBackedAccess): + """Access to all an organization's teams and projects.""" + + def __init__( + self, + *, + rpc_user_organization_context: RpcUserOrganizationContext, + auth_state: RpcAuthState, + scopes: Iterable[str] | None, + ): + super().__init__( + rpc_user_organization_context=rpc_user_organization_context, + auth_state=auth_state, + scopes_upper_bound=_wrap_scopes(scopes), + ) + + @cached_property + def scopes(self) -> frozenset[str]: + return frozenset(self.scopes_upper_bound or []) + + @property + def has_global_access(self) -> bool: + return True + + def has_team_access(self, team: Team) -> bool: + return bool( + team.organization_id == self.rpc_user_organization_context.organization.id + and team.status == TeamStatus.ACTIVE + ) + + def has_project_access(self, project: Project) -> bool: + return bool( + project.organization_id == self.rpc_user_organization_context.organization.id + and project.status == ObjectStatus.ACTIVE + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset( + t.id + for t in self.rpc_user_organization_context.organization.teams + if t.status == TeamStatus.ACTIVE + ) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset( + p.id + for p in self.rpc_user_organization_context.organization.projects + if p.status == ObjectStatus.ACTIVE + ) + + +class OrganizationGlobalMembership(OrganizationGlobalAccess): + """Access to all an organization's teams and projects with simulated membership.""" + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return self.accessible_team_ids + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return self.accessible_project_ids + + def has_team_membership(self, team: Team) -> bool: + return self.has_team_access(team) + + def has_project_membership(self, project: Project) -> bool: + return self.has_project_access(project) + + +class ApiOrganizationGlobalMembership(ApiBackedOrganizationGlobalAccess): + """Access to all an organization's teams and projects with simulated membership.""" + + @property + def is_integration_token(self) -> bool: + return True + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return self.accessible_team_ids + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return self.accessible_project_ids + + def has_team_membership(self, team: Team) -> bool: + return self.has_team_access(team) + + def has_project_membership(self, project: Project) -> bool: + return self.has_project_access(project) + + +@dataclass +class OrganizationlessAccess(Access): + auth_state: RpcAuthState + + @cached_property + def permissions(self) -> frozenset[str]: + return frozenset(self.auth_state.permissions) + + def has_team_access(self, team: Team) -> bool: + return False + + def has_project_access(self, project: Project) -> bool: + return False + + @property + def sso_is_valid(self) -> bool: + return self.auth_state.sso_state.is_valid + + @property + def requires_sso(self) -> bool: + return self.auth_state.sso_state.is_required + + @property + def has_open_membership(self) -> bool: + return False + + @property + def has_global_access(self) -> bool: + return False + + @property + def scopes(self) -> frozenset[str]: + return frozenset() + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + return None + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + if user_id: + return has_role_in_organization(role=role, organization=organization, user_id=user_id) + return False + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset() + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset() + + def has_team_scope(self, team: Team, scope: str) -> bool: + check_scope_declaration(scope) + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + return None + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + + return any(self.has_scope(scope) for scope in scopes) + + +class SystemAccess(OrganizationlessAccess): + def __init__(self) -> None: + super().__init__( + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_required=False, is_valid=False), + permissions=[], + ), + ) + + @property + def has_global_access(self) -> bool: + return True + + def has_permission(self, permission: str) -> bool: + return True + + def has_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + return True + + def has_team_access(self, team: Team) -> bool: + return True + + def has_project_access(self, project: Project) -> bool: + return True + + # The semantically correct behavior for accessible_(team|project)_ids would be to + # query for all teams or projects in the system, which we don't want to attempt. + # Code paths that may have SystemAccess must avoid looking at these properties. + @property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset() + + +class NoAccess(OrganizationlessAccess): + def __init__(self) -> None: + super().__init__( + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_required=False, is_valid=True), + permissions=[], + ), + ) + + +def _propagate_superuser_access( + request: _HttpRequestWithUser, + org_context: RpcUserOrganizationContext, + su: Superuser | None, +) -> None: + # Members use their normal callback permissions and user flags; adding this + # claim would instead force read-only access and a five-minute VC lifetime. + if org_context.member is not None: + return + if request.auth is not None or su is None or not su.is_active: + return + if su.requires_org_auth(org_context.organization): + return + expires_at = get_superuser_access_expiry(su, org_context.organization.id) + if expires_at is not None: + set_viewer_context_superuser( + user_id=request.user.id, + organization_id=org_context.organization.id, + superuser=SuperuserAccess(expires_at=expires_at), + ) + + +def from_request_org_and_scopes( + *, + request: HttpRequest, + rpc_user_org_context: RpcUserOrganizationContext | None = None, + scopes: Iterable[str] | None = None, +) -> Access: + """ + Note that `scopes` is usually None because request.auth is not set at `get_authorization_header` + when the request is made from the frontend using cookies + """ + if isinstance(request.auth, AuthenticatedToken) and request.auth.superuser is not None: + if rpc_user_org_context is None: + return DEFAULT + return from_superuser_access(request.auth, rpc_user_org_context) + if is_agent_auth(request.auth): + if rpc_user_org_context is None: + return DEFAULT + return from_agent_auth(request.auth, rpc_user_org_context) + + is_staff = is_active_staff(request) + + if not rpc_user_org_context: + return from_user_and_rpc_user_org_context( + user=request.user, + rpc_user_org_context=rpc_user_org_context, + is_superuser=is_active_superuser(request), + is_staff=is_staff, + scopes=scopes, + ) + + if getattr(request, "actual_user", None) is not None: + if not should_allow_superuser_access(rpc_user_org_context): + raise DataSecrecyError() + + if getattr(request.user, "is_sentry_app", False): + return _from_rpc_sentry_app(rpc_user_org_context) + + if is_active_superuser(request): + su = getattr(request, "superuser", None) + if su is None and request.auth is None: + su = Superuser(request) + requires_org_auth = su is not None and su.requires_org_auth( + rpc_user_org_context.organization + ) + if requires_org_auth and rpc_user_org_context.member is None: + setattr(request, "_superuser_needs_org_auth", rpc_user_org_context.organization.slug) + + member = rpc_user_org_context.member + auth_state = access_service.get_user_auth_state( + user_id=request.user.id, + organization_id=rpc_user_org_context.organization.id, + is_superuser=True, + is_staff=is_staff, + org_member=member, + ) + + superuser_scopes = get_superuser_scopes(auth_state, rpc_user_org_context) + if scopes: + superuser_scopes = superuser_scopes.union(set(scopes)) + if member and member.scopes: + superuser_scopes = superuser_scopes.union(set(member.scopes)) + + _propagate_superuser_access(request, rpc_user_org_context, su) + + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=auth_state, + scopes=superuser_scopes, + ) + + if request.auth is not None and not has_user(request.user): + return from_rpc_auth(request.auth, rpc_user_org_context) + + return from_user_and_rpc_user_org_context( + user=request.user, + rpc_user_org_context=rpc_user_org_context, + is_superuser=False, + is_staff=is_staff, + scopes=scopes, + ) + + +def organizationless_access( + user: User | RpcUser | AnonymousUser, is_superuser: bool, is_staff: bool +) -> Access: + return OrganizationlessAccess( + auth_state=access_service.get_user_auth_state( + user_id=user.id, + is_superuser=is_superuser, + is_staff=is_staff, + organization_id=None, + org_member=None, + ) + ) + + +def normalize_valid_user(user: User | RpcUser | AnonymousUser | None) -> User | RpcUser | None: + if not user or not has_user(user) or not user.is_active: + return None + return user + + +def from_user_and_rpc_user_org_context( + *, + user: User | AnonymousUser | RpcUser | None, + rpc_user_org_context: RpcUserOrganizationContext | None = None, + is_superuser: bool = False, + is_staff: bool = False, + scopes: Iterable[str] | None = None, + auth_state: RpcAuthState | None = None, +) -> Access: + if (user := normalize_valid_user(user)) is None: + return DEFAULT + + if not rpc_user_org_context or not rpc_user_org_context.member: + return organizationless_access(user, is_superuser, is_staff) + + return from_rpc_member( + rpc_user_organization_context=rpc_user_org_context, + scopes=scopes, + is_superuser=is_superuser, + is_staff=is_staff, + auth_state=auth_state, + ) + + +def from_request( + request: Request, organization: Organization | None = None, scopes: Iterable[str] | None = None +) -> Access: + if isinstance(request.auth, AuthenticatedToken) and ( + is_agent_auth(request.auth) or request.auth.superuser is not None + ): + if organization is None: + return DEFAULT + return from_auth(request.auth, organization) + + is_staff = is_active_staff(request) + + if not organization: + return from_user( + request.user, + organization=organization, + scopes=scopes, + is_superuser=is_active_superuser(request), + is_staff=is_staff, + ) + + if getattr(request, "actual_user", None) is not None: + if not should_allow_superuser_access(organization): + raise DataSecrecyError() + + if request.user.is_authenticated and request.user.is_sentry_app: + return _from_sentry_app(request.user, organization=organization) + + if is_active_superuser(request): + member: OrganizationMember | None = None + try: + member = OrganizationMember.objects.get( + user_id=request.user.id, organization_id=organization.id + ) + except OrganizationMember.DoesNotExist: + pass + auth_state = access_service.get_user_auth_state( + user_id=request.user.id, + organization_id=organization.id, + is_superuser=True, + is_staff=is_staff, + org_member=(summarize_member(member) if member is not None else None), + ) + sso_state = auth_state.sso_state + + superuser_scopes = get_superuser_scopes(auth_state, organization) + if scopes: + superuser_scopes = superuser_scopes.union(set(scopes)) + if member and (member_scopes := member.get_scopes()): + superuser_scopes = superuser_scopes.union(set(member_scopes)) + + return OrganizationGlobalAccess( + organization=organization, + _member=member, + scopes=superuser_scopes, + sso_is_valid=sso_state.is_valid, + requires_sso=sso_state.is_required, + permissions=access_service.get_permissions_for_user(request.user.id), + ) + + if request.auth is not None and not has_user(request.user): + return from_auth(request.auth, organization) + + return from_user(user=request.user, organization=organization, scopes=scopes, is_staff=is_staff) + + +# only used internally +def _from_sentry_app(user: User, organization: Organization | None = None) -> Access: + if not organization: + return NoAccess() + + sentry_app_query = SentryApp.objects.filter(proxy_user=user) + + if not sentry_app_query.exists(): + return NoAccess() + + sentry_app = sentry_app_query.get() + + if not sentry_app.is_installed_on(organization): + return NoAccess() + + return OrganizationGlobalMembership( + organization, translate_deprecated_scopes(sentry_app.scope_list), sso_is_valid=True + ) + + +def _from_rpc_sentry_app(context: RpcUserOrganizationContext | None = None) -> Access: + from sentry.sentry_apps.services.app import app_service + + if not context or context.user_id is None: + return NoAccess() + + installation = app_service.find_installation_by_proxy_user( + proxy_user_id=context.user_id, organization_id=context.organization.id + ) + if installation is None: + return NoAccess() + + return ApiOrganizationGlobalMembership( + rpc_user_organization_context=context, + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState( + is_valid=True, + is_required=False, + ), + permissions=[], + ), + scopes=translate_deprecated_scopes(installation.sentry_app.scope_list), + ) + + +def from_user( + user: User | RpcUser | AnonymousUser | None, + organization: Organization | None = None, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, +) -> Access: + if (user := normalize_valid_user(user)) is None: + return DEFAULT + + if not organization: + return organizationless_access(user, is_superuser, is_staff) + + try: + om = OrganizationMember.objects.get(user_id=user.id, organization_id=organization.id) + except OrganizationMember.DoesNotExist: + return organizationless_access(user, is_superuser, is_staff) + + # ensure cached relation + om.organization = organization + + return from_member(om, scopes=scopes, is_superuser=is_superuser, is_staff=is_staff) + + +def from_member( + member: OrganizationMember, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, +) -> Access: + scope_intersection = _intersect_member_and_token_scopes(member.get_scopes(), scopes) + + if (is_superuser or is_staff) and member.user_id is not None: + # "permissions" is a bit of a misnomer -- these are all admin level permissions, and the intent is that if you + # have them, you can only use them when you are acting, as a superuser or staff. This is intentional. + permissions = access_service.get_permissions_for_user(member.user_id) + else: + permissions = frozenset() + + return OrganizationMemberAccess(member, scope_intersection, permissions, scopes) + + +def from_rpc_member( + rpc_user_organization_context: RpcUserOrganizationContext, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, + auth_state: RpcAuthState | None = None, +) -> Access: + if rpc_user_organization_context.user_id is None: + return DEFAULT + + return RpcBackedAccess( + rpc_user_organization_context=rpc_user_organization_context, + scopes_upper_bound=_wrap_scopes(scopes), + auth_state=auth_state + or access_service.get_user_auth_state( + user_id=rpc_user_organization_context.user_id, + organization_id=rpc_user_organization_context.organization.id, + is_superuser=is_superuser, + is_staff=is_staff, + org_member=rpc_user_organization_context.member, + ), + ) + + +def from_auth(auth: AuthenticatedToken, organization: Organization) -> Access: + if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: + context = organization_service.get_organization_by_id( + id=organization.id, user_id=auth.user_id + ) + return from_superuser_access(auth, context) if context is not None else DEFAULT + if is_system_auth(auth): + return SystemAccess() + if is_service_account_auth(auth): + return from_service_account_auth(auth, organization.id) + if is_agent_auth(auth): + access: Access = DEFAULT + if auth.user_id is not None and auth.organization_id == organization.id: + try: + member = OrganizationMember.objects.get( + user_id=auth.user_id, organization_id=organization.id + ) + except OrganizationMember.DoesNotExist: + pass + else: + member.organization = organization + access = from_member(member, scopes=auth.get_scopes()) + return access + auth_organization_id = auth.organization_id + if auth_organization_id is not None and auth_organization_id == organization.id: + return OrganizationGlobalAccess( + auth_organization_id, settings.SENTRY_SCOPES, sso_is_valid=True + ) + else: + return DEFAULT + + +def from_rpc_auth( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: + return from_superuser_access(auth, rpc_user_org_context) + if is_system_auth(auth): + return SystemAccess() + if is_service_account_auth(auth): + return from_service_account_auth(auth, rpc_user_org_context.organization.id) + if is_agent_auth(auth): + # Agents are non-user actors with member-derived, capped authority -- never the + # org-global access an org token gets. Dispatched here so the cap holds at the + # shared userless-auth choke, not only in determine_access. + return from_agent_auth(auth, rpc_user_org_context) + if auth.organization_id == rpc_user_org_context.organization.id: + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=RpcAuthState( + permissions=[], + sso_state=RpcMemberSsoState( + is_valid=True, + is_required=False, + ), + ), + scopes=settings.SENTRY_SCOPES, + ) + else: + return DEFAULT + + +def from_agent_auth( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + """Access for a Seer agent capability token: a non-user actor acting on behalf of a + member. Unlike an org token, the agent is not org-global — its authority is the + delegating member's, capped by the token's scopes, so project access follows the + member's teams. The context MUST be resolved for the delegating user_id.""" + # Bound to the org it was minted for; never honored elsewhere, even if the + # delegating user is also a member of the requested org. + if auth.organization_id != rpc_user_org_context.organization.id: + return DEFAULT + if auth.user_id != rpc_user_org_context.user_id: + return DEFAULT + # No membership (never a member, or revoked since mint) -> no access. Required + # explicitly because RpcBackedAccess would otherwise hand back the full token + # scopes uncapped when member is None. + if rpc_user_org_context.member is None: + return DEFAULT + return from_rpc_member(rpc_user_org_context, scopes=auth.get_scopes()) + + +def from_service_account_auth(auth: AuthenticatedToken, organization_id: int) -> Access: + # Bound to the account's own org, like agent tokens. + if auth.entity_id is None or auth.organization_id != organization_id: + return DEFAULT + # The token doesn't carry `all_projects`, so read it from the replica, like the + # project links. An account deactivated since authentication gets no access. + all_projects = ( + ServiceAccountReplica.objects.filter( + service_account_id=auth.entity_id, organization_id=organization_id, is_active=True + ) + .values_list("all_projects", flat=True) + .first() + ) + if all_projects is None: + return DEFAULT + return ServiceAccountAccess( + service_account_id=auth.entity_id, + organization_id=organization_id, + scopes=auth.get_scopes(), + all_projects=all_projects, + ) + + +def from_superuser_access( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + if ( + auth.superuser is None + or auth.organization_id != rpc_user_org_context.organization.id + or auth.user_id != rpc_user_org_context.user_id + ): + return DEFAULT + users = user_service.get_many(filter={"user_ids": [auth.user_id]}) + user = users[0] if users else None + if user is None: + return DEFAULT + delegated = resolve_superuser_access(auth.superuser, user, rpc_user_org_context) + if delegated is None: + return DEFAULT + scopes, _ = delegated + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_valid=True, is_required=False), permissions=[] + ), + scopes=scopes & set(auth.get_scopes()), + ) + + +DEFAULT = NoAccess() diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py new file mode 100644 index 000000000..54d7541cf --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py @@ -0,0 +1,593 @@ +from collections.abc import Mapping +from typing import Any, Literal, TypedDict + +from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import serializers, status +from rest_framework.exceptions import ValidationError +from rest_framework.request import Request +from rest_framework.response import Response + +from sentry import audit_log, features, roles +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import cell_silo_endpoint +from sentry.api.bases import OrganizationMemberEndpoint +from sentry.api.bases.organization import OrganizationPermission +from sentry.api.exceptions import ResourceDoesNotExist +from sentry.api.serializers import Serializer, serialize +from sentry.api.serializers.models.team import ( + BaseTeamSerializer, + TeamSerializer, + TeamSerializerResponse, +) +from sentry.apidocs.constants import ( + RESPONSE_ACCEPTED, + RESPONSE_BAD_REQUEST, + RESPONSE_NO_CONTENT, + RESPONSE_NOT_FOUND, + RESPONSE_UNAUTHORIZED, +) +from sentry.apidocs.examples.team_examples import TeamExamples +from sentry.apidocs.omissions import sentry_schema_serializer +from sentry.apidocs.parameters import GlobalParams +from sentry.apidocs.response_types import DetailResponse +from sentry.auth.access import Access +from sentry.auth.superuser import superuser_has_permission +from sentry.core.endpoints.organization_member_utils import can_admin_team, can_set_team_role +from sentry.models.groupassignee import GroupAssignee +from sentry.models.groupsubscription import GroupSubscription +from sentry.models.organization import Organization +from sentry.models.organizationaccessrequest import OrganizationAccessRequest +from sentry.models.organizationmember import OrganizationMember +from sentry.models.organizationmemberteam import OrganizationMemberTeam +from sentry.models.team import Team +from sentry.notifications.types import GroupSubscriptionReason +from sentry.roles import organization_roles, team_roles +from sentry.roles.manager import TeamRole +from sentry.utils import metrics + +ERR_INSUFFICIENT_ROLE = "You do not have permission to edit that user's membership." + + +class OrganizationMemberTeamSerializerResponse(TypedDict): + isActive: bool + # This must be manually kept up to date, because we cannot dynamically + # unpack into static type annotations. See https://github.com/microsoft/pylance-release/issues/4084 + teamRole: Literal["contributor", "admin"] + + +@sentry_schema_serializer( + omit_from_public_schema={ + "isActive": "Reported in responses, but the update handler only applies teamRole; " + "a value sent here is validated and discarded.", + } +) +class OrganizationMemberTeamSerializer(serializers.Serializer[dict[str, Any]]): + isActive = serializers.BooleanField() + teamRole = serializers.ChoiceField( + choices=team_roles.get_descriptions(), + default=team_roles.get_default().id, + help_text="The team-level role to switch to. Valid roles include:", + # choices will follow in the docs + ) + + +class OrganizationMemberTeamDetailsSerializer(Serializer[OrganizationMemberTeamSerializerResponse]): + def serialize( + self, obj: OrganizationMemberTeam, attrs: Mapping[Any, Any], user: Any, **kwargs: Any + ) -> OrganizationMemberTeamSerializerResponse: + return { + "isActive": obj.is_active, + "teamRole": obj.role, # type:ignore[typeddict-item] + } + + +class OrganizationTeamMemberPermission(OrganizationPermission): + scope_map = { + "GET": [ + "org:read", + "org:write", + "org:admin", + "member:read", + "member:write", + "member:admin", + ], + "POST": ["org:read", "org:write", "team:write"], + "PUT": [ + "org:read", + "org:write", + "org:admin", + "member:read", + "member:write", + "member:admin", + "team:admin", + "team:write", + ], + "DELETE": ["org:read", "org:write", "org:admin", "team:admin"], + } + + +def _has_elevated_scope(access: Access) -> bool: + """ + Validate that the token has more than just org:read + """ + return access.has_scope("org:write") or access.has_scope("team:write") + + +def _is_org_owner_or_manager(access: Access) -> bool: + role = access.get_organization_role() + # only org owners and managers have org:write scope + return "org:write" in role.scopes if role else False + + +@extend_schema(tags=["Teams"]) +@cell_silo_endpoint +class OrganizationMemberTeamDetailsEndpoint(OrganizationMemberEndpoint): + def convert_args( + self, + request: Request, + organization_id_or_slug: int | str | None = None, + *args: Any, + **kwargs: Any, + ) -> tuple[tuple[Any, ...], dict[str, Any]]: + args, kwargs = super().convert_args(request, organization_id_or_slug, *args, **kwargs) + + team_id_or_slug = kwargs.pop("team_id_or_slug") + organization = kwargs["organization"] + member = kwargs["member"] + + if request.method == "GET": + try: + omt = OrganizationMemberTeam.objects.get( + team__slug__id_or_slug=team_id_or_slug, organizationmember=member + ) + except OrganizationMemberTeam.DoesNotExist: + raise ResourceDoesNotExist + + kwargs["omt"] = omt + + else: + try: + team = Team.objects.get( + organization__slug__id_or_slug=organization.slug, + slug__id_or_slug=team_id_or_slug, + ) + except Team.DoesNotExist: + raise ResourceDoesNotExist + kwargs["team"] = team + + return (args, kwargs) + + publish_status = { + "DELETE": ApiPublishStatus.PUBLIC, + "GET": ApiPublishStatus.PRIVATE, + "PUT": ApiPublishStatus.PUBLIC, + "POST": ApiPublishStatus.PUBLIC, + } + owner = ApiOwner.FOUNDATIONS + permission_classes = (OrganizationTeamMemberPermission,) + + def _can_create_team_member(self, request: Request, team: Team) -> bool: + """ + User can join or add a member to a team: + + * If they are an active superuser + * If they are a team admin or have global write access + * If the open membership organization setting is enabled + """ + access = request.access + + # When open membership is disabled, we need to check if the token has elevated permissions + # in order to ensure integration tokens with only "org:read" scope cannot add members. This check + # comes first because access.has_global_access is True for all integration tokens + if access.is_integration_token and not access.has_open_membership: + return _has_elevated_scope(access) + return access.has_global_access or can_admin_team(access, team) + + def _can_delete( + self, + request: Request, + member: OrganizationMember, + team: Team, + ) -> bool: + """ + User can remove a member from a team: + + * If they are an active superuser + * If they are removing their own membership + * If they are a team admin or have global write access + """ + if superuser_has_permission(request): + return True + + if not request.user.is_authenticated: + return False + + if request.user.id == member.user_id: + return True + + # There is an edge case where org owners/managers cannot remove a member from a team they + # are not part of using team:admin. We cannot explicitly check for team:admin b/c org admins + # also have it but are only allowed to remove members from teams they are on. + if _is_org_owner_or_manager(request.access): + return True + + return can_admin_team(request.access, team) + + def _create_access_request( + self, request: Request, team: Team, member: OrganizationMember + ) -> None: + omt, created = OrganizationAccessRequest.objects.get_or_create(team=team, member=member) + + if not created: + return + + requester = request.user.id if request.user.id != member.user_id else None + if requester: + omt.update(requester_id=requester) + + omt.send_request_email() + + def get( + self, + request: Request, + organization: Organization, + member: OrganizationMember, + omt: OrganizationMemberTeam, + ) -> Response: + return Response( + serialize(omt, request.user, OrganizationMemberTeamDetailsSerializer()), status=200 + ) + + @extend_schema( + operation_id="addOrganizationMemberTeam", + summary="Add an Organization Member to a Team", + parameters=[ + GlobalParams.ORG_ID_OR_SLUG, + GlobalParams.member_id("The ID of the organization member to add to the team"), + GlobalParams.TEAM_ID_OR_SLUG, + ], + request=None, + responses={ + 201: BaseTeamSerializer, + 202: RESPONSE_ACCEPTED, + 204: RESPONSE_NO_CONTENT, + 401: RESPONSE_UNAUTHORIZED, + 403: OpenApiResponse( + description="This team is managed through your organization's identity provider" + ), + 404: RESPONSE_NOT_FOUND, + }, + examples=TeamExamples.ADD_TO_TEAM, + ) + def post( + self, + request: Request, + organization: Organization, + member: OrganizationMember, + team: Team, + ) -> Response[TeamSerializerResponse] | Response[None] | Response[DetailResponse]: + # NOTE: Required to use HTML for table b/c this markdown version doesn't support colspan. + r""" + This request can return various success codes depending on the context of the team: + - **`201`**: The member has been successfully added. + - **`202`**: The member needs permission to join the team and an access request + has been generated. + - **`204`**: The member is already on the team. + + If the team is provisioned through an identity provider, the member cannot join the + team through Sentry. + + Note the permission scopes vary depending on the organization setting `"Open Membership"` + and the type of authorization token. The following table outlines the accepted scopes. + + + + + + + + + + + + + + + + + + + + + + + + +
Open Membership
OnOff
Internal Integration Token +
    +
  • • org:read
    • +
+ 		+
    +
  • • org:write
    • +
  • • team:write
    • +
+
User Auth Token +
    +
  • • org:read
    • +
+ 		+
    +
  • • org:read*
    • +
  • • org:write
    • +
  • • org:read +
    • +
  •    team:write**
    • +
+
+ + + *Organization members are restricted to this scope. When sending a request, it will always + return a 202 and request an invite to the team. + + + \*\*Team Admins must have both **`org:read`** and **`team:write`** scopes in their user + authorization token to add members to their teams. + """ + if not request.user.is_authenticated: + return Response(status=status.HTTP_401_UNAUTHORIZED) + + if not organization_roles.get(member.role).is_team_roles_allowed: + return Response( + { + "detail": f"The user with a '{member.role}' role cannot have team-level permissions." + }, + status=403, + ) + + if OrganizationMemberTeam.objects.filter(team=team, organizationmember=member).exists(): + return Response(status=204) + + if team.idp_provisioned: + return Response( + {"detail": "This team is managed through your organization's identity provider."}, + status=403, + ) + + if not self._can_create_team_member(request, team): + self._create_access_request(request, team, member) + return Response(status=202) + + omt = OrganizationMemberTeam.objects.create(team=team, organizationmember=member) + + self.create_audit_entry( + request=request, + organization=organization, + target_object=omt.id, + target_user_id=member.user_id, + event=audit_log.get_event_id("MEMBER_JOIN_TEAM"), + data=omt.get_audit_log_data(), + ) + + body: TeamSerializerResponse = serialize(team, request.user, TeamSerializer()) + return Response(body, status=201) + + @extend_schema( + operation_id="updateOrganizationMemberTeam", + summary="Update an Organization Member's Team Role", + parameters=[ + GlobalParams.ORG_ID_OR_SLUG, + GlobalParams.member_id("The ID of the organization member to change"), + GlobalParams.TEAM_ID_OR_SLUG, + ], + request=OrganizationMemberTeamSerializer, + responses={ + 200: OrganizationMemberTeamDetailsSerializer, + 400: RESPONSE_BAD_REQUEST, + 404: RESPONSE_NOT_FOUND, + }, + examples=TeamExamples.UPDATE_TEAM_ROLE, + ) + def put( + self, + request: Request, + organization: Organization, + member: OrganizationMember, + team: Team, + ) -> ( + Response[OrganizationMemberTeamSerializerResponse] + | Response[None] + | Response[DetailResponse] + ): + """ + The relevant organization member must already be a part of the team. + + Note that for organization admins, managers, and owners, they are + automatically granted a minimum team role of `admin` on all teams they + are part of. Read more about [team roles](https://docs.sentry.io/product/teams/roles/). + """ + try: + omt = OrganizationMemberTeam.objects.get(team=team, organizationmember=member) + except OrganizationMemberTeam.DoesNotExist: + raise ResourceDoesNotExist + + serializer = OrganizationMemberTeamSerializer(data=request.data, partial=True) + if not serializer.is_valid(): + raise ValidationError(serializer.errors) + result = serializer.validated_data + + if "teamRole" in result and features.has("organizations:team-roles", organization): + new_role_id = result["teamRole"] + try: + new_role = team_roles.get(new_role_id) + except KeyError: + return Response(status=400) + can_set_new_role = can_set_team_role(request, team, new_role) + + try: + old_role = team_roles.get(omt.role) if omt.role else None + except KeyError: + old_role = None + if old_role is None: + old_role = roles.get_minimum_team_role(member.role) + can_set_old_role = can_set_team_role(request, team, old_role) + + # Verify that the request is allowed to set both the old and new role to prevent role downgrades by low-privilege users + if not (can_set_new_role and can_set_old_role): + return Response({"detail": ERR_INSUFFICIENT_ROLE}, status=400) + + self._change_team_member_role(omt, new_role) + + self.create_audit_entry( + request=request, + organization=organization, + target_object=omt.id, + target_user_id=member.user_id, + event=audit_log.get_event_id("MEMBER_EDIT"), + data={ + "email": member.get_email(), + "role": member.role, + "team_slug": team.slug, + "team_role": new_role_id, + }, + ) + + put_body: OrganizationMemberTeamSerializerResponse = serialize( + omt, request.user, OrganizationMemberTeamDetailsSerializer() + ) + return Response(put_body, status=200) + + @staticmethod + def _change_team_member_role( + team_membership: OrganizationMemberTeam, team_role: TeamRole + ) -> None: + """Modify a member's team-level role.""" + minimum_team_role = roles.get_minimum_team_role(team_membership.organizationmember.role) + if team_role.priority > minimum_team_role.priority: + applying_minimum = False + team_membership.update(role=team_role.id) + else: + # The new team role is redundant to the role that this member would + # receive as their minimum team role anyway. This makes it effectively + # invisible in the UI, and it would be surprising if it were suddenly + # left over after the user's org-level role is demoted. So, write a null + # value to the database and let the minimum team role take over. + applying_minimum = True + team_membership.update(role=None) + + metrics.incr( + "team_roles.assign", + tags={"target_team_role": team_role.id, "applying_minimum": str(applying_minimum)}, + ) + + @extend_schema( + operation_id="deleteOrganizationMemberTeam", + summary="Delete an Organization Member from a Team", + parameters=[ + GlobalParams.ORG_ID_OR_SLUG, + GlobalParams.member_id("The ID of the organization member to delete from the team"), + GlobalParams.TEAM_ID_OR_SLUG, + ], + responses={ + 200: BaseTeamSerializer, + 400: RESPONSE_BAD_REQUEST, + 403: OpenApiResponse( + description="This team is managed through your organization's identity provider" + ), + 404: RESPONSE_NOT_FOUND, + }, + examples=TeamExamples.DELETE_FROM_TEAM, + ) + def delete( + self, + request: Request, + organization: Organization, + member: OrganizationMember, + team: Team, + ) -> Response[TeamSerializerResponse] | Response[DetailResponse]: + r""" + Delete an organization member from a team. + + Note the permission scopes vary depending on the type of authorization token. The following + table outlines the accepted scopes. + + + + + + + + + +
Org Auth Token +
    +
  • • org:write
    • +
  • • org:admin
    • +
  • • team:admin
    • +
+
User Auth Token +
    +
  • • org:read*
    • +
  • • org:write
    • +
  • • org:admin
    • +
  • • team:admin
    • +
  • • org:read + team:admin**
    • +
+
+ + + \***`org:read`** can only be used to remove yourself from the teams you are a member of. + + + \*\*Team Admins must have both **`org:read`** and **`team:admin`** scopes in their user + authorization token to delete members from their teams. + """ + + if not self._can_delete(request, member, team): + return Response({"detail": ERR_INSUFFICIENT_ROLE}, status=400) + + if team.idp_provisioned: + return Response( + {"detail": "This team is managed through your organization's identity provider."}, + status=403, + ) + + omt = None + try: + omt = OrganizationMemberTeam.objects.get(team=team, organizationmember=member) + except OrganizationMemberTeam.DoesNotExist: + pass + + else: + self.create_audit_entry( + request=request, + organization=organization, + target_object=omt.id, + target_user_id=member.user_id, + event=audit_log.get_event_id("MEMBER_LEAVE_TEAM"), + data=omt.get_audit_log_data(), + ) + omt.delete() + + self._unsubscribe_issues(team, member) + + delete_body: TeamSerializerResponse = serialize(team, request.user, TeamSerializer()) + return Response(delete_body, status=200) + + @staticmethod + def _unsubscribe_issues(team: Team, member: OrganizationMember) -> None: + """ + Unsubscribe user from issues the team is subscribed to + """ + team_assigned_groups = GroupAssignee.objects.filter(team_id=team.id).values_list( + "group_id", flat=True + ) + team_subscribed_groups = GroupSubscription.objects.filter( + team_id=team.id, reason=GroupSubscriptionReason.assigned + ).values_list("group_id", flat=True) + group_ids_to_unsubscribe = set(team_assigned_groups) | set(team_subscribed_groups) + GroupSubscription.objects.filter( + group_id__in=group_ids_to_unsubscribe, user_id=member.user_id + ).delete() diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py new file mode 100644 index 000000000..d5fa08b5d --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py @@ -0,0 +1,1537 @@ +from __future__ import annotations + +import abc +from collections.abc import Collection, Iterable, Mapping +from dataclasses import dataclass +from functools import cached_property +from typing import Any + +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.http.request import HttpRequest +from rest_framework.request import Request +from sentry_sdk import traces + +from sentry import features, roles +from sentry.api.exceptions import DataSecrecyError +from sentry.auth.deprecated_scope_translation import translate_deprecated_scopes +from sentry.auth.principal import has_user +from sentry.auth.scope_declaration import check_scope_declaration, check_scope_declarations +from sentry.auth.services.access.service import access_service +from sentry.auth.services.auth import AuthenticatedToken, RpcAuthState, RpcMemberSsoState +from sentry.auth.staff import is_active_staff +from sentry.auth.superuser import ( + Superuser, + get_superuser_access_expiry, + get_superuser_scopes, + is_active_superuser, + resolve_superuser_access, +) +from sentry.auth.system import is_system_auth +from sentry.constants import ObjectStatus +from sentry.data_secrecy.logic import should_allow_superuser_access +from sentry.models.apiscopes import add_scope_hierarchy +from sentry.models.organization import Organization +from sentry.models.organizationmember import OrganizationMember +from sentry.models.organizationmemberteam import OrganizationMemberTeam +from sentry.models.project import Project +from sentry.models.team import Team, TeamStatus +from sentry.organizations.services.organization import ( + RpcTeamMember, + RpcUserOrganizationContext, + organization_service, +) +from sentry.organizations.services.organization.serial import summarize_member +from sentry.roles import organization_roles +from sentry.roles.manager import OrganizationRole, TeamRole +from sentry.seer.agent_token import is_agent_auth +from sentry.sentry_apps.models.sentry_app import SentryApp +from sentry.service_accounts.auth import is_service_account_auth +from sentry.service_accounts.models import ServiceAccountProjectReplica +from sentry.types.request import _HttpRequestWithUser +from sentry.users.models.user import User +from sentry.users.services.user import RpcUser +from sentry.users.services.user.service import user_service +from sentry.utils import metrics +from sentry.viewer_context import SuperuserAccess, set_viewer_context_superuser + +__all__ = ( + "from_user", + "from_member", + "DEFAULT", + "from_user_and_rpc_user_org_context", + "from_rpc_member", +) + + +def has_role_in_organization(role: str, organization: Organization, user_id: int) -> bool: + query = OrganizationMember.objects.filter( + user_is_active=True, + user_id=user_id, + organization_id=organization.id, + role=role, + ) + return query.exists() + + +class Access(abc.ABC): + @property + @abc.abstractmethod + def sso_is_valid(self) -> bool: + pass + + @property + @abc.abstractmethod + def requires_sso(self) -> bool: + pass + + @property + @abc.abstractmethod + def has_open_membership(self) -> bool: + pass + + @property + @abc.abstractmethod + def has_global_access(self) -> bool: + pass + + @property + @abc.abstractmethod + def scopes(self) -> frozenset[str]: + pass + + @property + @abc.abstractmethod + def permissions(self) -> frozenset[str]: + pass + + # TODO(cathy): remove this + @property + @abc.abstractmethod + def role(self) -> str | None: + pass + + @property + @abc.abstractmethod + def team_ids_with_membership(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def accessible_team_ids(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def project_ids_with_team_membership(self) -> frozenset[int]: + pass + + @property + @abc.abstractmethod + def accessible_project_ids(self) -> frozenset[int]: + pass + + @property + def is_integration_token(self) -> bool: + return False + + def has_permission(self, permission: str) -> bool: + """ + Return bool representing if the user has the given permission. + >>> access.has_permission('broadcasts.admin') + """ + return permission in self.permissions + + def has_scope(self, scope: str) -> bool: + """ + Return bool representing if the user has the given scope. + >>> access.has_project('org:read') + """ + check_scope_declaration(scope) + return scope in self.scopes + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + """Whether adding ``scope`` to the current auth scope cap would grant it.""" + check_scope_declaration(scope) + return False + + def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: + """Whether adding ``scope`` to the current auth scope cap would grant it for a project.""" + check_scope_declaration(scope) + return False + + def get_organization_role(self) -> OrganizationRole | None: + if self.role is not None: + return organization_roles.get(self.role) + return None + + @abc.abstractmethod + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + pass + + @abc.abstractmethod + def has_team_access(self, team: Team) -> bool: + """ + Return bool representing if a user should have access to information for the given team. + >>> access.has_team_access(team) + """ + + @abc.abstractmethod + def has_team_scope(self, team: Team, scope: str) -> bool: + pass + + def has_team_membership(self, team: Team) -> bool: + return team.id in self.team_ids_with_membership + + @abc.abstractmethod + def get_team_role(self, team: Team) -> TeamRole | None: + pass + + @abc.abstractmethod + def has_project_access(self, project: Project) -> bool: + """ + Return bool representing if a user should have access to information for the given project. + >>> access.has_project_access(project) + """ + raise NotImplementedError + + def has_projects_access(self, projects: Iterable[Project]) -> bool: + """ + Returns bool representing if a user should have access to every requested project + """ + return all([self.has_project_access(project) for project in projects]) + + def has_project_membership(self, project: Project) -> bool: + """ + Return bool representing if a user has explicit membership for the given project. + >>> access.has_project_membership(project) + """ + return project.id in self.project_ids_with_team_membership + + def has_project_scope(self, project: Project, scope: str) -> bool: + """ + Return bool representing if a user should have access with the given scope to information + for the given project. + + >>> access.has_project_scope(project, 'project:read') + """ + return self.has_any_project_scope(project, [scope]) + + @abc.abstractmethod + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + pass + + +def _intersect_member_and_token_scopes( + member_scopes: Collection[str], token_scopes: Iterable[str] | None +) -> frozenset[str]: + member_scopes = frozenset(member_scopes) + if token_scopes is None: + return member_scopes + + token_scopes = frozenset(token_scopes) + token_only_scopes = token_scopes & settings.SENTRY_TOKEN_ONLY_SCOPES + + return (token_scopes & member_scopes) | token_only_scopes + + +@dataclass +class DbAccess(Access): + # TODO(dcramer): this is still a little gross, and ideally backend access + # would be based on the same scopes as API access so there's clarity in + # what things mean + + sso_is_valid: bool = False + requires_sso: bool = False + has_open_membership: bool = False + + # if has_global_access is True, then any project + # matching organization_id is valid. This is used for + # both `organization.allow_joinleave` and to indicate + # that the role is global / a user is an active superuser + has_global_access: bool = False + + scopes: frozenset[str] = frozenset() + scopes_upper_bound: frozenset[str] | None = None + permissions: frozenset[str] = frozenset() + + _member: OrganizationMember | None = None + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + return self._member.role if self._member else None + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + if self._member is None or self.scopes_upper_bound is None: + return False + candidate_scopes = _intersect_member_and_token_scopes( + self._member.get_scopes(), self.scopes_upper_bound | {scope} + ) + return scope in add_scope_hierarchy(list(candidate_scopes)) + + @cached_property + def _team_memberships(self) -> Mapping[Team, OrganizationMemberTeam]: + if self._member is None: + return {} + return { + omt.team: omt + for omt in OrganizationMemberTeam.objects.filter( + organizationmember=self._member, is_active=True, team__status=TeamStatus.ACTIVE + ).select_related("team") + } + + @cached_property + def team_ids_with_membership(self) -> frozenset[int]: + """Return the IDs of teams in which the user has actual membership. + + This represents the set of all teams for which `has_team_membership` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + + Compare to accessible_team_ids, which is equal to this property in the + typical case but represents a superset of IDs in case of superuser access. + """ + return frozenset(team.id for team in self._team_memberships.keys()) + + @property + def accessible_team_ids(self) -> frozenset[int]: + """Return the IDs of teams to which the user has access. + + This represents the set of all teams for which `has_team_access` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + """ + return self.team_ids_with_membership + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + """Return the IDs of projects to which the user has access via actual team membership. + + This represents the set of all projects for which `has_project_membership` + returns true. Use that method where possible and use this property only when + you need to iterate or query for all such teams. + + Compare to accessible_project_ids, which is equal to this property in the + typical case but represents a superset of IDs in case of superuser access. + """ + teams = self._team_memberships.keys() + if not teams: + return frozenset() + + with traces.start_span( + name="get_project_access_in_teams", + attributes={"sentry.op": "get_project_access_in_teams"}, + ) as span: + projects = frozenset( + Project.objects.filter(status=ObjectStatus.ACTIVE, teams__in=teams) + .distinct() + .values_list("id", flat=True) + ) + span.set_attribute("Project Count", len(projects)) + span.set_attribute("Team Count", len(teams)) + + return projects + + @property + def accessible_project_ids(self) -> frozenset[int]: + """Return the IDs of projects to which the user has access. + + This represents the set of all teams for which `has_project_access` returns + true. Use that method where possible and use this property only when you need + to iterate or query for all such teams. + """ + return self.project_ids_with_team_membership + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + if self._member and self._member.user_id is not None: + return has_role_in_organization( + role=role, organization=organization, user_id=self._member.user_id + ) + return False + + def has_team_scope(self, team: Team, scope: str) -> bool: + """ + Return bool representing if a user should have access with the given scope to information + for the given team. + + >>> access.has_team_scope(team, 'team:read') + """ + check_scope_declaration(scope) + if not self.has_team_access(team): + return False + if self.has_scope(scope): + return True + + membership = self._team_memberships.get(team) + if not membership: + return False + + team_scopes = membership.get_scopes() + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + if membership and scope in team_scopes: + metrics.incr( + "team_roles.pass_by_team_scope", + tags={"team_role": membership.role, "scope": scope}, + ) + return True + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + team_member = self._team_memberships.get(team) + if team_member: + return team_member.get_team_role() + else: + return None + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + """ + Represent if a user should have access with any one of the given scopes to + information for the given project. + + For performance's sake, prefer this over multiple calls to `has_project_scope`. + """ + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + if any(self.has_scope(scope) for scope in scopes): + return True + + if self._member and features.has("organizations:team-roles", self._member.organization): + with traces.start_span( + name="check_access_for_all_project_teams", + attributes={"sentry.op": "check_access_for_all_project_teams"}, + ) as span: + memberships = [ + self._team_memberships[team] + for team in project.teams.all() + if team in self._team_memberships + ] + span.set_attribute("organization", self._member.organization.id) + span.set_attribute("organization.slug", self._member.organization.slug) + span.set_attribute("membership_count", len(memberships)) + + for membership in memberships: + team_scopes = membership.get_scopes() + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + for scope in scopes: + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_project_scope", + tags={"team_role": membership.role, "scope": scope}, + ) + return True + return False + + +@dataclass +class SingularRpcAccessOrgOptimization: + access: RpcBackedAccess + + +def maybe_singular_rpc_access_org_context( + access: Access, org_ids: set[int] +) -> SingularRpcAccessOrgOptimization | None: + if ( + isinstance(access, RpcBackedAccess) + and len(org_ids) == 1 + and access.rpc_user_organization_context.organization.id in org_ids + ): + return SingularRpcAccessOrgOptimization(access) + return None + + +maybe_singular_api_access_org_context = maybe_singular_rpc_access_org_context + + +@dataclass +class RpcBackedAccess(Access): + rpc_user_organization_context: RpcUserOrganizationContext + scopes_upper_bound: frozenset[str] | None + auth_state: RpcAuthState + + # TODO: remove once getsentry has updated to use the new names. + @property + def api_user_organization_context(self) -> RpcUserOrganizationContext: + return self.rpc_user_organization_context + + @cached_property + def permissions(self) -> frozenset[str]: + return frozenset(self.auth_state.permissions) + + @property + def sso_is_valid(self) -> bool: + return self.auth_state.sso_state.is_valid + + @property + def requires_sso(self) -> bool: + return self.auth_state.sso_state.is_required + + @property + def has_open_membership(self) -> bool: + return self.rpc_user_organization_context.organization.flags.allow_joinleave + + @property + def has_global_access(self) -> bool: + if self.has_open_membership: + return True + + if ( + self.rpc_user_organization_context.member + and roles.get(self.rpc_user_organization_context.member.role).is_global + ): + return True + + return False + + @cached_property + def scopes(self) -> frozenset[str]: + if self.rpc_user_organization_context.member is None: + return frozenset(self.scopes_upper_bound or []) + + if self.scopes_upper_bound is None: + return frozenset(self.rpc_user_organization_context.member.scopes) + + return _intersect_member_and_token_scopes( + self.rpc_user_organization_context.member.scopes, + self.scopes_upper_bound, + ) + + def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + member = self.rpc_user_organization_context.member + if member is None or self.scopes_upper_bound is None: + return False + candidate_scopes = _intersect_member_and_token_scopes( + member.scopes, self.scopes_upper_bound | {scope} + ) + return scope in add_scope_hierarchy(list(candidate_scopes)) + + def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: + """ + Return whether adding a scope to the token would grant access for this project. + + Check both organization-level scopes and scopes from the member's role on + a project team. This does not modify the token or access instance. + """ + if not self.has_project_access(project): + return False + if self.would_have_scope_with_added_auth_scope(scope): + return True + + # Team roles can only grant project scopes for member-backed token access. + member = self.rpc_user_organization_context.member + if ( + member is None + or self.scopes_upper_bound is None + or not features.has( + "organizations:team-roles", self.rpc_user_organization_context.organization + ) + ): + return False + + # A role on any team linked to the project may grant the requested scope... + project_team_ids = set(project.teams.values_list("id", flat=True)) + return any( + member_team.role + and member_team.team_id in project_team_ids + and scope in member_team.role.scopes + for member_team in member.member_teams + ) + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + if self.rpc_user_organization_context.member is None: + return None + return self.rpc_user_organization_context.member.role + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + member = self.rpc_user_organization_context.member + if member and member.user_id: + return has_role_in_organization( + role=role, organization=organization, user_id=member.user_id + ) + return False + + @cached_property + def team_ids_with_membership(self) -> frozenset[int]: + if self.rpc_user_organization_context.member is None: + return frozenset() + return frozenset( + mt.team_id for mt in self.rpc_user_organization_context.member.member_teams + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return self.team_ids_with_membership + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + if self.rpc_user_organization_context.member is None: + return frozenset() + return frozenset(self.rpc_user_organization_context.member.project_ids) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return self.project_ids_with_team_membership + + def has_team_access(self, team: Team) -> bool: + if team.status != TeamStatus.ACTIVE: + return False + if ( + self.has_global_access + and self.rpc_user_organization_context.organization.id == team.organization_id + ): + return True + return team.id in self.team_ids_with_membership + + def get_team_membership(self, team_id: int) -> RpcTeamMember | None: + if self.rpc_user_organization_context.member is None: + return None + + for team_membership in self.rpc_user_organization_context.member.member_teams: + if team_membership.team_id == team_id: + return team_membership + return None + + def has_team_scope(self, team: Team, scope: str) -> bool: + check_scope_declaration(scope) + if not self.has_team_access(team): + return False + if self.has_scope(scope): + return True + + if self.rpc_user_organization_context.member is None: + return False + + team_membership = self.get_team_membership(team.id) + if not team_membership: + return False + + team_scopes = frozenset(team_membership.scopes) + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_team_scope", + tags={"team_role": f"{team_membership.role}", "scope": scope}, + ) + return True + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + team_member = self.get_team_membership(team.id) + if team_member: + return team_member.role + return None + + def has_project_access(self, project: Project) -> bool: + if project.status != ObjectStatus.ACTIVE: + return False + if ( + self.has_global_access + and self.rpc_user_organization_context.member + and self.rpc_user_organization_context.organization.id == project.organization_id + ): + return True + return project.id in self.project_ids_with_team_membership + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + """ + Represent if a user should have access with any one of the given scopes to + information for the given project. + + For performance's sake, prefer this over multiple calls to `has_project_scope`. + """ + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + if any(self.has_scope(scope) for scope in scopes): + return True + + if self.rpc_user_organization_context.member and features.has( + "organizations:team-roles", self.rpc_user_organization_context.organization + ): + with traces.start_span( + name="check_access_for_all_project_teams", + attributes={"sentry.op": "check_access_for_all_project_teams"}, + ) as span: + project_teams_id = set(project.teams.values_list("id", flat=True)) + orgmember_teams = self.rpc_user_organization_context.member.member_teams + span.set_attribute( + "organization", self.rpc_user_organization_context.organization.id + ) + span.set_attribute( + "organization.slug", self.rpc_user_organization_context.organization.slug + ) + span.set_attribute("membership_count", len(orgmember_teams)) + + for member_team in orgmember_teams: + if not member_team.role: + continue + if member_team.team_id not in project_teams_id: + continue + + team_scopes = member_team.role.scopes + if self.scopes_upper_bound: + team_scopes = team_scopes & self.scopes_upper_bound + + for scope in scopes: + if scope in team_scopes: + metrics.incr( + "team_roles.pass_by_project_scope", + tags={"team_role": f"{member_team.role.id}", "scope": scope}, + ) + return True + return False + + +def _wrap_scopes(scopes_upper_bound: Iterable[str] | None) -> frozenset[str] | None: + if scopes_upper_bound is not None: + return frozenset(scopes_upper_bound) + return None + + +class OrganizationMemberAccess(DbAccess): + def __init__( + self, + member: OrganizationMember, + scopes: Iterable[str], + permissions: Iterable[str], + scopes_upper_bound: Iterable[str] | None, + ) -> None: + auth_state = access_service.get_user_auth_state( + organization_id=member.organization_id, + is_superuser=False, + is_staff=False, + org_member=summarize_member(member), + user_id=member.user_id, + ) + sso_state = auth_state.sso_state + has_global_access = ( + bool(member.organization.flags.allow_joinleave) or roles.get(member.role).is_global + ) + + super().__init__( + _member=member, + sso_is_valid=sso_state.is_valid, + requires_sso=sso_state.is_required, + has_global_access=has_global_access, + scopes=frozenset(scopes), + permissions=frozenset(permissions), + scopes_upper_bound=_wrap_scopes(scopes_upper_bound), + ) + + def has_team_access(self, team: Team) -> bool: + assert self._member is not None + if team.status != TeamStatus.ACTIVE: + return False + if self.has_global_access and self._member.organization.id == team.organization_id: + return True + return team.id in self.team_ids_with_membership + + def has_project_access(self, project: Project) -> bool: + assert self._member is not None + if project.status != ObjectStatus.ACTIVE: + return False + if self.has_global_access and self._member.organization.id == project.organization_id: + return True + return project.id in self.project_ids_with_team_membership + + +class OrganizationGlobalAccess(DbAccess): + """Access to all an organization's teams and projects.""" + + def __init__( + self, organization: Organization | int, scopes: Iterable[str], **kwargs: Any + ) -> None: + self._organization_id = ( + organization.id if isinstance(organization, Organization) else organization + ) + super().__init__(has_global_access=True, scopes=frozenset(scopes), **kwargs) + + def has_team_access(self, team: Team) -> bool: + return bool( + team.organization_id == self._organization_id and team.status == TeamStatus.ACTIVE + ) + + def has_project_access(self, project: Project) -> bool: + return bool( + project.organization_id == self._organization_id + and project.status == ObjectStatus.ACTIVE + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset( + Team.objects.filter( + organization_id=self._organization_id, status=TeamStatus.ACTIVE + ).values_list("id", flat=True) + ) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset( + Project.objects.filter( + organization_id=self._organization_id, status=ObjectStatus.ACTIVE + ).values_list("id", flat=True) + ) + + +class ServiceAccountAccess(DbAccess): + """ + A service account's access to its organization. It has no membership, so its + scopes are the account's own. With open membership it reaches every active team + and project, like `OrganizationGlobalAccess`. Without it, it reaches only the + active projects linked to the account, and no teams. + """ + + def __init__( + self, + *, + service_account_id: int, + organization_id: int, + scopes: Iterable[str], + has_open_membership: bool, + ) -> None: + self._service_account_id = service_account_id + self._organization_id = organization_id + super().__init__( + sso_is_valid=True, + has_open_membership=has_open_membership, + has_global_access=has_open_membership, + scopes=frozenset(scopes), + ) + + def has_team_access(self, team: Team) -> bool: + return bool( + self.has_global_access + and team.organization_id == self._organization_id + and team.status == TeamStatus.ACTIVE + ) + + def has_project_access(self, project: Project) -> bool: + if ( + project.organization_id != self._organization_id + or project.status != ObjectStatus.ACTIVE + ): + return False + return self.has_global_access or project.id in self.accessible_project_ids + + @cached_property + def project_ids_with_team_membership(self) -> frozenset[int]: + # With no teams, the account's project links stand in for membership, so + # endpoints that default to "my projects" see the projects it can reach. + return self.accessible_project_ids + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + if not self.has_global_access: + return frozenset() + return frozenset( + Team.objects.filter( + organization_id=self._organization_id, status=TeamStatus.ACTIVE + ).values_list("id", flat=True) + ) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + if self.has_global_access: + return frozenset( + Project.objects.filter( + organization_id=self._organization_id, status=ObjectStatus.ACTIVE + ).values_list("id", flat=True) + ) + # A replica link can briefly outlive a transfer of its project to another org. + return frozenset( + ServiceAccountProjectReplica.objects.filter( + service_account__service_account_id=self._service_account_id, + project__organization_id=self._organization_id, + project__status=ObjectStatus.ACTIVE, + ).values_list("project_id", flat=True) + ) + + +class ApiBackedOrganizationGlobalAccess(RpcBackedAccess): + """Access to all an organization's teams and projects.""" + + def __init__( + self, + *, + rpc_user_organization_context: RpcUserOrganizationContext, + auth_state: RpcAuthState, + scopes: Iterable[str] | None, + ): + super().__init__( + rpc_user_organization_context=rpc_user_organization_context, + auth_state=auth_state, + scopes_upper_bound=_wrap_scopes(scopes), + ) + + @cached_property + def scopes(self) -> frozenset[str]: + return frozenset(self.scopes_upper_bound or []) + + @property + def has_global_access(self) -> bool: + return True + + def has_team_access(self, team: Team) -> bool: + return bool( + team.organization_id == self.rpc_user_organization_context.organization.id + and team.status == TeamStatus.ACTIVE + ) + + def has_project_access(self, project: Project) -> bool: + return bool( + project.organization_id == self.rpc_user_organization_context.organization.id + and project.status == ObjectStatus.ACTIVE + ) + + @cached_property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset( + t.id + for t in self.rpc_user_organization_context.organization.teams + if t.status == TeamStatus.ACTIVE + ) + + @cached_property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset( + p.id + for p in self.rpc_user_organization_context.organization.projects + if p.status == ObjectStatus.ACTIVE + ) + + +class OrganizationGlobalMembership(OrganizationGlobalAccess): + """Access to all an organization's teams and projects with simulated membership.""" + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return self.accessible_team_ids + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return self.accessible_project_ids + + def has_team_membership(self, team: Team) -> bool: + return self.has_team_access(team) + + def has_project_membership(self, project: Project) -> bool: + return self.has_project_access(project) + + +class ApiOrganizationGlobalMembership(ApiBackedOrganizationGlobalAccess): + """Access to all an organization's teams and projects with simulated membership.""" + + @property + def is_integration_token(self) -> bool: + return True + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return self.accessible_team_ids + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return self.accessible_project_ids + + def has_team_membership(self, team: Team) -> bool: + return self.has_team_access(team) + + def has_project_membership(self, project: Project) -> bool: + return self.has_project_access(project) + + +@dataclass +class OrganizationlessAccess(Access): + auth_state: RpcAuthState + + @cached_property + def permissions(self) -> frozenset[str]: + return frozenset(self.auth_state.permissions) + + def has_team_access(self, team: Team) -> bool: + return False + + def has_project_access(self, project: Project) -> bool: + return False + + @property + def sso_is_valid(self) -> bool: + return self.auth_state.sso_state.is_valid + + @property + def requires_sso(self) -> bool: + return self.auth_state.sso_state.is_required + + @property + def has_open_membership(self) -> bool: + return False + + @property + def has_global_access(self) -> bool: + return False + + @property + def scopes(self) -> frozenset[str]: + return frozenset() + + # TODO(cathy): remove this + @property + def role(self) -> str | None: + return None + + def has_role_in_organization( + self, role: str, organization: Organization, user_id: int | None + ) -> bool: + if user_id: + return has_role_in_organization(role=role, organization=organization, user_id=user_id) + return False + + @property + def team_ids_with_membership(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset() + + @property + def project_ids_with_team_membership(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset() + + def has_team_scope(self, team: Team, scope: str) -> bool: + check_scope_declaration(scope) + return False + + def get_team_role(self, team: Team) -> TeamRole | None: + return None + + def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: + check_scope_declarations(scopes) + if not self.has_project_access(project): + return False + + return any(self.has_scope(scope) for scope in scopes) + + +class SystemAccess(OrganizationlessAccess): + def __init__(self) -> None: + super().__init__( + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_required=False, is_valid=False), + permissions=[], + ), + ) + + @property + def has_global_access(self) -> bool: + return True + + def has_permission(self, permission: str) -> bool: + return True + + def has_scope(self, scope: str) -> bool: + check_scope_declaration(scope) + return True + + def has_team_access(self, team: Team) -> bool: + return True + + def has_project_access(self, project: Project) -> bool: + return True + + # The semantically correct behavior for accessible_(team|project)_ids would be to + # query for all teams or projects in the system, which we don't want to attempt. + # Code paths that may have SystemAccess must avoid looking at these properties. + @property + def accessible_team_ids(self) -> frozenset[int]: + return frozenset() + + @property + def accessible_project_ids(self) -> frozenset[int]: + return frozenset() + + +class NoAccess(OrganizationlessAccess): + def __init__(self) -> None: + super().__init__( + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_required=False, is_valid=True), + permissions=[], + ), + ) + + +def _propagate_superuser_access( + request: _HttpRequestWithUser, + org_context: RpcUserOrganizationContext, + su: Superuser | None, +) -> None: + # Members use their normal callback permissions and user flags; adding this + # claim would instead force read-only access and a five-minute VC lifetime. + if org_context.member is not None: + return + if request.auth is not None or su is None or not su.is_active: + return + if su.requires_org_auth(org_context.organization): + return + expires_at = get_superuser_access_expiry(su, org_context.organization.id) + if expires_at is not None: + set_viewer_context_superuser( + user_id=request.user.id, + organization_id=org_context.organization.id, + superuser=SuperuserAccess(expires_at=expires_at), + ) + + +def from_request_org_and_scopes( + *, + request: HttpRequest, + rpc_user_org_context: RpcUserOrganizationContext | None = None, + scopes: Iterable[str] | None = None, +) -> Access: + """ + Note that `scopes` is usually None because request.auth is not set at `get_authorization_header` + when the request is made from the frontend using cookies + """ + if isinstance(request.auth, AuthenticatedToken) and request.auth.superuser is not None: + if rpc_user_org_context is None: + return DEFAULT + return from_superuser_access(request.auth, rpc_user_org_context) + if is_agent_auth(request.auth): + if rpc_user_org_context is None: + return DEFAULT + return from_agent_auth(request.auth, rpc_user_org_context) + + is_staff = is_active_staff(request) + + if not rpc_user_org_context: + return from_user_and_rpc_user_org_context( + user=request.user, + rpc_user_org_context=rpc_user_org_context, + is_superuser=is_active_superuser(request), + is_staff=is_staff, + scopes=scopes, + ) + + if getattr(request, "actual_user", None) is not None: + if not should_allow_superuser_access(rpc_user_org_context): + raise DataSecrecyError() + + if getattr(request.user, "is_sentry_app", False): + return _from_rpc_sentry_app(rpc_user_org_context) + + if is_active_superuser(request): + su = getattr(request, "superuser", None) + if su is None and request.auth is None: + su = Superuser(request) + requires_org_auth = su is not None and su.requires_org_auth( + rpc_user_org_context.organization + ) + if requires_org_auth and rpc_user_org_context.member is None: + setattr(request, "_superuser_needs_org_auth", rpc_user_org_context.organization.slug) + + member = rpc_user_org_context.member + auth_state = access_service.get_user_auth_state( + user_id=request.user.id, + organization_id=rpc_user_org_context.organization.id, + is_superuser=True, + is_staff=is_staff, + org_member=member, + ) + + superuser_scopes = get_superuser_scopes(auth_state, rpc_user_org_context) + if scopes: + superuser_scopes = superuser_scopes.union(set(scopes)) + if member and member.scopes: + superuser_scopes = superuser_scopes.union(set(member.scopes)) + + _propagate_superuser_access(request, rpc_user_org_context, su) + + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=auth_state, + scopes=superuser_scopes, + ) + + if request.auth is not None and not has_user(request.user): + return from_rpc_auth(request.auth, rpc_user_org_context) + + return from_user_and_rpc_user_org_context( + user=request.user, + rpc_user_org_context=rpc_user_org_context, + is_superuser=False, + is_staff=is_staff, + scopes=scopes, + ) + + +def organizationless_access( + user: User | RpcUser | AnonymousUser, is_superuser: bool, is_staff: bool +) -> Access: + return OrganizationlessAccess( + auth_state=access_service.get_user_auth_state( + user_id=user.id, + is_superuser=is_superuser, + is_staff=is_staff, + organization_id=None, + org_member=None, + ) + ) + + +def normalize_valid_user(user: User | RpcUser | AnonymousUser | None) -> User | RpcUser | None: + if not user or not has_user(user) or not user.is_active: + return None + return user + + +def from_user_and_rpc_user_org_context( + *, + user: User | AnonymousUser | RpcUser | None, + rpc_user_org_context: RpcUserOrganizationContext | None = None, + is_superuser: bool = False, + is_staff: bool = False, + scopes: Iterable[str] | None = None, + auth_state: RpcAuthState | None = None, +) -> Access: + if (user := normalize_valid_user(user)) is None: + return DEFAULT + + if not rpc_user_org_context or not rpc_user_org_context.member: + return organizationless_access(user, is_superuser, is_staff) + + return from_rpc_member( + rpc_user_organization_context=rpc_user_org_context, + scopes=scopes, + is_superuser=is_superuser, + is_staff=is_staff, + auth_state=auth_state, + ) + + +def from_request( + request: Request, organization: Organization | None = None, scopes: Iterable[str] | None = None +) -> Access: + if isinstance(request.auth, AuthenticatedToken) and ( + is_agent_auth(request.auth) or request.auth.superuser is not None + ): + if organization is None: + return DEFAULT + return from_auth(request.auth, organization) + + is_staff = is_active_staff(request) + + if not organization: + return from_user( + request.user, + organization=organization, + scopes=scopes, + is_superuser=is_active_superuser(request), + is_staff=is_staff, + ) + + if getattr(request, "actual_user", None) is not None: + if not should_allow_superuser_access(organization): + raise DataSecrecyError() + + if request.user.is_authenticated and request.user.is_sentry_app: + return _from_sentry_app(request.user, organization=organization) + + if is_active_superuser(request): + member: OrganizationMember | None = None + try: + member = OrganizationMember.objects.get( + user_id=request.user.id, organization_id=organization.id + ) + except OrganizationMember.DoesNotExist: + pass + auth_state = access_service.get_user_auth_state( + user_id=request.user.id, + organization_id=organization.id, + is_superuser=True, + is_staff=is_staff, + org_member=(summarize_member(member) if member is not None else None), + ) + sso_state = auth_state.sso_state + + superuser_scopes = get_superuser_scopes(auth_state, organization) + if scopes: + superuser_scopes = superuser_scopes.union(set(scopes)) + if member and (member_scopes := member.get_scopes()): + superuser_scopes = superuser_scopes.union(set(member_scopes)) + + return OrganizationGlobalAccess( + organization=organization, + _member=member, + scopes=superuser_scopes, + sso_is_valid=sso_state.is_valid, + requires_sso=sso_state.is_required, + permissions=access_service.get_permissions_for_user(request.user.id), + ) + + if request.auth is not None and not has_user(request.user): + return from_auth(request.auth, organization) + + return from_user(user=request.user, organization=organization, scopes=scopes, is_staff=is_staff) + + +# only used internally +def _from_sentry_app(user: User, organization: Organization | None = None) -> Access: + if not organization: + return NoAccess() + + sentry_app_query = SentryApp.objects.filter(proxy_user=user) + + if not sentry_app_query.exists(): + return NoAccess() + + sentry_app = sentry_app_query.get() + + if not sentry_app.is_installed_on(organization): + return NoAccess() + + return OrganizationGlobalMembership( + organization, translate_deprecated_scopes(sentry_app.scope_list), sso_is_valid=True + ) + + +def _from_rpc_sentry_app(context: RpcUserOrganizationContext | None = None) -> Access: + from sentry.sentry_apps.services.app import app_service + + if not context or context.user_id is None: + return NoAccess() + + installation = app_service.find_installation_by_proxy_user( + proxy_user_id=context.user_id, organization_id=context.organization.id + ) + if installation is None: + return NoAccess() + + return ApiOrganizationGlobalMembership( + rpc_user_organization_context=context, + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState( + is_valid=True, + is_required=False, + ), + permissions=[], + ), + scopes=translate_deprecated_scopes(installation.sentry_app.scope_list), + ) + + +def from_user( + user: User | RpcUser | AnonymousUser | None, + organization: Organization | None = None, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, +) -> Access: + if (user := normalize_valid_user(user)) is None: + return DEFAULT + + if not organization: + return organizationless_access(user, is_superuser, is_staff) + + try: + om = OrganizationMember.objects.get(user_id=user.id, organization_id=organization.id) + except OrganizationMember.DoesNotExist: + return organizationless_access(user, is_superuser, is_staff) + + # ensure cached relation + om.organization = organization + + return from_member(om, scopes=scopes, is_superuser=is_superuser, is_staff=is_staff) + + +def from_member( + member: OrganizationMember, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, +) -> Access: + scope_intersection = _intersect_member_and_token_scopes(member.get_scopes(), scopes) + + if (is_superuser or is_staff) and member.user_id is not None: + # "permissions" is a bit of a misnomer -- these are all admin level permissions, and the intent is that if you + # have them, you can only use them when you are acting, as a superuser or staff. This is intentional. + permissions = access_service.get_permissions_for_user(member.user_id) + else: + permissions = frozenset() + + return OrganizationMemberAccess(member, scope_intersection, permissions, scopes) + + +def from_rpc_member( + rpc_user_organization_context: RpcUserOrganizationContext, + scopes: Iterable[str] | None = None, + is_superuser: bool = False, + is_staff: bool = False, + auth_state: RpcAuthState | None = None, +) -> Access: + if rpc_user_organization_context.user_id is None: + return DEFAULT + + return RpcBackedAccess( + rpc_user_organization_context=rpc_user_organization_context, + scopes_upper_bound=_wrap_scopes(scopes), + auth_state=auth_state + or access_service.get_user_auth_state( + user_id=rpc_user_organization_context.user_id, + organization_id=rpc_user_organization_context.organization.id, + is_superuser=is_superuser, + is_staff=is_staff, + org_member=rpc_user_organization_context.member, + ), + ) + + +def from_auth(auth: AuthenticatedToken, organization: Organization) -> Access: + if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: + context = organization_service.get_organization_by_id( + id=organization.id, user_id=auth.user_id + ) + return from_superuser_access(auth, context) if context is not None else DEFAULT + if is_system_auth(auth): + return SystemAccess() + if is_service_account_auth(auth): + return from_service_account_auth( + auth, organization.id, bool(organization.flags.allow_joinleave) + ) + if is_agent_auth(auth): + access: Access = DEFAULT + if auth.user_id is not None and auth.organization_id == organization.id: + try: + member = OrganizationMember.objects.get( + user_id=auth.user_id, organization_id=organization.id + ) + except OrganizationMember.DoesNotExist: + pass + else: + member.organization = organization + access = from_member(member, scopes=auth.get_scopes()) + return access + auth_organization_id = auth.organization_id + if auth_organization_id is not None and auth_organization_id == organization.id: + return OrganizationGlobalAccess( + auth_organization_id, settings.SENTRY_SCOPES, sso_is_valid=True + ) + else: + return DEFAULT + + +def from_rpc_auth( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: + return from_superuser_access(auth, rpc_user_org_context) + if is_system_auth(auth): + return SystemAccess() + if is_service_account_auth(auth): + organization = rpc_user_org_context.organization + return from_service_account_auth(auth, organization.id, organization.flags.allow_joinleave) + if is_agent_auth(auth): + # Agents are non-user actors with member-derived, capped authority -- never the + # org-global access an org token gets. Dispatched here so the cap holds at the + # shared userless-auth choke, not only in determine_access. + return from_agent_auth(auth, rpc_user_org_context) + if auth.organization_id == rpc_user_org_context.organization.id: + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=RpcAuthState( + permissions=[], + sso_state=RpcMemberSsoState( + is_valid=True, + is_required=False, + ), + ), + scopes=settings.SENTRY_SCOPES, + ) + else: + return DEFAULT + + +def from_agent_auth( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + """Access for a Seer agent capability token: a non-user actor acting on behalf of a + member. Unlike an org token, the agent is not org-global — its authority is the + delegating member's, capped by the token's scopes, so project access follows the + member's teams. The context MUST be resolved for the delegating user_id.""" + # Bound to the org it was minted for; never honored elsewhere, even if the + # delegating user is also a member of the requested org. + if auth.organization_id != rpc_user_org_context.organization.id: + return DEFAULT + if auth.user_id != rpc_user_org_context.user_id: + return DEFAULT + # No membership (never a member, or revoked since mint) -> no access. Required + # explicitly because RpcBackedAccess would otherwise hand back the full token + # scopes uncapped when member is None. + if rpc_user_org_context.member is None: + return DEFAULT + return from_rpc_member(rpc_user_org_context, scopes=auth.get_scopes()) + + +def from_service_account_auth( + auth: AuthenticatedToken, organization_id: int, has_open_membership: bool +) -> Access: + # Bound to the account's own org, like agent tokens. + if auth.entity_id is None or auth.organization_id != organization_id: + return DEFAULT + return ServiceAccountAccess( + service_account_id=auth.entity_id, + organization_id=organization_id, + scopes=auth.get_scopes(), + has_open_membership=has_open_membership, + ) + + +def from_superuser_access( + auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext +) -> Access: + if ( + auth.superuser is None + or auth.organization_id != rpc_user_org_context.organization.id + or auth.user_id != rpc_user_org_context.user_id + ): + return DEFAULT + users = user_service.get_many(filter={"user_ids": [auth.user_id]}) + user = users[0] if users else None + if user is None: + return DEFAULT + delegated = resolve_superuser_access(auth.superuser, user, rpc_user_org_context) + if delegated is None: + return DEFAULT + scopes, _ = delegated + return ApiBackedOrganizationGlobalAccess( + rpc_user_organization_context=rpc_user_org_context, + auth_state=RpcAuthState( + sso_state=RpcMemberSsoState(is_valid=True, is_required=False), permissions=[] + ), + scopes=scopes & set(auth.get_scopes()), + ) + + +DEFAULT = NoAccess() diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py new file mode 100644 index 000000000..ce7956c0e --- /dev/null +++ b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py @@ -0,0 +1,483 @@ +import logging +import random +import string +from email.headerregistry import Address +from typing import Any, TypedDict, TypeIs + +from django.contrib.auth.models import AnonymousUser +from django.db import IntegrityError, router, transaction +from django.db.models import Q +from django.db.models.query import QuerySet +from django.utils.text import slugify +from drf_spectacular.utils import extend_schema +from rest_framework.exceptions import NotAuthenticated, ParseError, PermissionDenied +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.serializers import ValidationError + +from sentry import audit_log +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import cell_silo_endpoint +from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission +from sentry.api.exceptions import ConflictError +from sentry.api.helpers.environments import get_environment_id +from sentry.api.paginator import OffsetPaginator +from sentry.api.permissions import StaffPermissionMixin +from sentry.api.serializers import serialize +from sentry.api.serializers.models.project import ( + OrganizationProjectResponse, + ProjectSummarySerializer, +) +from sentry.apidocs.constants import ( + RESPONSE_BAD_REQUEST, + RESPONSE_CONFLICT, + RESPONSE_FORBIDDEN, + RESPONSE_NOT_FOUND, + RESPONSE_UNAUTHORIZED, +) +from sentry.apidocs.examples.organization_examples import OrganizationExamples +from sentry.apidocs.examples.project_examples import ProjectExamples +from sentry.apidocs.parameters import ( + CursorQueryParam, + GlobalParams, + OrganizationParams, + VisibilityParams, +) +from sentry.apidocs.response_types import DetailResponse +from sentry.apidocs.utils import inline_sentry_response_serializer +from sentry.auth.principal import has_user, require_user +from sentry.constants import ObjectStatus +from sentry.core.endpoints.team_projects import ( + AuditData, + ProjectPostSerializer, + apply_default_project_settings, +) +from sentry.models.organization import Organization +from sentry.models.organizationmember import OrganizationMember +from sentry.models.organizationmemberteam import OrganizationMemberTeam +from sentry.models.project import Project +from sentry.models.team import Team +from sentry.search.utils import tokenize_query +from sentry.seer.agent_token import is_agent_auth +from sentry.signals import project_created, team_created +from sentry.snuba import discover, metrics_enhanced_performance +from sentry.users.models.user import User +from sentry.utils.snowflake import MaxSnowflakeRetryError + +ERR_INVALID_STATS_PERIOD = ( + "Invalid stats_period. Valid choices are '', '1h', '24h', '7d', '14d', '30d', and '90d'" +) + + +class _LegacyParamErrorResponse(TypedDict): + # Legacy nested error envelope used by this endpoint's stats_period / id + # validation; kept as-is for wire compatibility. + error: dict[str, dict[str, dict[str, str]]] + + +class _OrganizationProjectCreateResponse(OrganizationProjectResponse): + """Project-creation response: the base `OrganizationProjectResponse` shape + plus a `team_slug` key that the endpoint appends post-serialize to surface + the auto-created personal team.""" + + team_slug: str + + +# Only applies to the error `stats`; `transactionStats` always queries spans. Kept for +# now until we have the frontend stop sending it, though every option resolves to discover for the error query anyway since the +# metrics datasets is not compatible with `!event.type:transaction`. +DATASETS = { + "": discover, # in case they pass an empty query string fall back on default + "discover": discover, + "metricsEnhanced": metrics_enhanced_performance, + "metrics": metrics_enhanced_performance, +} + +CONFLICTING_TEAM_SLUG_ERROR = "A team with this slug already exists." +DISABLED_FEATURE_ERROR_STRING = "Your organization has disabled this feature for members." + + +def get_dataset(dataset_label: str) -> Any: + if dataset_label not in DATASETS: + raise ParseError(detail=f"dataset must be one of: {', '.join(DATASETS.keys())}") + return DATASETS[dataset_label] + + +def _generate_suffix() -> str: + letters = string.ascii_lowercase + return "".join(random.choice(letters) for _ in range(3)) + + +def fetch_slugifed_email_username(email: str) -> str: + return slugify(Address(addr_spec=email).username) + + +class OrganizationProjectsPermission(StaffPermissionMixin, OrganizationPermission): + scope_map = { + "GET": ["org:read", "org:write", "org:admin"], + # Intentionally lowered: org members can create projects when + # allowMemberProjectCreation is enabled on the org. + "POST": ["project:read", "project:write", "project:admin"], + } + + +@extend_schema(tags=["Organizations"]) +@cell_silo_endpoint +class OrganizationProjectsEndpoint(OrganizationEndpoint): + publish_status = { + "GET": ApiPublishStatus.PUBLIC, + "POST": ApiPublishStatus.PUBLIC, + } + permission_classes = (OrganizationProjectsPermission,) + owner = ApiOwner.FOUNDATIONS + logger = logging.getLogger("team-project.create") + + @extend_schema( + operation_id="listOrganizationProjects", + summary="List an Organization's Projects", + parameters=[ + GlobalParams.ORG_ID_OR_SLUG, + CursorQueryParam, + VisibilityParams.PER_PAGE, + OrganizationParams.PROJECT_QUERY, + ], + request=None, + responses={ + 200: inline_sentry_response_serializer( + "OrganizationProjectResponseDict", list[OrganizationProjectResponse] + ), + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + }, + examples=OrganizationExamples.LIST_PROJECTS, + ) + def get( + self, request: Request, organization: Organization + ) -> ( + Response[list[OrganizationProjectResponse]] + | Response[DetailResponse] + | Response[_LegacyParamErrorResponse] + ): + """ + Return a list of projects bound to a organization. + """ + stats_period = request.GET.get("statsPeriod") + collapse = request.GET.getlist("collapse", []) + if stats_period not in (None, "", "1h", "24h", "7d", "14d", "30d", "90d"): + return Response( + {"error": {"params": {"stats_period": {"message": ERR_INVALID_STATS_PERIOD}}}}, + status=400, + ) + elif not stats_period: + # disable stats + stats_period = None + + datasetName = request.GET.get("dataset", "discover") + dataset = get_dataset(datasetName) + + queryset: QuerySet[Project] + if is_agent_auth(request.auth): + queryset = Project.objects.filter(organization=organization) + if not request.access.has_global_access: + queryset = queryset.filter(id__in=request.access.accessible_project_ids) + elif request.auth and not has_user(request.user): + # TODO: remove this, no longer supported probably + if hasattr(request.auth, "project"): + queryset = Project.objects.filter(id=request.auth.project.id) + elif request.auth.organization_id is not None: + org = request.auth.organization_id + team_list = list(Team.objects.filter(organization_id=org)) + queryset = Project.objects.filter(teams__in=team_list) + else: + return Response( + {"detail": "Current access does not point to organization."}, status=400 + ) + else: + queryset = Project.objects.filter(organization=organization) + + order_by = ["slug"] + + if has_user(request.user): + queryset = queryset.extra( + select={ + "is_bookmarked": """exists ( + select * + from sentry_projectbookmark spb + where spb.project_id = sentry_project.id and spb.user_id = %s + )""" + }, + select_params=(request.user.id,), + ) + order_by.insert(0, "-is_bookmarked") + + query = request.GET.get("query") + if query: + tokens = tokenize_query(query) + for key, value in tokens.items(): + if key == "query": + value_s = " ".join(value) + queryset = queryset.filter( + Q(name__icontains=value_s) | Q(slug__icontains=value_s) + ) + elif key == "id": + if all(v.isdigit() for v in value): + queryset = queryset.filter(id__in=value) + else: + return Response( + { + "error": { + "params": { + "stats_period": { + "message": "All 'id' values must be integers." + } + } + } + }, + status=400, + ) + elif key == "slug": + queryset = queryset.filter(slug__in=value) + elif key == "team": + team_list = list(Team.objects.filter(organization=organization, slug__in=value)) + queryset = queryset.filter(teams__in=team_list) + elif key == "!team": + team_list = list(Team.objects.filter(organization=organization, slug__in=value)) + queryset = queryset.exclude(teams__in=team_list) + elif key == "is_member": + queryset = queryset.filter(teams__organizationmember__user_id=request.user.id) + else: + queryset = queryset.none() + + queryset = queryset.filter(status=ObjectStatus.ACTIVE).distinct() + + # TODO(davidenwang): remove this after frontend requires only paginated projects + get_all_projects = request.GET.get("all_projects") == "1" + + if get_all_projects: + queryset = queryset.order_by("slug").select_related("organization") + return Response( + serialize( + list(queryset), + request.user, + ProjectSummarySerializer(collapse=collapse, dataset=dataset), + ) + ) + else: + expand = set() + if request.GET.get("transactionStats"): + expand.add("transaction_stats") + if request.GET.get("sessionStats"): + expand.add("session_stats") + + expand_context = {"options": request.GET.getlist("options") or []} + if expand_context: + expand.add("options") + + def serialize_on_result(result): + environment_id = get_environment_id(request, organization.id) + serializer = ProjectSummarySerializer( + environment_id=environment_id, + stats_period=stats_period, + expand=expand, + expand_context=expand_context, + collapse=collapse, + dataset=dataset, + ) + return serialize(result, request.user, serializer) + + return self.paginate( + request=request, + queryset=queryset, + order_by=order_by, + on_results=serialize_on_result, + paginator_cls=OffsetPaginator, + ) + + def should_add_creator_to_team(self, user: User | AnonymousUser) -> TypeIs[User]: + return has_user(user) + + @extend_schema( + tags=["Projects"], + operation_id="createOrganizationProject", + summary="Create a Project for an Organization", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request=ProjectPostSerializer, + responses={ + 201: ProjectSummarySerializer, + 400: RESPONSE_BAD_REQUEST, + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + 409: RESPONSE_CONFLICT, + }, + examples=ProjectExamples.CREATE_PROJECT, + description=( + "Create a new project for an organization. A personal team (`team-{username}`) " + "is automatically created for the caller with Team Admin role, and the project is " + "bound to it. If the org has member project creation disabled " + "(`disable_member_project_creation`), `org:write` scope is required." + ), + ) + def post( + self, request: Request, organization: Organization + ) -> Response[_OrganizationProjectCreateResponse]: + """ + Create a new project for an organization. + + Auto-creates a personal team (``team-{username}``) for the caller with Team Admin + role, then creates the project under that team. A random suffix is appended to the + team slug if the default name is already taken (up to five attempts). + """ + serializer = ProjectPostSerializer( + data=request.data, context={"organization": organization} + ) + + if not serializer.is_valid(): + raise ValidationError(serializer.errors) + # The new project's default team is the creator's own, which needs a person. + require_user(request) + if not self.should_add_creator_to_team(request.user): + raise NotAuthenticated("User is not authenticated") + + result = serializer.validated_data + + if organization.flags.disable_member_project_creation and not request.access.has_scope( + "org:write" + ): + raise PermissionDenied(detail=DISABLED_FEATURE_ERROR_STRING) + + # parse the email to retrieve the username before the "@" + parsed_email = fetch_slugifed_email_username(request.user.email) + + project_name = result["name"] + default_team_slug = f"team-{parsed_email}" + suffixed_team_slug = default_team_slug + + # attempt to a maximum of 5 times to add a suffix to team slug until it is unique + for _ in range(5): + if not Team.objects.filter(organization=organization, slug=suffixed_team_slug).exists(): + break + suffixed_team_slug = f"{default_team_slug}-{_generate_suffix()}" + else: + raise ConflictError( + { + "detail": "Unable to create a default team for this user. Please try again.", + } + ) + default_team_slug = suffixed_team_slug + + try: + with transaction.atomic(router.db_for_write(Team)): + team = Team.objects.create( + name=default_team_slug, + slug=default_team_slug, + idp_provisioned=result.get("idp_provisioned", False), + organization=organization, + ) + member = OrganizationMember.objects.get( + user_id=request.user.id, organization=organization + ) + OrganizationMemberTeam.objects.create( + team=team, + organizationmember=member, + role="admin", + ) + project = Project.objects.create( + name=project_name, + # slug is *not* set so we get an automatic one + organization=organization, + platform=result.get("platform"), + ) + project.add_team(team) + except (IntegrityError, MaxSnowflakeRetryError): + raise ConflictError( + { + "non_field_errors": [CONFLICTING_TEAM_SLUG_ERROR], + "detail": CONFLICTING_TEAM_SLUG_ERROR, + } + ) + except OrganizationMember.DoesNotExist: + raise PermissionDenied( + detail="You must be a member of the organization to join a new team as a Team Admin" + ) + + team_created.send_robust( + organization=organization, + user=request.user, + team=team, + sender=self.__class__, + ) + self.create_audit_entry( + request=request, + organization=organization, + target_object=team.id, + event=audit_log.get_event_id("TEAM_ADD"), + data=team.get_audit_log_data(), + ) + + common_audit_data: AuditData = { + "request": request, + "organization": team.organization, + "target_object": project.id, + } + origin = request.data.get("origin") + if origin: + self.create_audit_entry( + **common_audit_data, + event=audit_log.get_event_id("PROJECT_ADD_WITH_ORIGIN"), + data={ + **project.get_audit_log_data(), + "origin": origin, + }, + ) + else: + self.create_audit_entry( + **common_audit_data, + event=audit_log.get_event_id("PROJECT_ADD"), + data={**project.get_audit_log_data()}, + ) + + apply_default_project_settings(organization, project) + + project_created.send_robust( + project=project, + user=request.user, + default_rules=result.get("default_rules", True), + origin=origin, + sender=self, + ) + self.create_audit_entry( + request=request, + organization=team.organization, + event=audit_log.get_event_id("TEAM_AND_PROJECT_CREATED"), + data={"team_slug": default_team_slug, "project_slug": project_name}, + ) + self.logger.info( + "created team through project creation flow", + extra={"team_slug": default_team_slug, "project_slug": project_name}, + ) + base: OrganizationProjectResponse = serialize( + project, request.user, ProjectSummarySerializer(collapse=["unusedFeatures"]) + ) + serialized_response: _OrganizationProjectCreateResponse = { + **base, + "team_slug": team.slug, + } + return Response(serialized_response, status=201) + + +@cell_silo_endpoint +class OrganizationProjectsCountEndpoint(OrganizationEndpoint): + publish_status = { + "GET": ApiPublishStatus.PRIVATE, + } + + def get(self, request: Request, organization: Organization) -> Response: + queryset = Project.objects.filter(organization=organization) + + all_projects = queryset.count() + my_projects = queryset.filter(teams__organizationmember__user_id=request.user.id).count() + + return Response({"allProjects": all_projects, "myProjects": my_projects}) diff --git a/packages/evals/src/index.test.ts b/packages/evals/src/index.test.ts index 91de072c8..392ec4631 100644 --- a/packages/evals/src/index.test.ts +++ b/packages/evals/src/index.test.ts @@ -231,6 +231,7 @@ describe('standalone scenario files', () => { ...discoverEvalScenarioFiles('code-review', evalsDir), ...discoverEvalScenarioFiles('security-review', evalsDir), ...discoverEvalScenarioFiles('verification', evalsDir), + ...discoverEvalScenarioFiles('wrdn-authz', evalsDir), ]; const missingMetadata: string[] = []; diff --git a/packages/evals/src/security-matrix.eval.ts b/packages/evals/src/security-matrix.eval.ts new file mode 100644 index 000000000..0240ec13d --- /dev/null +++ b/packages/evals/src/security-matrix.eval.ts @@ -0,0 +1,92 @@ +import { join } from 'node:path'; +import { expect } from 'vitest'; +import { describeEval } from 'vitest-evals'; +import { resolveSkillAsync } from '@sentry/warden'; +import { + createWardenEvalHarness, + createWardenEvalJudge, + WardenEvalOutputSchema, +} from './harness.js'; +import { discoverEvalScenarios } from './index.js'; +import { formatEvalId, formatEvalTestName } from './names.js'; +import { resolveEvalProvider } from './provider.js'; +import type { EvalMeta } from './types.js'; + +/** + * Pinned so a skill edit in warden-skills cannot silently move the matrix. + * Bump deliberately and rerun with `run-evals` to record new results. + */ +const WARDEN_SKILLS_REMOTE = 'getsentry/warden-skills@daf01f970a28450627a8d2d61d9915a742a735e2'; + +/** Remote security skill under test. Its scenarios live in the category of the same name. */ +const WRDN_SKILL = 'wrdn-authz'; + +const SECURITY_REVIEW_SKILL = '../warden/src/builtin-skills/security-review/SKILL.md'; +const MATRIX_RUN_TIMEOUT_MS = 240_000; +const MATRIX_EVAL_TIMEOUT_MS = MATRIX_RUN_TIMEOUT_MS + 60_000; + +const provider = resolveEvalProvider(); +let wrdnSkillPath: Promise | undefined; + +// Fetch lazily inside tests so suites without credentials never touch the network. +function resolveWrdnSkillPath(): Promise { + wrdnSkillPath ??= resolveSkillAsync(WRDN_SKILL, undefined, { remote: WARDEN_SKILLS_REMOTE }).then((skill) => { + if (!skill.rootDir) { + throw new Error(`Remote skill ${WRDN_SKILL} resolved without a rootDir`); + } + return join(skill.rootDir, 'SKILL.md'); + }); + return wrdnSkillPath; +} + +function describeLane(lane: string, evals: EvalMeta[], resolveMeta: (meta: EvalMeta) => Promise): void { + describeEval( + lane, + { + harness: createWardenEvalHarness({ + apiKey: provider.apiKey, + model: provider.model, + runtime: provider.runtime, + auxiliaryModel: provider.auxiliaryModel, + timeoutMs: MATRIX_RUN_TIMEOUT_MS, + verbose: true, + }), + judges: [createWardenEvalJudge({ apiKey: provider.apiKey, model: provider.judgeModel })], + judgeThreshold: 1, + skipIf: () => !provider.enabled, + }, + (it) => { + for (const scenario of evals) { + it( + formatEvalTestName(scenario), + { timeout: MATRIX_EVAL_TIMEOUT_MS }, + async ({ run }) => { + const meta = await resolveMeta(scenario); + const result = await run(meta); + const output = WardenEvalOutputSchema.parse(result.output); + + expect(output.name).toBe(formatEvalId(meta)); + console.log(`\n${output.summary}`); + console.log(` Findings: ${output.findings.length}`); + }, + ); + } + }, + ); +} + +// Each scenario runs under security-review and under the wrdn skill that owns the +// category. A scenario only the wrdn lane passes is coverage security-review lacks. +const evals = discoverEvalScenarios({ + category: WRDN_SKILL, + skill: SECURITY_REVIEW_SKILL, + runtime: 'pi', + model: 'anthropic/claude-sonnet-4-6', +}); + +describeLane(`security-matrix/${WRDN_SKILL}/security-review`, evals, async (meta) => meta); +describeLane( + `security-matrix/${WRDN_SKILL}/${WRDN_SKILL}`, + evals.map((meta) => ({ ...meta, skillName: WRDN_SKILL })), + async (meta) => ({ ...meta, skillPath: await resolveWrdnSkillPath() }), +); diff --git a/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json b/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json new file mode 100644 index 000000000..b4bbd4280 --- /dev/null +++ b/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json @@ -0,0 +1,30 @@ +{ + "given": "SCIM permission class only checks that the org's auth provider has SCIM enabled, so any member or token with member/team scopes can call SCIM endpoints, not just the SCIM identity provider", + "files": [ + "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py" + ], + "should_find": [ + { + "finding": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams" + } + ], + "should_not_find": [ + "SCIM filter parsing", + "pagination parameter validation" + ], + "supporting_files": [ + "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/pull/130141", + "repository": "getsentry/sentry", + "source_ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2", + "source_files": [ + { + "fixturePath": "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py", + "sourcePath": "src/sentry/core/endpoints/scim/utils.py", + "ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2" + } + ] + } +} diff --git a/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json b/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json new file mode 100644 index 000000000..15b5127b2 --- /dev/null +++ b/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json @@ -0,0 +1,36 @@ +{ + "given": "Seer workflow config PUT validates only the project IDs in the request and then replaces the whole project selection, so a limited member can unlink projects they cannot access or narrow an all-projects workflow", + "files": [ + "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py" + ], + "should_find": [ + { + "finding": "PUT replaces the workflow's project selection with the caller-supplied IDs after checking access only to those IDs, which deletes existing links to projects the caller cannot access and lets a member without global access convert an all_projects workflow to a narrower selection" + } + ], + "should_not_find": [ + "monitor selection type validation", + "config ID parsing" + ], + "supporting_files": [ + "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py", + "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/pull/129734", + "repository": "getsentry/sentry", + "source_ref": "d74d48356f2c822f69463c41922823908683e084", + "source_files": [ + { + "fixturePath": "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py", + "sourcePath": "src/sentry/seer/endpoints/organization_seer_workflow_configs.py", + "ref": "d74d48356f2c822f69463c41922823908683e084" + }, + { + "fixturePath": "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py", + "sourcePath": "src/sentry/seer/workflows/workflow.py", + "ref": "d74d48356f2c822f69463c41922823908683e084" + } + ] + } +} diff --git a/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json b/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json new file mode 100644 index 000000000..4f198c309 --- /dev/null +++ b/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json @@ -0,0 +1,36 @@ +{ + "given": "service account access maps all_projects to has_global_access, and team membership creation allows any caller with has_global_access, so an org:read service account can add members to any team", + "files": [ + "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py", + "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py" + ], + "should_find": [ + { + "finding": "A service account with all_projects gets has_global_access, and _can_create_team_member allows has_global_access callers, so an org:read service account can add an organization member to any team even though all_projects is meant to grant project access only" + } + ], + "should_not_find": [ + "team role escalation through team admin roles", + "member invite email handling" + ], + "supporting_files": [ + "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/pull/129343", + "repository": "getsentry/sentry", + "source_ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034", + "source_files": [ + { + "fixturePath": "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py", + "sourcePath": "src/sentry/auth/access.py", + "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034" + }, + { + "fixturePath": "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py", + "sourcePath": "src/sentry/core/endpoints/organization_member_team_details.py", + "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034" + } + ] + } +} diff --git a/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json b/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json new file mode 100644 index 000000000..2ae296348 --- /dev/null +++ b/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json @@ -0,0 +1,36 @@ +{ + "given": "organization projects index sends non-human org-scoped auth through a branch that lists every project in the org, so a service account without all_projects sees projects it is not linked to", + "files": [ + "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py" + ], + "should_find": [ + { + "finding": "GET organization projects returns every project in the organization to a service account without global access, because the non-human auth branch does not filter by request.access.accessible_project_ids the way the agent-token branch does" + } + ], + "should_not_find": [ + "stats_period validation", + "project creation rate limits" + ], + "supporting_files": [ + "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py", + "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/pull/129154", + "repository": "getsentry/sentry", + "source_ref": "595efb30334e9427e572792031929c0298d24703", + "source_files": [ + { + "fixturePath": "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py", + "sourcePath": "src/sentry/core/endpoints/organization_projects.py", + "ref": "595efb30334e9427e572792031929c0298d24703" + }, + { + "fixturePath": "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py", + "sourcePath": "src/sentry/auth/access.py", + "ref": "595efb30334e9427e572792031929c0298d24703" + } + ] + } +} From 12d141d5166013265f9207d3bfebe226919fa15f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:18:30 +0000 Subject: [PATCH 02/11] feat(evals): Extend security matrix to code-exec, data-exfil, and dos skills Add scenario categories for wrdn-code-execution, wrdn-data-exfil, and wrdn-dos-review, each run under the skill and under security-review. Bug fixtures come from pre-fix parents of real getsentry/sentry fixes and corpus entries; clean fixtures use the fixed state of the same file to count false positives. Print a per-skill summary after the matrix runs, led by the number of bugs only the wrdn skill caught. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/evals/INTERNAL.md | 28 +- .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/scripts/analyze-styled.ts | 1655 ++++++++++++++++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../api/endpoints/data_forwarding_index.py | 101 + .../api/serializers/models/data_forwarder.py | 119 ++ .../integrations/models/data_forwarder.py | 38 + .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/eventstream/item_helpers.py | 116 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/eventstream/item_helpers.py | 106 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/src/sentry/search/eap/resolver.py | 1668 +++++++++++++++++ .../.github/workflows/getsentry-dispatch.yml | 96 + .../workflows/scripts/parse-pr-files.py | 50 + .../github/getsentry/sentry/LICENSE.md | 105 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/grouping/parameterization.py | 695 +++++++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/grouping/parameterization.py | 696 +++++++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../api/endpoints/internal/feature_flags.py | 71 + .../github/getsentry/sentry/LICENSE.md | 105 ++ .../api/endpoints/internal/feature_flags.py | 117 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/api/endpoints/system_options.py | 142 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../sentry/api/endpoints/system_options.py | 141 ++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/uptime/autodetect/tasks.py | 303 +++ .../github/getsentry/sentry/LICENSE.md | 105 ++ .../src/sentry/uptime/autodetect/tasks.py | 291 +++ packages/evals/src/index.test.ts | 3 + packages/evals/src/security-matrix.eval.ts | 105 +- packages/evals/src/security-matrix.test.ts | 54 + packages/evals/src/security-matrix.ts | 71 + ...analyze-styled-branch-shell-injection.json | 30 + ...ntry-dispatch-github-output-injection.json | 36 + ...ternal-feature-flags-config-injection.json | 30 + ...ernal-feature-flags-restricted-values.json | 31 + ...forwarder-serializer-decrypted-config.json | 42 + .../sentry-system-options-logs-redacted.json | 31 + ...try-system-options-logs-secret-values.json | 30 + .../sentry-uptime-robots-txt-safe-fetch.json | 31 + .../sentry-uptime-robots-txt-ssrf.json | 30 + .../sentry-eap-item-encode-depth-limited.json | 31 + ...y-eap-item-encode-unbounded-recursion.json | 30 + ...-eap-search-boolean-nesting-recursion.json | 30 + ...entry-grouping-url-regex-backtracking.json | 30 + ...try-grouping-url-regex-bounded-scheme.json | 31 + 50 files changed, 8538 insertions(+), 41 deletions(-) create mode 100644 packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts create mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py create mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py create mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py create mode 100644 packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py create mode 100644 packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py create mode 100644 packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py create mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml create mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py create mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py create mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py create mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py create mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py create mode 100644 packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py create mode 100644 packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py create mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py create mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md create mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py create mode 100644 packages/evals/src/security-matrix.test.ts create mode 100644 packages/evals/src/security-matrix.ts create mode 100644 packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json create mode 100644 packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json create mode 100644 packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json create mode 100644 packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json create mode 100644 packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json create mode 100644 packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json create mode 100644 packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json create mode 100644 packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json create mode 100644 packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json create mode 100644 packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json create mode 100644 packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json create mode 100644 packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json create mode 100644 packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json create mode 100644 packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json diff --git a/packages/evals/INTERNAL.md b/packages/evals/INTERNAL.md index 4fb11b473..45ed12ea2 100644 --- a/packages/evals/INTERNAL.md +++ b/packages/evals/INTERNAL.md @@ -79,12 +79,26 @@ and `security-matrix//`. A case only the `wrdn-*` lane passes is coverage that skill adds. A skill whose cases security-review also passes is redundant on that evidence. +- After the lanes finish, the suite prints a `Security matrix` block. Per + skill, the first line is the number of bugs caught only by that skill. Zero + across a real fixture set is the evidence that the skill adds nothing + security-review does not already catch. +- Bug scenarios come from real code: the parent of a fix commit, a corpus + entry, or a finding only the `wrdn-*` skill posted on a PR. Write + `should_find` for the real bug even when a lane misses it today. +- Clean scenarios use the fixed state of the same file, with no required + `should_find` and the fixed bug in `should_not_find`. They count false + positives per lane. +- Fixtures are whole files, so a lane sees the vulnerable code but not the + diff that introduced it. A bug that only shows as a removed check is harder + to catch here than on the real PR. +- The matrix ignores `warden.toml` path filters. A catch in `scripts/` or + `.github/` proves the skill can find it, not that the production config + would run the skill there. - Remote skills load from the pinned `WARDEN_SKILLS_REMOTE` ref through Warden's normal remote skill cache. Bump the SHA on purpose and rerun. -- Seed scenarios from real findings that only the `wrdn-*` skill posted on a - PR. Write `should_find` for the real bug even when a lane misses it today. -- These cases are expected to fail in the security-review lane. That lowers the - aggregate eval score by design; read the per-lane results, not the average. +- Some cases are expected to fail in one lane. That lowers the aggregate eval + score by design; read the matrix block, not the average. - Roach records them like any other eval. Add `run-evals` to record new cases. **Vulnerability corpus (repeated benchmark).** The @@ -98,10 +112,8 @@ It is not a CI eval: runs cost real money and are scored by hand. 3 of 3 times that drops to 0 of 3 is a regression; a 1 of 3 entry moving is not evidence. - The corpus is mostly authorization bugs, with a few webhook, crypto, - injection, and secret-logging entries. It has close to nothing for DoS, data - exfiltration, or code execution, so `wrdn-dos-review`, `wrdn-data-exfil`, - and `wrdn-code-execution` need their own fixtures in the matrix before any - claim about their coverage holds. + injection, and secret-logging entries, and no DoS entries. It only scores + security-review, so `wrdn-*` coverage claims come from the matrix. ## Adding Full-Pipeline Evals diff --git a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts new file mode 100644 index 000000000..2dafee92c --- /dev/null +++ b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts @@ -0,0 +1,1655 @@ +#!/usr/bin/env node +'use strict'; + +import * as child_process from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import * as ts from 'typescript'; + +// Main execution +// analyze-styled.ts is a script that analyzes styled components in TypeScript React files. +// It can be invoked via node analyze-styled.ts and accepts the following options: + +// Execute with: node scripts/analyze-styled.ts + +// -- : Analyze a specific .tsx file instead of searching directories +// -n : Show top N most styled components (default: 10) +// -c : Filter analysis to specific components (comma-separated) +// -l : Show file locations for styled components (requires -c) +// -debug : Enable debug logging +// -g : Use glob pattern to analyze only a subset of files +// --out csv : Output results in CSV format +// : Directory to search for .tsx files (default: './static/app') + +// Examples: +// node scripts/analyze-styled.ts -- ./path/to/file.tsx +// node scripts/analyze-styled.ts -n 20 -c div,span ./static/app +// node scripts/analyze-styled.ts -l -c div ./static/app +// node scripts/analyze-styled.ts -g 'static/app/components/core/**/*.tsx' +// node scripts/analyze-styled.ts --out csv ./static/app + +class Logger { + private debugEnabled: boolean; + + constructor(debugEnabled: boolean) { + this.debugEnabled = debugEnabled; + } + + log(...args: any[]): void { + console.log(...args); + } + + debug(...args: any[]): void { + if (this.debugEnabled) { + console.log(...args); + } + } + + error(...args: any[]): void { + console.error(...args); + } + + fatal(...args: any[]): void { + console.error(...args); + } + + table(...args: any[]): void { + console.table(...args); + } + + setDebugEnabled(enabled: boolean): void { + this.debugEnabled = enabled; + } +} + +// Global logger instance, initialized with debug disabled +const logger = new Logger(false); + +// Metadata collection interfaces +interface Metadata { + analyzedFiles: number; + gitBranch: string | null; + gitCommit: string | null; + searchDirectory: string; + timestamp: string; + totalFiles: number; +} + +// Metadata collection functions +function getGitInfo(): { + branch: string | null; + commit: string | null; + status: string | null; +} { + try { + const commit = child_process + .execSync('git rev-parse HEAD', {encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe']}) + .trim(); + + const branch = child_process + .execSync('git rev-parse --abbrev-ref HEAD', { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }) + .trim(); + + const status = child_process + .execSync('git status --porcelain', { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }) + .trim(); + + return {commit, branch, status: status || 'clean'}; + } catch (error) { + logger.debug('❌ Error getting git info:', (error as Error).message); + return {commit: null, branch: null, status: null}; + } +} + +function collectMetadata( + searchDir: string, + totalFiles: number, + analyzedFiles: number +): Metadata { + const gitInfo = getGitInfo(); + + return { + timestamp: new Date().toISOString(), + gitCommit: gitInfo.commit, + gitBranch: gitInfo.branch, + totalFiles, + analyzedFiles, + searchDirectory: searchDir, + }; +} + +function outputMetadata(metadata: Metadata): void { + if (config.outputFormat === 'csv') { + logger.log('=== METADATA ==='); + const metadataData = [ + {Key: 'Timestamp', Value: metadata.timestamp}, + {Key: 'GitCommit', Value: metadata.gitCommit || 'unknown'}, + {Key: 'GitBranch', Value: metadata.gitBranch || 'unknown'}, + {Key: 'TotalFiles', Value: metadata.totalFiles || 0}, + {Key: 'AnalyzedFiles', Value: metadata.analyzedFiles || 0}, + ]; + logger.log(arrayToCSV(metadataData)); + logger.log(); + } else { + logger.log('\n📋 Analysis Metadata:\n'); + logger.log(` • Timestamp: ${metadata.timestamp}`); + logger.log(` • Git Commit: ${metadata.gitCommit || 'unknown'}`); + logger.log(` • Git Branch: ${metadata.gitBranch || 'unknown'}`); + logger.log(` • Total Files: ${metadata.totalFiles || 0}`); + logger.log(` • Analyzed Files: ${metadata.analyzedFiles || 0}`); + } +} + +// Git history functions +interface GitCommit { + date: string; + hash: string; + timestamp: number; +} + +function getCommitsInDateRange(startDate: string, intervalDays: number): GitCommit[] { + // Precompute interval dates to avoid processing too many commits + const intervalDates: string[] = []; + const start = new Date(startDate); + const now = new Date(); + + let currentDate = new Date(start); + + while (currentDate <= now) { + intervalDates.push(currentDate.toISOString().split('T')[0]!); + currentDate = new Date(currentDate.getTime() + intervalDays * 24 * 60 * 60 * 1000); + } + + logger.log(`Getting first commit for ${intervalDates.length} interval dates...`); + + const commits: GitCommit[] = []; + + // Get the first commit for each interval date + for (const date of intervalDates) { + try { + const nextDate = new Date(date); + nextDate.setDate(nextDate.getDate() + 1); + const nextDateStr = nextDate.toISOString().split('T')[0]!; + + const output = child_process.execSync( + `git log --since="${date}" --until="${nextDateStr}" --format="%H,%ct" --reverse -n 1`, + {encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe']} + ); + + const line = output.trim(); + if (line.length > 0) { + const [hash, timestamp] = line.split(','); + commits.push({ + hash: hash!, + date, + timestamp: parseInt(timestamp!, 10), + }); + } + } catch (error) { + // Skip dates with no commits + continue; + } + } + + return commits; +} + +// CSV helper functions +function escapeCsvField(field: string | number): string { + if (typeof field === 'number') return field.toString(); + if (field === null || field === undefined) return ''; + const fieldStr = field.toString(); + if (fieldStr.includes(',') || fieldStr.includes('"') || fieldStr.includes('\n')) { + return `"${fieldStr.replace(/"/g, '""')}"`; + } + return fieldStr; +} + +function arrayToCSV(data: Array>): string { + if (data.length === 0) return ''; + + const headers = Object.keys(data[0] ?? {}); + const csvHeaders = headers.map(escapeCsvField).join(','); + + const csvRows = data.map(row => + headers + .map(header => { + const value = row[header]; + return escapeCsvField(value !== null && value !== undefined ? value : ''); + }) + .join(',') + ); + + return [csvHeaders, ...csvRows].join('\n'); +} +interface Config { + components: Set | null; + debug: boolean; + interval: number; + outdir: string; + outputFormat: 'table' | 'csv'; + searchDir: string | null; + showLocations: boolean; + startDate: string | null; + targetFile: string | null; + topN: number; + useGlob: boolean; +} + +function parseArguments(args: string[]): Config { + const config: Config = { + debug: false, + topN: 10, + outputFormat: 'table', + searchDir: null, + targetFile: null, + showLocations: false, + useGlob: false, + components: null, + startDate: null, + interval: 7, + outdir: '/tmp/analyze-styled-output', + }; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--' && i + 1 < args.length) { + config.targetFile = args[i + 1] ?? null; + break; + } else if (args[i] === '-n' && i + 1 < args.length) { + const nextArgV = args[i + 1]; + if (!nextArgV) { + throw new Error('-n option requires a number argument'); + } + const nValue = parseInt(nextArgV, 10); + if (isNaN(nValue) || nValue <= 0) { + throw new Error('-n option must be a positive number'); + } + config.topN = nValue; + i++; + } else if (args[i] === '-c' && i + 1 < args.length) { + config.components = config.components || new Set(); + const componentsArg = args[i + 1]?.split(',') ?? []; + componentsArg.map(c => c.trim()).forEach(c => config.components!.add(c)); + i++; + } else if (args[i] === '-l') { + config.showLocations = true; + } else if (args[i] === '-g') { + config.useGlob = true; + } else if (args[i] === '-debug') { + config.debug = true; + } else if (args[i] === '--out') { + if (i + 1 >= args.length) { + throw new Error( + '--out option requires a format argument. Only "csv" is supported.' + ); + } + const format = args[i + 1]; + if (format === 'csv') { + config.outputFormat = 'csv'; + } else { + throw new Error(`Invalid output format: ${format}. Only 'csv' is supported.`); + } + i++; + } else if (args[i] === '--start-date' && i + 1 < args.length) { + config.startDate = args[i + 1] ?? null; + i++; + } else if (args[i] === '--interval' && i + 1 < args.length) { + const intervalValue = parseInt(args[i + 1] ?? '7', 10); + if (isNaN(intervalValue) || intervalValue <= 0) { + throw new Error('--interval option must be a positive number'); + } + config.interval = intervalValue; + i++; + } else if (args[i] === '--outdir' && i + 1 < args.length) { + config.outdir = args[i + 1] ?? './history-output'; + i++; + } else if (!config.searchDir) { + config.searchDir = args[i] ?? null; + } + } + + return config; +} + +function validateConfig(config: Config): void { + if (config.showLocations && !config.components) { + throw new Error( + '-l option requires -c to specify components filter, otherwise it will analyze all styled components and produce a lot of output.' + ); + } + + if (config.useGlob && !config.searchDir) { + throw new Error('-g option requires a glob pattern argument'); + } + + // Default to './static/app' if no directory specified + if (!config.searchDir && !config.targetFile) { + config.searchDir = './static/app'; + } +} + +function findTsxFiles(dir: string): string[] { + try { + const output = child_process.execSync( + `rg --files --type-add 'tsx:*.tsx' --type tsx "${dir}"`, + { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + } + ); + + return output + .trim() + .split('\n') + .filter(file => file.length > 0); + } catch (error) { + logger.debug( + `❌ Error running ripgrep: ${(error as Error).message}, falling back to fs` + ); + // Fallback to original implementation if ripgrep fails or is missing + const files = fs.readdirSync(dir); + const results: string[] = []; + + for (const file of files) { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + + if (stat.isDirectory() && !file.startsWith('.') && file !== 'node_modules') { + results.push(...findTsxFiles(filePath)); + } else if (file.endsWith('.tsx')) { + results.push(filePath); + } + } + + return results; + } +} + +function collectTsxFilesForCurrentState(c: Config): string[] { + if (c.targetFile) { + if (!fs.existsSync(c.targetFile)) { + logger.debug(`❌ Target file not found: ${c.targetFile}`); + return []; + } + if (!c.targetFile.endsWith('.tsx')) { + logger.debug(`❌ Target file is not a .tsx file: ${c.targetFile}`); + return []; + } + return [c.targetFile]; + } + const searchDir = c.searchDir || './static/app'; + + if (!fs.existsSync(searchDir)) { + logger.debug(`❌ Search directory not found: ${searchDir}`); + return []; + } + + if (c.useGlob) { + try { + return fs.globSync(searchDir); + } catch (error) { + logger.debug(`❌ Error with glob pattern: ${(error as Error).message}`); + return []; + } + } else { + return findTsxFiles(searchDir); + } +} + +const config = parseArguments(process.argv.slice(2)); +validateConfig(config); + +// Update global logger with debug setting +logger.setDebugEnabled(config.debug); + +// For non-historical analysis, collect files at startup +// For historical analysis, files will be collected per commit +let tsxFiles: string[] = []; + +if (!config.startDate) { + // Only collect files at startup for non-historical analysis + tsxFiles = collectTsxFilesForCurrentState(config); + + // For non-historical analysis, exit if no files found + if (tsxFiles.length === 0) { + if (config.targetFile) { + logger.fatal(`❌ File not found: ${config.targetFile}`); + } else { + logger.fatal('❌ No .tsx files found in search directory'); + } + process.exit(1); + } +} + +// Detector configuration interfaces +interface DetectorContext { + config: Config; + fileName: string; + sourceFile: ts.SourceFile; +} + +abstract class BaseDetector { + abstract name: string; + abstract execute(node: ts.Node, context: DetectorContext): void; + abstract results(): void; + abstract reset(): void; +} + +interface DetectorConfiguration { + detectors: BaseDetector[]; +} + +// Detector classes + +class CoreComponentImportsDetector extends BaseDetector { + name = 'CoreComponents'; + private components = new Set(); + private usage = new Map}>(); + + execute(node: ts.Node, context: DetectorContext): void { + // Handle import declarations + if (ts.isImportDeclaration(node)) { + const importDecl = node; + const moduleSpecifier = importDecl.moduleSpecifier; + + if (moduleSpecifier.kind === ts.SyntaxKind.StringLiteral) { + const moduleName = (moduleSpecifier as ts.StringLiteral).text; + if ( + moduleName.includes('sentry/components/core') && + importDecl.importClause?.namedBindings + ) { + const namedBindings = importDecl.importClause.namedBindings; + if (namedBindings.kind === ts.SyntaxKind.NamedImports) { + const namedImports = namedBindings; + namedImports.elements.forEach(element => { + this.components.add(element.name.text); + }); + } + } + } + } + + // Handle JSX element usage + if (ts.isJsxElement(node)) { + const jsxElement = node; + if (jsxElement.openingElement.tagName.kind === ts.SyntaxKind.Identifier) { + const tagName = jsxElement.openingElement.tagName.text; + + if (this.components.has(tagName)) { + const usage = this.usage.get(tagName); + if (usage) { + usage.files.add(context.fileName); + usage.count++; + } else { + this.usage.set(tagName, { + files: new Set([context.fileName]), + count: 1, + }); + } + } + } + } + } + + reset(): void { + this.components.clear(); + this.usage.clear(); + } + + results(): void { + if (this.usage.size === 0) { + // Early return for empty usage - no computation needed + if (config.outputFormat === 'csv') { + logger.log('=== CORE COMPONENT USAGE (LAYOUT) ==='); + logger.log(arrayToCSV([{Type: 'Layout', Component: '', Instances: 0}])); + logger.log(); + + logger.log('=== CORE COMPONENT USAGE (TEXT) ==='); + logger.log(arrayToCSV([{Type: 'Text', Component: '', Instances: 0}])); + logger.log(); + + logger.log('=== CORE COMPONENT USAGE (ALL COMPONENTS) ==='); + logger.log(arrayToCSV([{Component: '', Instances: 0}])); + logger.log(); + + logger.log('=== CORE COMPONENT USAGE (SUMMARY) ==='); + logger.log( + arrayToCSV([ + {Component: 'Total Files', Instances: 0}, + {Component: 'Core Components', Instances: 0}, + ]) + ); + logger.log(); + } else { + logger.log('\n🧩 Core Component Usage (from sentry/components/core):'); + logger.log('No core component usage found.'); + } + return; + } + + // Parse and categorize components + const layout = Array.from(this.usage.entries()) + .filter( + ([component]) => + component.includes('Flex') || + component.includes('Grid') || + component.includes('Stack') || + component.includes('Container') + ) + .sort((a, b) => b[1].count - a[1].count); + + const text = Array.from(this.usage.entries()) + .filter( + ([component]) => component.includes('Text') || component.includes('Heading') + ) + .sort((a, b) => b[1].count - a[1].count); + + // Prepare data structures for both output formats + const layoutData = layout.map(([component, {count, files: _files}]) => ({ + Type: 'Layout', + Component: component, + Instances: count, + })); + + const textData = text.map(([component, {count, files: _files}]) => ({ + Type: 'Text', + Component: component, + Instances: count, + })); + + const totalCount = Array.from(this.usage.values()).reduce( + (sum, {count}) => sum + count, + 0 + ); + + // Prepare all components data sorted by usage + const allComponentsData = Array.from(this.usage.entries()) + .map(([component, {count, files: _files}]) => ({ + Component: component, + Instances: count, + })) + .sort((a, b) => b.Instances - a.Instances); + + if (config.outputFormat === 'csv') { + // Always output layout section (with empty data if no results) + logger.log('=== CORE COMPONENT USAGE (LAYOUT) ==='); + logger.log( + arrayToCSV( + layoutData.length > 0 + ? layoutData + : [{Type: 'Layout', Component: '', Instances: 0}] + ) + ); + logger.log(); + + // Always output text section (with empty data if no results) + logger.log('=== CORE COMPONENT USAGE (TEXT) ==='); + logger.log( + arrayToCSV( + textData.length > 0 ? textData : [{Type: 'Text', Component: '', Instances: 0}] + ) + ); + logger.log(); + + // Always output all components listing + logger.log('=== CORE COMPONENT USAGE (ALL COMPONENTS) ==='); + logger.log( + arrayToCSV( + allComponentsData.length > 0 + ? allComponentsData + : [{Component: '', Instances: 0}] + ) + ); + logger.log(); + + // Always output summary + logger.log('=== CORE COMPONENT USAGE (SUMMARY) ==='); + logger.log( + arrayToCSV([ + {Component: 'Total Files', Instances: tsxFiles.length || 0}, + {Component: 'Core Components', Instances: totalCount || 0}, + ]) + ); + logger.log(); + } else { + logger.log('\n🧩 Core Component Usage (from sentry/components/core):\n'); + logger.table(layoutData); + logger.table(textData); + logger.log('\n📋 All Core Components:\n'); + logger.table(allComponentsData); + } + } +} + +interface StyledComponent { + component: string; + componentType: 'intrinsic' | 'component' | 'unknown'; + cssRules: string; + expressionCount: number; + file: string; + hasExpressions: boolean; + location: { + column: number; + line: number; + }; +} +class StyledComponentsDetector extends BaseDetector { + name = 'StyledComponents'; + private styledComponents: StyledComponent[] = []; + private componentCounts = new Map(); + private cssRuleCounts = new Map(); + + execute(node: ts.Node, context: DetectorContext): void { + if (!ts.isTaggedTemplateExpression(node)) return; + + const taggedExpr = node; + + if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { + const callExpr = taggedExpr.tag as ts.CallExpression; + + if (callExpr.expression.getText() === 'styled') { + const component = callExpr.arguments[0]; + if (!component) return; + + let componentName = ''; + let componentType: 'intrinsic' | 'component' | 'unknown' = 'unknown'; + + // Extract component name and type + if ( + component.kind === ts.SyntaxKind.StringLiteral || + component.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral + ) { + componentName = (component as ts.StringLiteral).text; + componentType = 'intrinsic'; + } else if (component.kind === ts.SyntaxKind.Identifier) { + componentName = (component as ts.Identifier).text; + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.PropertyAccessExpression) { + componentName = component.getText(); + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.ArrowFunction) { + componentName = 'InlineArrowFunction'; + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.AsExpression) { + const asExpr = component as ts.AsExpression; + componentName = asExpr.expression.getText(); + componentType = 'component'; + } else if ( + component.kind === ts.SyntaxKind.FunctionExpression || + component.kind === ts.SyntaxKind.CallExpression + ) { + componentName = 'InlineFunction'; + componentType = 'component'; + } else { + logger.debug( + 'Debug: ', + component.kind, + 'Text:', + component.getText(), + 'File:', + context.fileName + ); + } + + const cssRules = taggedExpr.template.getText(); + const {line, character} = context.sourceFile.getLineAndCharacterOfPosition( + node.getStart() + ); + + const expressions = + taggedExpr.template.kind === ts.SyntaxKind.TemplateExpression + ? taggedExpr.template.templateSpans + : []; + const hasExpressions = expressions.length > 0; + + const shouldInclude = + context.config.components === null || + context.config.components.has(componentName); + + if (shouldInclude) { + const styledComponent: StyledComponent = { + file: context.fileName, + component: componentName, + componentType, + cssRules: cssRules.trim(), + location: { + line: line + 1, + column: character + 1, + }, + hasExpressions, + expressionCount: expressions.length, + }; + + this.styledComponents.push(styledComponent); + + // Track component usage count + const currentComponentCount = this.componentCounts.get(componentName) || 0; + this.componentCounts.set(componentName, currentComponentCount + 1); + + // Track CSS rules usage count + this.trackCssRules(cssRules.trim()); + } + } + } + } + + reset(): void { + this.styledComponents = []; + this.componentCounts.clear(); + this.cssRuleCounts.clear(); + } + + private trackCssRules(cssRules: string): void { + for (const line of cssRules.split('\n')) { + if ( + line.match(/^\s*$/) || + line.trim() === '`' || + line.match(/^\s*}/) || + line.match(/^\s*\/\*.*\*\/\s*$/) + ) { + continue; + } + + // Skip selectors and media queries for rule counting + if ( + line.match(/^\s*[>&]/) || + line.match(/^\s*@media/) || + line.match(/^\s*@container/) + ) { + continue; + } + + // Handle special expressions + if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { + const currentCount = this.cssRuleCounts.get('overflowEllipsis') || 0; + this.cssRuleCounts.set('overflowEllipsis', currentCount + 1); + continue; + } + + // Parse CSS property:value pairs + const [property] = line.split(':'); + const trimmedProperty = property?.trim(); + + if (trimmedProperty && trimmedProperty.length > 0) { + const currentCount = this.cssRuleCounts.get(trimmedProperty) || 0; + this.cssRuleCounts.set(trimmedProperty, currentCount + 1); + } + } + } + + results(): void { + if (this.styledComponents.length === 0) { + // Early return for empty results - no computation needed + if (config.outputFormat === 'csv') { + logger.log(`=== TOP ${config.topN} MOST COMMONLY STYLED COMPONENTS ===`); + logger.log(arrayToCSV([{Component: '', Instances: 0}])); + logger.log(); + + logger.log(`=== TOP ${config.topN} MOST COMMONLY USED CSS RULES ===`); + logger.log(arrayToCSV([{CSSRule: '', Instances: 0}])); + logger.log(); + } else { + logger.log('\n📊 Found 0 styled components.'); + } + return; + } + + // ==== COMPUTATION PHASE ==== + // Calculate statistics from analysis results + let totalIntrinsic = 0; + let totalReactComponents = 0; + let unknownComponents = 0; + let dynamicExpressions = 0; + let staticExpressions = 0; + + this.styledComponents.forEach(sc => { + if (sc.componentType === 'component') totalReactComponents++; + else if (sc.componentType === 'intrinsic') totalIntrinsic++; + else unknownComponents++; + + // Count expressions + if (sc.hasExpressions) { + dynamicExpressions += sc.expressionCount; + } else { + staticExpressions++; + } + }); + + const totalStyledComponents = this.styledComponents.length; + + // Calculate expression statistics + const avgDynamic = + totalStyledComponents > 0 + ? (dynamicExpressions / totalStyledComponents).toFixed(2) + : '0.00'; + const avgStatic = + totalStyledComponents > 0 + ? (staticExpressions / totalStyledComponents).toFixed(2) + : '0.00'; + const totalExpressions = dynamicExpressions + staticExpressions; + const dynamicPerc = + totalExpressions > 0 + ? ((dynamicExpressions / totalExpressions) * 100).toFixed(1) + : '0.0'; + const staticPerc = + totalExpressions > 0 + ? ((staticExpressions / totalExpressions) * 100).toFixed(1) + : '0.0'; + + // Prepare top components and CSS rules + const topComponents = Array.from(this.componentCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, config.topN); + + const topCssRules = Array.from(this.cssRuleCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, config.topN); + + // Process styled info and parse errors + const styledInfo: Record< + string, + Array<{ + count: number; + expression: StyledComponent; + location: string; + ruleInfo: Record; + }> + > = {}; + + type RuleInfo = { + dynamic: number; + value: string; + children?: RuleInfo[]; + }; + + const parseErrors: string[] = []; + this.styledComponents.forEach(sc => { + const count = (styledInfo[sc.component]?.length ?? 0) + 1; + styledInfo[sc.component] = styledInfo[sc.component] || []; + + const ruleInfo: Record = {}; + + for (const line of sc.cssRules.split('\n')) { + if ( + line.match(/^\s*$/) || + line.trim() === '`' || + line.match(/^\s*}/) || + line.match(/^\s*\/\*.*\*\/\s*$/) + ) { + continue; + } + + if (line.match(/^\s*[>&]/)) { + const subSelector = line.match(/^\s*([>&])/)?.[1] || 'unknown sub selector'; + ruleInfo[subSelector] = ruleInfo[subSelector] || []; + ruleInfo[subSelector].push({ + value: line, + dynamic: 0, + }); + continue; + } + + if (line.match(/^\s*@media/) || line.match(/^\s*@container/)) { + const mediaQuery = '@media'; + ruleInfo[mediaQuery] = ruleInfo[mediaQuery] || []; + ruleInfo[mediaQuery].push({ + value: line, + dynamic: 0, + }); + continue; + } + + if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { + ruleInfo['@expression: p.theme.overflowEllipsis'] = + ruleInfo['@expression: p.theme.overflowEllipsis'] || []; + ruleInfo['@expression: p.theme.overflowEllipsis'].push({ + value: '${p => p.theme.overflowEllipsis}', + dynamic: 1, + }); + continue; + } + + let [property, value] = line.split(':'); + property = property?.trim(); + value = value?.trim().replace(/;$/, ''); + + if (property && value) { + ruleInfo[property] = ruleInfo[property] || []; + ruleInfo[property]!.push({ + value, + dynamic: (value.match(/\$\{[^}]+\}/g) || []).length, + }); + } else { + ruleInfo[line] = ruleInfo[line] || []; + ruleInfo[line].push({ + value: line, + dynamic: 0, + }); + logger.debug('❌ Error parsing line:\n', JSON.stringify(line, null, 2)); + parseErrors.push(`/////////////////\n${line}\n/////////////////\n`); + } + } + + styledInfo[sc.component]!.push({ + count, + location: `${sc.file}:${sc.location.line}:${sc.location.column}`, + expression: sc, + ruleInfo, + }); + }); + + const errorCounts = new Map(); + + for (const error of parseErrors) { + const existing = errorCounts.get(error); + if (existing) { + existing.count++; + } else { + errorCounts.set(error, { + count: 1, + }); + } + } + + // const deduplicatedErrors = Array.from(errorCounts.entries()) + // .sort((a, b) => b[1].count - a[1].count) + // .map(([error, info]) => { + // return `// Error occurred ${info.count} time(s)\n${error}`; + // }) + // .join('\n\n'); + + // ==== LOGGING PHASE ==== + if (config.outputFormat === 'csv') { + // Always output top components section + logger.log(`=== TOP ${config.topN} MOST COMMONLY STYLED COMPONENTS ===`); + const componentsData = + topComponents.length > 0 + ? topComponents.map(([component, count]) => ({ + Component: component, + Instances: count, + })) + : [{Component: '', Instances: 0}]; + logger.log(arrayToCSV(componentsData)); + logger.log(); + + // Always output top CSS rules section + logger.log(`=== TOP ${config.topN} MOST COMMONLY USED CSS RULES ===`); + const cssRulesData = + topCssRules.length > 0 + ? topCssRules.map(([rule, count]) => ({ + CSSRule: rule, + Instances: count, + })) + : [{CSSRule: '', Instances: 0}]; + logger.log(arrayToCSV(cssRulesData)); + logger.log(); + } else { + // Output summary + logger.log(` \n📊 Found ${totalStyledComponents} styled components:\n`); + logger.log(` • Intrinsic elements (div, span, etc.): ${totalIntrinsic}`); + logger.log(` • React components: ${totalReactComponents}`); + logger.log(` • Unknown components: ${unknownComponents}`); + logger.log(` • Total files analyzed: ${tsxFiles.length}`); + + // Output expression statistics + logger.log(' \n📄 Types of expressions:\n '); + logger.log(` • Dynamic expressions per styled component: ~${avgDynamic}`); + logger.log(` • Static expressions per styled component: ~${avgStatic}`); + logger.log(` • Total expressions: ${totalExpressions}`); + logger.log(` • Dynamic/Static ratio: ${dynamicPerc}% / ${staticPerc}%`); + + if (topComponents.length > 0) { + logger.log(`\n🏆 Top ${config.topN} most commonly styled components:\n`); + logger.table( + topComponents.map(([component, count], index) => ({ + Rank: index + 1, + Component: component, + Instances: count, + })) + ); + } + + if (topCssRules.length > 0) { + logger.log(`\n📏 Top ${config.topN} most commonly used CSS rules:\n`); + logger.table( + topCssRules.map(([rule, count], index) => ({ + Rank: index + 1, + 'CSS Rule': rule, + Instances: count, + })) + ); + } + } + } +} + +class StyledUsagePerFileDetector extends BaseDetector { + name = 'StyledUsagePerFile'; + private fileStyledCounts = new Map(); + private filesWithStyled = new Set(); + private totalTsxFiles = new Set(); + + execute(node: ts.Node, context: DetectorContext): void { + // Track all TSX files we analyze + this.totalTsxFiles.add(context.fileName); + + if (!ts.isTaggedTemplateExpression(node)) return; + + const taggedExpr = node; + + if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { + const callExpr = taggedExpr.tag as ts.CallExpression; + + if (callExpr.expression.getText() === 'styled') { + const currentCount = this.fileStyledCounts.get(context.fileName) || 0; + this.fileStyledCounts.set(context.fileName, currentCount + 1); + this.filesWithStyled.add(context.fileName); + } + } + } + + reset(): void { + this.fileStyledCounts.clear(); + this.filesWithStyled.clear(); + this.totalTsxFiles.clear(); + } + + results(): void { + const totalFiles = this.totalTsxFiles.size; + const filesWithStyledCount = this.filesWithStyled.size; + const percentageWithStyled = + totalFiles > 0 ? ((filesWithStyledCount / totalFiles) * 100).toFixed(1) : '0.0'; + + // Get top N files by styled call count + const sortedFiles = Array.from(this.fileStyledCounts.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, config.topN); + + if (config.outputFormat === 'csv') { + // Files with most styled calls + logger.log(`=== TOP ${config.topN} FILES BY STYLED CALL COUNT ===`); + const filesData = + sortedFiles.length > 0 + ? sortedFiles.map(([file, count]) => ({ + File: file, + StyledCalls: count, + })) + : [{File: '', StyledCalls: 0}]; + logger.log(arrayToCSV(filesData)); + logger.log(); + + // Summary statistics + logger.log('=== STYLED USAGE STATISTICS ==='); + logger.log( + arrayToCSV([ + {Metric: 'Total TSX Files', Value: totalFiles}, + {Metric: 'Files with Styled Calls', Value: filesWithStyledCount}, + {Metric: 'Percentage with Styled Calls', Value: `${percentageWithStyled}%`}, + ]) + ); + logger.log(); + } else { + logger.log('\n📄 Styled Usage Per File:\n'); + + if (sortedFiles.length > 0) { + logger.log(`🏆 Top ${config.topN} files by styled call count:\n`); + logger.table( + sortedFiles.map(([file, count], index) => ({ + Rank: index + 1, + File: file.replace(process.cwd() + '/', ''), + 'Styled Calls': count, + })) + ); + } + + logger.log('\n📊 Usage Statistics:\n'); + logger.log(` • Total TSX files analyzed: ${totalFiles}`); + logger.log(` • Files containing styled calls: ${filesWithStyledCount}`); + logger.log( + ` • Percentage of TSX files with styled calls: ${percentageWithStyled}%` + ); + } + } +} + +class FlexOnlyDivsDetector extends BaseDetector { + name = 'FlexOnlyDivs'; + private styledComponents: StyledComponent[] = []; + + execute(node: ts.Node, context: DetectorContext): void { + if (!ts.isTaggedTemplateExpression(node)) return; + + const taggedExpr = node; + + if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { + const callExpr = taggedExpr.tag as ts.CallExpression; + + if (callExpr.expression.getText() === 'styled') { + const component = callExpr.arguments[0]; + if (!component) return; + + let componentName = ''; + let componentType: 'intrinsic' | 'component' | 'unknown' = 'unknown'; + + // Extract component name and type + if ( + component.kind === ts.SyntaxKind.StringLiteral || + component.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral + ) { + componentName = (component as ts.StringLiteral).text; + componentType = 'intrinsic'; + } else if (component.kind === ts.SyntaxKind.Identifier) { + componentName = (component as ts.Identifier).text; + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.PropertyAccessExpression) { + componentName = component.getText(); + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.ArrowFunction) { + componentName = 'InlineArrowFunction'; + componentType = 'component'; + } else if (component.kind === ts.SyntaxKind.AsExpression) { + const asExpr = component as ts.AsExpression; + componentName = asExpr.expression.getText(); + componentType = 'component'; + } else if ( + component.kind === ts.SyntaxKind.FunctionExpression || + component.kind === ts.SyntaxKind.CallExpression + ) { + componentName = 'InlineFunction'; + componentType = 'component'; + } + + // Only process divs and respect component filter + const shouldInclude = + componentName === 'div' && + (context.config.components === null || + context.config.components.has(componentName)); + + if (shouldInclude) { + const cssRules = taggedExpr.template.getText(); + const {line, character} = context.sourceFile.getLineAndCharacterOfPosition( + node.getStart() + ); + + const expressions = + taggedExpr.template.kind === ts.SyntaxKind.TemplateExpression + ? taggedExpr.template.templateSpans + : []; + + const styledComponent: StyledComponent = { + file: context.fileName, + component: componentName, + componentType, + cssRules: cssRules.trim(), + location: { + line: line + 1, + column: character + 1, + }, + hasExpressions: expressions.length > 0, + expressionCount: expressions.length, + }; + + this.styledComponents.push(styledComponent); + } + } + } + } + + reset(): void { + this.styledComponents = []; + } + + results(): void { + // ==== COMPUTATION PHASE ==== + const flexRules = new Set([ + 'flex', + 'flex-direction', + 'flex-wrap', + 'justify-content', + 'align-items', + 'align-content', + 'gap', + ]); + + const flexOnlyComponents: Array<{component: string; location: string}> = []; + + for (const sc of this.styledComponents) { + let hasOnlyFlexRules = true; + let hasDisplayFlexRule = false; + + // Parse CSS rules to check if it only contains flex rules + for (const line of sc.cssRules.split('\n')) { + if ( + line.match(/^\s*$/) || + line.trim() === '`' || + line.match(/^\s*}/) || + line.match(/^\s*\/\*.*\*\/\s*$/) || + line.match(/^\s*[>&]/) || + line.match(/^\s*@media/) || + line.match(/^\s*@container/) + ) { + continue; + } + + // Handle special expressions + if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { + // This is not a flex rule, so exclude this component + hasOnlyFlexRules = false; + break; + } + + // Parse CSS property:value pairs + const [property] = line.split(':'); + const trimmedProperty = property?.trim(); + + if (trimmedProperty && trimmedProperty.length > 0) { + if (trimmedProperty === 'display') { + // Check if the value contains 'flex' + const [, value] = line.split(':'); + const trimmedValue = value?.trim().replace(/;$/, ''); + if (trimmedValue === 'flex') { + hasDisplayFlexRule = true; + } + } else if (!flexRules.has(trimmedProperty)) { + hasOnlyFlexRules = false; + break; + } + } + } + + if (hasDisplayFlexRule && hasOnlyFlexRules) { + flexOnlyComponents.push({ + location: `${sc.file}:${sc.location.line}:${sc.location.column}`, + component: sc.component, + }); + } + } + + // Limit results to topN + const results = flexOnlyComponents.slice(0, config.topN); + + // ==== LOGGING PHASE ==== + if (results.length === 0) { + if (config.outputFormat === 'csv') { + logger.log('=== STYLED DIVS WITH ONLY FLEX RULES ==='); + logger.log(arrayToCSV([{Rank: 0, Component: '', Location: ''}])); + logger.log(); + } else { + logger.log('\n🎯 Styled Divs with Only Flex Rules:'); + logger.log('No styled divs found that only use flexbox rules.'); + } + return; + } + + if (config.outputFormat === 'csv') { + const csvData = results.map((result, index) => ({ + Rank: index + 1, + Component: result.component, + Location: result.location, + })); + logger.log('=== STYLED DIVS WITH ONLY FLEX RULES ==='); + logger.log(arrayToCSV(csvData)); + logger.log(); + } else { + logger.log(`\n🎯 Top ${config.topN} Styled Divs with Only Flex Rules:\n`); + logger.table( + results.map((result, index) => ({ + '#': index + 1, + Component: result.component, + Location: result.location, + })) + ); + } + } +} + +function analyze( + sourceFile: ts.SourceFile, + fileName: string, + detectorConfig: DetectorConfiguration +): void { + const context: DetectorContext = { + sourceFile, + fileName, + config, + }; + + function visit(node: ts.Node): void { + // Run all detectors for every node + for (const detector of detectorConfig.detectors) { + detector.execute(node, context); + } + + ts.forEachChild(node, visit); + } + + visit(sourceFile); +} + +// History analysis functions +let interruptRequested = false; + +interface CommitAnalysisResult { + commit: GitCommit; + filename: string; + output: string; +} + +function processCommit( + commit: GitCommit, + originalBranch: string +): CommitAnalysisResult | null { + if (interruptRequested) { + logger.log('🛑 Interrupt detected, skipping commit processing'); + return null; + } + + // Clean any uncommitted changes before checkout + // Use -f to force removal and -x to include ignored files + child_process.execSync('git clean -d -f -x', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + // Reset any staged changes + child_process.execSync('git reset --hard', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + // Checkout the commit (force checkout to handle conflicts) + child_process.execSync(`git checkout -f ${commit.hash}`, { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + // Check if analyze-styled.ts exists in this commit + if (!fs.existsSync('scripts/analyze-styled.ts')) { + logger.debug('Script not found in commit, copying from current version...'); + // Get the current script content and write it temporarily + child_process.execSync( + `git show ${originalBranch}:scripts/analyze-styled.ts > scripts/analyze-styled.ts` + ); + } + + // Collect files for this specific commit + const commitTsxFiles = collectTsxFilesForCurrentState(config); + + if (commitTsxFiles.length === 0) { + logger.debug( + `No .tsx files found in commit ${commit.hash.substring(0, 7)}, skipping analysis.` + ); + return null; + } + + logger.debug( + `Found ${commitTsxFiles.length} .tsx files in commit ${commit.hash.substring(0, 7)}` + ); + + // Reset all detectors before analyzing this commit + for (const detector of detectorConfig.detectors) { + detector.reset(); + } + + // Capture the current stdout to save to file + const originalLog = logger.log; + const originalTable = logger.table; + let output = ''; + + logger.log = (...args: any[]) => { + output += args.join(' ') + '\n'; + }; + logger.table = (...args: any[]) => { + output += JSON.stringify(args[0], null, 2) + '\n'; + }; + + // Run the analysis for this commit with commit-specific files + runCurrentAnalysis(commitTsxFiles); + + // Restore original logging + logger.log = originalLog; + logger.table = originalTable; + + // Return analysis result instead of writing immediately + const filename = `${commit.date}-${commit.hash.substring(0, 7)}.txt`; + + logger.log(`✅ Completed analysis for ${commit.hash.substring(0, 7)}`); + + return { + commit, + output, + filename, + }; +} + +function runHistoryAnalysis(c: Config) { + // Reset interrupt flag + interruptRequested = false; + + if (!c.startDate) { + throw new Error('--start-date is required for historical analysis'); + } + + // Safety check: prevent running on dirty git tree + const gitInfo = getGitInfo(); + if (gitInfo.status !== 'clean') { + throw new Error( + 'Git tree is dirty. Please commit or stash your changes before running historical analysis.\n' + + 'This safety check prevents potential data loss during git operations.' + ); + } + + // Ensure output directory exists + if (!fs.existsSync(c.outdir)) { + fs.mkdirSync(c.outdir, {recursive: true}); + } + + // Get current git state for restoration + const originalBranch = child_process + .execSync('git rev-parse --abbrev-ref HEAD', {encoding: 'utf8'}) + .trim(); + const hasUnstagedChanges = + child_process.execSync('git status --porcelain', {encoding: 'utf8'}).trim().length > + 0; + + // Set up cleanup function for process interrupts + + // Set up interrupt handlers that set flag instead of immediately exiting + const handleInterrupt = (_signal: string) => { + // Set flag first, before any other operations + interruptRequested = true; + + logger.log('\n⚠️ Process interrupted, cleaning up...'); + try { + // Clean any uncommitted changes (including ignored files) + child_process.execSync('git clean -d -f -x', {stdio: ['pipe', 'pipe', 'pipe']}); + // Reset any staged changes + child_process.execSync('git reset --hard', {stdio: ['pipe', 'pipe', 'pipe']}); + // Force checkout to restore original branch + child_process.execSync(`git checkout -f ${originalBranch}`, { + stdio: ['pipe', 'pipe', 'pipe'], + }); + // Restore stashed changes if any existed + if (hasUnstagedChanges) { + child_process.execSync('git stash pop', {stdio: ['pipe', 'pipe', 'pipe']}); + } + logger.log('✅ Cleanup completed, restored to original state'); + } catch (error) { + logger.fatal('❌ Error during cleanup:', (error as Error).message); + } + process.exit(0); + }; + + // Register interrupt handlers with signal names + process.on('SIGINT', () => handleInterrupt('SIGINT')); + process.on('SIGTERM', () => handleInterrupt('SIGTERM')); + process.on('SIGUSR1', () => handleInterrupt('SIGUSR1')); + process.on('SIGUSR2', () => handleInterrupt('SIGUSR2')); + + // Collect all analysis results in memory + const analysisResults: CommitAnalysisResult[] = []; + + try { + // Stash changes if any exist + if (hasUnstagedChanges) { + child_process.execSync('git stash push -m "analyze-styled temporary stash"', { + stdio: 'inherit', + }); + } + const commits = getCommitsInDateRange(c.startDate, c.interval); + + logger.log(`Analyzing ${commits.length} commits at ${c.interval}-day intervals`); + + for (let i = 0; i < commits.length; i++) { + // Check for interrupt before processing each commit + if (interruptRequested) { + logger.log(`\n⚠️ Interrupt received, stopping after ${i} commits processed.`); + break; + } + + const commit = commits[i]!; + logger.log( + `\n[${i + 1}/${commits.length}] Processing commit ${commit.hash.substring(0, 7)} (${commit.date})` + ); + + try { + const result = processCommit(commit, originalBranch); + if (result) { + analysisResults.push(result); + } + } catch (error) { + logger.error( + `❌ Error processing commit ${commit.hash.substring(0, 7)}: ${(error as Error).message}` + ); + continue; + } + } + } finally { + // Remove interrupt handlers + process.removeAllListeners('SIGINT'); + process.removeAllListeners('SIGTERM'); + process.removeAllListeners('SIGUSR1'); + process.removeAllListeners('SIGUSR2'); + + // Restore original state + try { + // Clean any uncommitted changes before final checkout (including ignored files) + child_process.execSync('git clean -d -f -x', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + // Reset any staged changes + child_process.execSync('git reset --hard', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + // Force checkout to restore original branch + child_process.execSync(`git checkout -f ${originalBranch}`, { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + // Restore stashed changes if any existed + if (hasUnstagedChanges) { + child_process.execSync('git stash pop', {stdio: ['pipe', 'pipe', 'pipe']}); + } + } catch (error) { + logger.fatal('❌ Error restoring git state:', (error as Error).message); + } + + // Write all analysis results to files after git state is restored + if (analysisResults.length > 0) { + logger.log(`\n📝 Writing ${analysisResults.length} analysis files...`); + + // Ensure output directory exists + if (!fs.existsSync(c.outdir)) { + fs.mkdirSync(c.outdir, {recursive: true}); + } + + for (const result of analysisResults) { + try { + const filepath = path.join(c.outdir, result.filename); + fs.writeFileSync(filepath, result.output); + logger.log(`✅ Saved analysis to ${result.filename}`); + } catch (error) { + logger.error( + `❌ Error writing ${result.filename}: ${(error as Error).message}` + ); + } + } + } + } + + logger.log(`\n🎉 Historical analysis complete! Results saved in ${config.outdir}`); +} + +function runCurrentAnalysis(filesToAnalyze?: string[]) { + // Use provided files or fall back to global tsxFiles + const files = filesToAnalyze || tsxFiles; + + let successfullyAnalyzedFiles = 0; + // Process all files with detectors + for (const file of files) { + try { + const sourceCode = fs.readFileSync(file, 'utf8'); + const sourceFile = ts.createSourceFile( + file, + sourceCode, + ts.ScriptTarget?.Latest, + true + ); + + analyze(sourceFile, file, detectorConfig); + successfullyAnalyzedFiles++; + } catch (error) { + logger.fatal(`❌ Error parsing ${file}:`, (error as Error).message); + } + } + + // Collect and output metadata first (for CSV format) + const searchDirectory = config.searchDir || config.targetFile || './static/app'; + const metadata = collectMetadata( + searchDirectory, + files.length, + successfullyAnalyzedFiles + ); + outputMetadata(metadata); + + // Execute detector results + for (const detector of detectorConfig.detectors) { + detector.results(); + } +} + +// Create detector configuration +const detectorConfig: DetectorConfiguration = { + detectors: [ + new StyledComponentsDetector(), + new CoreComponentImportsDetector(), + new FlexOnlyDivsDetector(), + new StyledUsagePerFileDetector(), + ], +}; + +// Setup graceful exit handler for main analysis +function setupGracefulExit(): void { + const cleanup = () => { + logger.log('\n⚠️ Process interrupted, exiting gracefully...'); + process.exit(0); + }; + + // Register interrupt handlers + process.on('SIGINT', cleanup); + process.on('SIGTERM', cleanup); + process.on('SIGUSR1', cleanup); + process.on('SIGUSR2', cleanup); +} + +// Main execution: check if historical analysis is requested +if (config.startDate) { + runHistoryAnalysis(config); +} else { + setupGracefulExit(); + runCurrentAnalysis(); +} diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py new file mode 100644 index 000000000..b03b50a5c --- /dev/null +++ b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py @@ -0,0 +1,101 @@ +from django.utils.decorators import method_decorator +from django.views.decorators.cache import never_cache +from drf_spectacular.utils import extend_schema +from rest_framework import status +from rest_framework.request import Request +from rest_framework.response import Response + +from sentry import features +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import cell_silo_endpoint +from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission +from sentry.api.paginator import OffsetPaginator +from sentry.api.serializers import serialize +from sentry.apidocs.constants import RESPONSE_BAD_REQUEST, RESPONSE_FORBIDDEN +from sentry.apidocs.examples.integration_examples import IntegrationExamples +from sentry.apidocs.parameters import GlobalParams +from sentry.apidocs.utils import inline_sentry_response_serializer +from sentry.integrations.api.serializers.models.data_forwarder import DataForwarderResponse +from sentry.integrations.api.serializers.rest_framework.data_forwarder import ( + DataForwarderSerializer, +) +from sentry.integrations.models.data_forwarder import DataForwarder +from sentry.web.decorators import set_referrer_policy + + +class OrganizationDataForwardingDetailsPermission(OrganizationPermission): + scope_map = { + "GET": ["org:write"], + "POST": ["org:write"], + } + + +@cell_silo_endpoint +@extend_schema(tags=["Integrations"]) +class DataForwardingIndexEndpoint(OrganizationEndpoint): + owner = ApiOwner.INTEGRATIONS + publish_status = { + "GET": ApiPublishStatus.PUBLIC, + "POST": ApiPublishStatus.PUBLIC, + } + permission_classes = (OrganizationDataForwardingDetailsPermission,) + + @extend_schema( + operation_id="Retrieve Data Forwarders for an Organization", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + responses={ + 200: inline_sentry_response_serializer( + "ListDataForwarderResponse", list[DataForwarderResponse] + ) + }, + examples=IntegrationExamples.LIST_DATA_FORWARDERS, + ) + @set_referrer_policy("strict-origin-when-cross-origin") + @method_decorator(never_cache) + def get(self, request: Request, organization) -> Response: + """ + Returns a list of data forwarders for an organization. + """ + queryset = DataForwarder.objects.filter(organization_id=organization.id) + + return self.paginate( + request=request, + queryset=queryset, + on_results=lambda x: serialize(x, request.user), + paginator_cls=OffsetPaginator, + ) + + @extend_schema( + operation_id="Create a Data Forwarder for an Organization", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request=DataForwarderSerializer, + responses={ + 201: inline_sentry_response_serializer("DataForwarderResponse", DataForwarderResponse), + 400: RESPONSE_BAD_REQUEST, + 403: RESPONSE_FORBIDDEN, + }, + examples=IntegrationExamples.SINGLE_DATA_FORWARDER, + ) + @set_referrer_policy("strict-origin-when-cross-origin") + @method_decorator(never_cache) + def post(self, request: Request, organization) -> Response: + """ + Creates a new data forwarder for an organization. + Only one data forwarder can be created per provider for a given organization. + + Project-specific overrides can only be created after creating the data forwarder. + """ + if not features.has("organizations:data-forwarding", organization): + return self.respond(status=status.HTTP_403_FORBIDDEN) + + data = request.data + data["organization_id"] = organization.id + + serializer = DataForwarderSerializer(data=data, context={"organization": organization}) + if not serializer.is_valid(): + return self.respond(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + return self.respond( + serialize(serializer.save(), request.user), status=status.HTTP_201_CREATED + ) diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py new file mode 100644 index 000000000..a95001515 --- /dev/null +++ b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py @@ -0,0 +1,119 @@ +from collections import defaultdict +from collections.abc import Mapping, Sequence +from datetime import datetime +from typing import Any, TypedDict + +from django.contrib.auth.models import AnonymousUser + +from sentry.api.serializers import Serializer, register, serialize +from sentry.integrations.models.data_forwarder import DataForwarder +from sentry.integrations.models.data_forwarder_project import DataForwarderProject +from sentry.users.models.user import User +from sentry.users.services.user import RpcUser + + +class ProjectResponse(TypedDict): + id: str + slug: str + platform: str | None + + +class DataForwarderProjectResponse(TypedDict): + id: str + isEnabled: bool + dataForwarderId: str + project: ProjectResponse + overrides: dict[str, str] + effectiveConfig: dict[str, str] + dateAdded: datetime + dateUpdated: datetime + + +class DataForwarderResponse(TypedDict): + id: str + organizationId: str + isEnabled: bool + enrollNewProjects: bool + enrolledProjects: list[ProjectResponse] + provider: str + config: dict[str, str] + projectConfigs: list[DataForwarderProjectResponse] + dateAdded: datetime + dateUpdated: datetime + + +@register(DataForwarder) +class DataForwarderSerializer(Serializer): + def get_attrs( + self, + item_list: Sequence[DataForwarder], + user: User | RpcUser | AnonymousUser, + **kwargs: Any, + ) -> dict[DataForwarder, dict[str, Any]]: + project_configs = DataForwarderProject.objects.filter( + data_forwarder__in=item_list + ).select_related("project", "data_forwarder") + project_config_attrs = defaultdict(set) + for project_config in project_configs: + project_config_attrs[project_config.data_forwarder].add(project_config) + + return { + data_forwarder: {"project_configs": project_config_attrs.get(data_forwarder, set())} + for data_forwarder in item_list + } + + def serialize( + self, + obj: DataForwarder, + attrs: Mapping[str, Any], + user: User | RpcUser | AnonymousUser, + **kwargs: Any, + ) -> DataForwarderResponse: + project_configs = attrs.get("project_configs", set()) + + return { + "id": str(obj.id), + "organizationId": str(obj.organization_id), + "isEnabled": obj.is_enabled, + "enrollNewProjects": obj.enroll_new_projects, + "enrolledProjects": [ + { + "id": str(project_config.project_id), + "slug": project_config.project.slug, + "platform": project_config.project.platform, + } + for project_config in project_configs + ], + "provider": obj.provider, + "config": obj.config, + "projectConfigs": [ + serialize(project_config, user=user) for project_config in project_configs + ], + "dateAdded": obj.date_added, + "dateUpdated": obj.date_updated, + } + + +@register(DataForwarderProject) +class DataForwarderProjectSerializer(Serializer): + def serialize( + self, + obj: DataForwarderProject, + attrs: Mapping[str, Any], + user: User | RpcUser | AnonymousUser, + **kwargs: Any, + ) -> DataForwarderProjectResponse: + return { + "id": str(obj.id), + "isEnabled": obj.is_enabled, + "dataForwarderId": str(obj.data_forwarder_id), + "project": { + "id": str(obj.project_id), + "slug": obj.project.slug, + "platform": obj.project.platform, + }, + "overrides": obj.overrides, + "effectiveConfig": obj.get_config(), + "dateAdded": obj.date_added, + "dateUpdated": obj.date_updated, + } diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py new file mode 100644 index 000000000..d748fb6f2 --- /dev/null +++ b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py @@ -0,0 +1,38 @@ +from django.db import models + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import DefaultFieldsModel, FlexibleForeignKey, cell_silo_model +from sentry.db.models.fields.encryption.encrypted_json_field import EncryptedJSONField + + +@cell_silo_model +class DataForwarder(DefaultFieldsModel): + """ + Configuration for data forwarding to external services (Segment, SQS, or Splunk). + This is a general config that can apply to specific projects or all projects in an organization. + """ + + __relocation_scope__ = RelocationScope.Organization + + organization = FlexibleForeignKey("sentry.Organization", on_delete=models.CASCADE) + is_enabled = models.BooleanField(default=True) + + enroll_new_projects = models.BooleanField(default=False) + enrolled_projects = models.ManyToManyField( + "sentry.Project", through="sentry.DataForwarderProject", related_name="data_forwarders" + ) + + provider = models.CharField( + max_length=64, + choices=[ + ("segment", "Segment"), + ("sqs", "Amazon SQS"), + ("splunk", "Splunk"), + ], + ) + config = EncryptedJSONField(default=dict) + + class Meta: + app_label = "sentry" + db_table = "sentry_dataforwarder" + unique_together = (("organization", "provider"),) diff --git a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py new file mode 100644 index 000000000..b454582fc --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py @@ -0,0 +1,116 @@ +from collections.abc import Mapping +from typing import Any + +from google.protobuf.timestamp_pb2 import Timestamp +from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE +from sentry_protos.snuba.v1.trace_item_pb2 import ( + AnyValue, + ArrayValue, + KeyValue, + KeyValueList, + TraceItem, +) + +from sentry.models.project import Project +from sentry.services.eventstore.models import Event, GroupEvent +from sentry.utils import json +from sentry.utils.eap import hex_to_item_id + +# Max depth for recursive encoding to protobuf AnyValue. +_ENCODE_MAX_DEPTH = 6 + + +def serialize_event_data_as_item( + event: Event | GroupEvent, event_data: Mapping[str, Any], project: Project +) -> TraceItem: + return TraceItem( + item_id=hex_to_item_id(event_data["event_id"]), + item_type=TRACE_ITEM_TYPE_OCCURRENCE, + trace_id=event_data["contexts"]["trace"]["trace_id"], + timestamp=Timestamp(seconds=int(event_data["timestamp"])), + organization_id=project.organization_id, + project_id=project.id, + received=( + Timestamp(seconds=int(event_data["received"])) if "received" in event_data else None + ), + retention_days=event_data.get("retention_days", 90), + attributes=encode_attributes( + event, event_data, ignore_fields={"event_id", "timestamp", "tags"} + ), + ) + + +def _encode_value(value: Any, _depth: int = 0) -> AnyValue: + if _depth > _ENCODE_MAX_DEPTH: + # Beyond max depth, stringify to prevent protobuf nesting limit errors. + return AnyValue(string_value=json.dumps(value)) + + if isinstance(value, str): + return AnyValue(string_value=value) + elif isinstance(value, bool): + # Note: bool check must come before int check since bool is a subclass of int + return AnyValue(bool_value=value) + elif isinstance(value, int): + # int_value is a signed int64, so it has a range of valid values. + # if value doesn't fit into an int64, cast it to string. + if abs(value) >= (2**63): + return AnyValue(string_value=str(value)) + return AnyValue(int_value=value) + elif isinstance(value, float): + return AnyValue(double_value=value) + elif isinstance(value, list) or isinstance(value, tuple): + # Not yet processed on EAP side + return AnyValue( + array_value=ArrayValue( + values=[_encode_value(v, _depth + 1) for v in value if v is not None] + ) + ) + elif isinstance(value, dict): + # Not yet processed on EAP side + return AnyValue( + kvlist_value=KeyValueList( + values=[ + KeyValue(key=str(kv[0]), value=_encode_value(kv[1], _depth + 1)) + for kv in value.items() + if kv[1] is not None + ] + ) + ) + else: + raise NotImplementedError(f"encode not supported for {type(value)}") + + +def encode_attributes( + event: Event | GroupEvent, event_data: Mapping[str, Any], ignore_fields: set[str] | None = None +) -> Mapping[str, AnyValue]: + attributes = {} + ignore_fields = ignore_fields or set() + + for key, value in event_data.items(): + if key in ignore_fields: + continue + if value is None: + continue + attributes[key] = _encode_value(value) + + if event.group_id: + attributes["group_id"] = AnyValue(int_value=event.group_id) + + format_tag_key = lambda key: f"tags[{key}]" + + tag_keys = set() + tags = event_data.get("tags") + if tags is not None: + for tag in tags: + if tag is None: + continue + key, value = tag + if value is None: + continue + formatted_key = format_tag_key(key) + attributes[formatted_key] = _encode_value(value) + tag_keys.add(formatted_key) + + attributes["tag_keys"] = _encode_value(sorted(tag_keys)) + + return attributes diff --git a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py new file mode 100644 index 000000000..bbff7d671 --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py @@ -0,0 +1,106 @@ +from collections.abc import Mapping +from typing import Any + +from google.protobuf.timestamp_pb2 import Timestamp +from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE +from sentry_protos.snuba.v1.trace_item_pb2 import ( + AnyValue, + ArrayValue, + KeyValue, + KeyValueList, + TraceItem, +) + +from sentry.models.project import Project +from sentry.services.eventstore.models import Event, GroupEvent +from sentry.utils.eap import hex_to_item_id + + +def serialize_event_data_as_item( + event: Event | GroupEvent, event_data: Mapping[str, Any], project: Project +) -> TraceItem: + return TraceItem( + item_id=hex_to_item_id(event_data["event_id"]), + item_type=TRACE_ITEM_TYPE_OCCURRENCE, + trace_id=event_data["contexts"]["trace"]["trace_id"], + timestamp=Timestamp(seconds=int(event_data["timestamp"])), + organization_id=project.organization_id, + project_id=project.id, + received=( + Timestamp(seconds=int(event_data["received"])) if "received" in event_data else None + ), + retention_days=event_data.get("retention_days", 90), + attributes=encode_attributes( + event, event_data, ignore_fields={"event_id", "timestamp", "tags"} + ), + ) + + +def _encode_value(value: Any) -> AnyValue: + if isinstance(value, str): + return AnyValue(string_value=value) + elif isinstance(value, bool): + # Note: bool check must come before int check since bool is a subclass of int + return AnyValue(bool_value=value) + elif isinstance(value, int): + # int_value is a signed int64, so it has a range of valid values. + # if value doesn't fit into an int64, cast it to string. + if abs(value) >= (2**63): + return AnyValue(string_value=str(value)) + return AnyValue(int_value=value) + elif isinstance(value, float): + return AnyValue(double_value=value) + elif isinstance(value, list) or isinstance(value, tuple): + # Not yet processed on EAP side + return AnyValue( + array_value=ArrayValue(values=[_encode_value(v) for v in value if v is not None]) + ) + elif isinstance(value, dict): + # Not yet processed on EAP side + return AnyValue( + kvlist_value=KeyValueList( + values=[ + KeyValue(key=str(kv[0]), value=_encode_value(kv[1])) + for kv in value.items() + if kv[1] is not None + ] + ) + ) + else: + raise NotImplementedError(f"encode not supported for {type(value)}") + + +def encode_attributes( + event: Event | GroupEvent, event_data: Mapping[str, Any], ignore_fields: set[str] | None = None +) -> Mapping[str, AnyValue]: + attributes = {} + ignore_fields = ignore_fields or set() + + for key, value in event_data.items(): + if key in ignore_fields: + continue + if value is None: + continue + attributes[key] = _encode_value(value) + + if event.group_id: + attributes["group_id"] = AnyValue(int_value=event.group_id) + + format_tag_key = lambda key: f"tags[{key}]" + + tag_keys = set() + tags = event_data.get("tags") + if tags is not None: + for tag in tags: + if tag is None: + continue + key, value = tag + if value is None: + continue + formatted_key = format_tag_key(key) + attributes[formatted_key] = _encode_value(value) + tag_keys.add(formatted_key) + + attributes["tag_keys"] = _encode_value(sorted(tag_keys)) + + return attributes diff --git a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py new file mode 100644 index 000000000..72bcdfb63 --- /dev/null +++ b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py @@ -0,0 +1,1668 @@ +import logging +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from datetime import datetime +from re import Match +from typing import Any, Literal, cast + +from parsimonious.exceptions import ParseError +from sentry_protos.snuba.v1.attribute_conditional_aggregation_pb2 import ( + AttributeConditionalAggregation, +) +from sentry_protos.snuba.v1.endpoint_trace_item_table_pb2 import ( + AggregationAndFilter, + AggregationComparisonFilter, + AggregationFilter, + AggregationOrFilter, + Column, +) +from sentry_protos.snuba.v1.formula_pb2 import Literal as LiteralValue +from sentry_protos.snuba.v1.request_common_pb2 import RequestMeta, TraceItemType +from sentry_protos.snuba.v1.trace_item_attribute_pb2 import ( + AttributeAggregation, + AttributeKey, + AttributeValue, + DoubleArray, + IntArray, + StrArray, + VirtualColumnContext, +) +from sentry_protos.snuba.v1.trace_item_filter_pb2 import ( + AndFilter, + ComparisonFilter, + ExistsFilter, + NotFilter, + OrFilter, + TraceItemFilter, +) +from sentry_sdk import traces + +from sentry.api import event_search +from sentry.discover import arithmetic +from sentry.exceptions import InvalidSearchQuery +from sentry.explore.models import ExploreSavedFormula +from sentry.models.group import Group, parse_short_id +from sentry.models.project import Project +from sentry.search.eap import constants +from sentry.search.eap.columns import ( + AggregateDefinition, + AttributeArgumentDefinition, + ColumnDefinitions, + FormulaDefinition, + ResolvedArgument, + ResolvedArguments, + ResolvedAttribute, + ResolvedColumn, + ResolvedEquation, + ResolvedFunction, + ResolvedLiteral, + ValueArgumentDefinition, + VirtualColumnDefinition, +) +from sentry.search.eap.rpc_utils import and_trace_item_filters +from sentry.search.eap.sampling import validate_sampling +from sentry.search.eap.spans.attributes import SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS +from sentry.search.eap.types import EAPResponse, SearchResolverConfig, SupportedTraceItemType +from sentry.search.events import constants as qb_constants +from sentry.search.events import fields +from sentry.search.events import filter as event_filter +from sentry.search.events.filter import to_list +from sentry.search.events.types import SAMPLING_MODES, SnubaParams +from sentry.search.exceptions import InvalidIssueSearchQuery + +logger = logging.getLogger(__name__) + + +def collect_issue_short_ids_from_parsed_terms(terms: Sequence[object]) -> set[str]: + """Collect non-empty issue filter values from a parsed search tree (including OR/AND).""" + out: set[str] = set() + for term in terms: + if isinstance(term, event_search.SearchFilter): + if term.key.name == "issue": + for v in to_list(term.value.value): + if v: + out.add(str(v)) + elif isinstance(term, event_search.ParenExpression): + out |= collect_issue_short_ids_from_parsed_terms(term.children) + return out + + +class HiddenApiAttribute(InvalidSearchQuery): + pass + + +@dataclass(frozen=True) +class SearchResolver: + """The only attributes are things we want to cache and params + + Please do not store any state on the SearchResolver + """ + + params: SnubaParams + config: SearchResolverConfig + definitions: ColumnDefinitions + granularity_secs: int | None = None + _query_result_cache: dict[str, EAPResponse] = field(default_factory=dict) + _resolved_attribute_cache: dict[ + str, tuple[ResolvedAttribute, list[VirtualColumnDefinition | None]] + ] = field(default_factory=dict) + _resolved_function_cache: dict[ + str, + tuple[ + ResolvedFunction, + list[VirtualColumnDefinition | None], + ], + ] = field(default_factory=dict) + qualified_short_id_to_group_id_cache: dict[int, dict[str, int]] = field(default_factory=dict) + _internal_name_to_column: dict[str, ResolvedAttribute] = field(default_factory=dict, repr=False) + + def _find_column_by_internal_name(self, internal_name: str) -> ResolvedAttribute | None: + """Look up a column definition by its internal name (e.g. 'sentry.item_id' -> 'id' column). + + Uses a lazily-built reverse mapping from internal_name -> column definition, + cached for the lifetime of this resolver instance. + """ + if not self._internal_name_to_column: + self._internal_name_to_column.update( + { + col.internal_name: col + for col in self.definitions.columns.values() + if not col.secondary_alias + } + ) + return self._internal_name_to_column.get(internal_name) + + def get_function_definition( + self, function_name: str + ) -> FormulaDefinition | AggregateDefinition: + if function_name in self.definitions.aggregates: + return self.definitions.aggregates[function_name] + elif function_name in self.definitions.formulas: + return self.definitions.formulas[function_name] + else: + raise InvalidSearchQuery(f"Unknown function {function_name}") + + @traces.trace + def resolve_meta( + self, + referrer: str, + sampling_mode: SAMPLING_MODES | None = None, + filter_project: Callable[[Project], bool] | None = None, + ) -> RequestMeta: + if self.params.organization_id is None: + raise Exception("An organization is required to resolve queries") + span = traces.get_current_span() + if span: + span.set_attribute("SearchResolver.params", repr(self.params)) + + projects = self.params.projects + + # If a filter is specified, use it to narrow down the list + # of projects to query on. + if filter_project: + projects = [project for project in projects if filter_project(project)] + + # if filtering removed all projects, we reset to all + # selected project again to prevent potential snuba errors + if not projects: + projects = self.params.projects + + return RequestMeta( + organization_id=self.params.organization_id, + referrer=referrer, + project_ids=[project.id for project in projects], + start_timestamp=self.params.rpc_start_date, + end_timestamp=self.params.rpc_end_date, + trace_item_type=self.definitions.trace_item_type, + downsampled_storage_config=validate_sampling(sampling_mode), + ) + + @traces.trace + def resolve_query( + self, querystring: str | None + ) -> tuple[ + TraceItemFilter | None, + AggregationFilter | None, + list[VirtualColumnDefinition | None], + ]: + """Given a query string in the public search syntax eg. `span.description:foo` construct the TraceItemFilter + + This is the public interface to resolver the query, for the logic see __resolve_query, this is because we + also append the environment before returning the final TraceItemFilter""" + environment_query = self.__resolve_environment_query() + where, having, contexts = self.__resolve_query(querystring) + span = traces.get_current_span() + if span and querystring is not None: + span.set_attribute("SearchResolver.query_string", querystring) + if span and where is not None: + span.set_attribute("SearchResolver.resolved_query", repr(where)) + if span and environment_query is not None: + span.set_attribute("SearchResolver.environment_query", repr(environment_query)) + + where = and_trace_item_filters( + where, + # The RPC request meta does not contain the environment. + # So we have to inject it as a query condition. + environment_query, + ) + + return where, having, contexts + + @traces.trace + def resolve_query_with_columns( + self, + querystring: str | None, + selected_columns: list[str] | None, + equations: list[str] | None, + ) -> tuple[ + TraceItemFilter | None, + AggregationFilter | None, + list[VirtualColumnDefinition | None], + ]: + where, having, contexts = self.resolve_query(querystring) + + # Some datasets like trace metrics require we inject additional + # conditions in the top level. + dataset_conditions = self.resolve_dataset_conditions(selected_columns, equations) + where = and_trace_item_filters(where, dataset_conditions) + + return where, having, contexts + + def __resolve_environment_query(self) -> TraceItemFilter | None: + resolved_column, _ = self.resolve_column("environment") + if not isinstance(resolved_column.proto_definition, AttributeKey): + return None + + envs = [env.name for env in self.params.environments if env is not None] + + if not envs: + return None + + return TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=ComparisonFilter.OP_IN, + value=AttributeValue(val_str_array=StrArray(values=envs)), + ) + ) + + def _init_issue_short_id_cache( + self, + parsed_terms: Sequence[object], + ) -> None: + """One bulk Group lookup for issue short ids in the query; store maps on this resolver.""" + if "issue" not in self.definitions.filter_aliases or self.params.organization_id is None: + return + + collected = collect_issue_short_ids_from_parsed_terms(parsed_terms) + if not collected: + return + try: + groups = list( + Group.objects.by_qualified_short_id_bulk( + organization_id=self.params.organization_id, + short_ids_raw=list(collected), + # org-wide: the Snuba query is already scoped to the requested projects. + project_ids=None, + ) + ) + except Group.DoesNotExist: + raise InvalidIssueSearchQuery(sorted(collected)) + + idx = {(g.project.slug.lower(), g.short_id): g for g in groups} + + for raw in collected: + parsed = parse_short_id(raw) + if parsed is None: + raise InvalidIssueSearchQuery(sorted(collected)) + g = idx.get((parsed.project_slug, parsed.short_id)) + if g is None: + raise InvalidIssueSearchQuery(sorted(collected)) + if g.project.id not in self.qualified_short_id_to_group_id_cache: + self.qualified_short_id_to_group_id_cache[g.project.id] = {} + self.qualified_short_id_to_group_id_cache[g.project.id][raw] = g.id + + def parse_search_query(self, querystring: str) -> Sequence[event_search.QueryToken]: + """Helper function so this can be called by validate separately""" + return event_search.parse_search_query( + querystring, + config=event_search.SearchConfig.create_from( + event_search.default_config, + wildcard_free_text=True, + allow_regex=self._allows_regex(), + ), + params=self.params.filter_params, + get_field_type=self.get_field_type, + get_function_result_type=self.get_field_type, + ) + + def _allows_regex(self) -> bool: + return self.definitions.trace_item_type == TraceItemType.TRACE_ITEM_TYPE_LOG + + def collect_terms(self, parsed_terms: Sequence[event_search.QueryToken]) -> list[str]: + """Helper function to collect all the search terms from a parsed query ignoring the actual query tree""" + terms = [] + for term in parsed_terms: + if event_search.SearchBoolean.is_operator(term): + continue + elif isinstance(term, event_search.ParenExpression): + for collected_term in self.collect_terms(term.children): + if collected_term not in terms: + terms.append(collected_term) + else: + if isinstance(term, event_search.SearchFilter): + for converted_term in self.convert_term(term): + column = converted_term.key.name + if column not in terms: + terms.append(column) + else: + column = term.key.name + if column not in terms: + terms.append(column) + return terms + + def __resolve_query( + self, querystring: str | None + ) -> tuple[ + TraceItemFilter | None, + AggregationFilter | None, + list[VirtualColumnDefinition | None], + ]: + if querystring is None: + return None, None, [] + try: + parsed_terms = self.parse_search_query(querystring) + except ParseError as e: + if e.expr is not None: + raise InvalidSearchQuery(f"Parse error: {e.expr.name} (column {e.column():d})") + else: + raise InvalidSearchQuery(f"Parse error for: {querystring}") + + # If occurrences dataset, cache group_id to issues mapping. + self._init_issue_short_id_cache(parsed_terms) + + has_boolean = False + for term in parsed_terms: + if (isinstance(term, event_search.ParenExpression)) or ( + event_search.SearchBoolean.is_operator(term) and len(parsed_terms) > 1 + ): + has_boolean = True + break + if has_boolean: + return self._resolve_boolean_conditions(parsed_terms) + else: + return self._resolve_terms(parsed_terms) + + def _resolve_boolean_conditions( + self, terms: event_filter.ParsedTerms + ) -> tuple[ + TraceItemFilter | None, + AggregationFilter | None, + list[VirtualColumnDefinition | None], + ]: + if len(terms) == 0: + return None, None, [] + elif len(terms) == 1: + if isinstance(terms[0], event_search.ParenExpression): + return self._resolve_boolean_conditions(terms[0].children) + elif isinstance(terms[0], event_search.SearchFilter): + return self._resolve_terms([terms[0]]) + elif isinstance(terms[0], event_search.AggregateFilter): + return self._resolve_terms([terms[0]]) + else: + return self._resolve_terms(terms) + + # Filter out any ANDs since we can assume anything without an OR is an AND. Also do some + # basic sanitization of the query: can't have two operators next to each other, and can't + # start or end a query with an operator. + previous_term: event_filter.ParsedTerm | None = None + new_terms = [] + term: event_filter.ParsedTerm | None = None + for term in terms: + if previous_term: + if event_search.SearchBoolean.is_operator( + previous_term + ) and event_search.SearchBoolean.is_operator(term): + raise InvalidSearchQuery( + f"Missing condition in between two condition operators: '{previous_term} {term}'" + ) + else: + if event_search.SearchBoolean.is_operator(term): + raise InvalidSearchQuery( + f"Condition is missing on the left side of '{term}' operator" + ) + + if term != event_search.SearchBoolean.BOOLEAN_AND: + new_terms.append(term) + + previous_term = term + + if term is not None and event_search.SearchBoolean.is_operator(term): + raise InvalidSearchQuery(f"Condition is missing on the right side of '{term}' operator") + terms = new_terms + + # We put precedence on AND, which sort of counter-intuitively means we have to split the query + # on ORs first, so the ANDs are grouped together. Search through the query for ORs and split the + # query on each OR. + # We want to maintain a binary tree, so split the terms on the first OR we can find and recurse on + # the two sides. If there is no OR, split the first element out to AND + index = None + lhs, rhs = None, None + try: + index = terms.index(event_search.SearchBoolean.BOOLEAN_OR) + lhs, rhs = terms[:index], terms[index + 1 :] + operator: Literal["and" | "or"] = "or" + except Exception: + lhs, rhs = terms[:1], terms[1:] + operator = "and" + + where_lhs, having_lhs, contexts_lhs = self._resolve_boolean_conditions(lhs) + where_rhs, having_rhs, contexts_rhs = self._resolve_boolean_conditions(rhs) + contexts = contexts_lhs + contexts_rhs + + where = None + having = None + + if where_lhs is not None and where_rhs is not None: + if operator == "and": + where = TraceItemFilter(and_filter=AndFilter(filters=[where_lhs, where_rhs])) + else: + where = TraceItemFilter(or_filter=OrFilter(filters=[where_lhs, where_rhs])) + elif where_lhs is None and where_rhs is not None: + where = where_rhs + elif where_lhs is not None and where_rhs is None: + where = where_lhs + + if having_lhs is not None and having_rhs is not None: + if operator == "and": + having = AggregationFilter( + and_filter=AggregationAndFilter(filters=[having_lhs, having_rhs]) + ) + else: + having = AggregationFilter( + or_filter=AggregationOrFilter(filters=[having_lhs, having_rhs]) + ) + elif having_lhs is None and having_rhs is not None: + having = having_rhs + elif having_lhs is not None and having_rhs is None: + having = having_lhs + + return where, having, contexts + + def _resolve_terms( + self, terms: event_filter.ParsedTerms + ) -> tuple[ + TraceItemFilter | None, + AggregationFilter | None, + list[VirtualColumnDefinition | None], + ]: + where, where_contexts = self._resolve_where(terms) + having, having_contexts = self._resolve_having(terms) + return where, having, where_contexts + having_contexts + + def _resolve_where( + self, terms: event_filter.ParsedTerms + ) -> tuple[TraceItemFilter | None, list[VirtualColumnDefinition | None]]: + parsed_terms = [] + resolved_contexts = [] + for item in terms: + if isinstance(item, event_search.SearchFilter): + resolved_term, resolved_context = self.resolve_term(item) + parsed_terms.extend(resolved_term) + resolved_contexts.extend(resolved_context) + + if len(parsed_terms) > 1: + return TraceItemFilter(and_filter=AndFilter(filters=parsed_terms)), resolved_contexts + elif len(parsed_terms) == 1: + return parsed_terms[0], resolved_contexts + return None, [] + + def _resolve_having( + self, terms: event_filter.ParsedTerms + ) -> tuple[AggregationFilter | None, list[VirtualColumnDefinition | None]]: + if not self.config.use_aggregate_conditions: + return None, [] + + parsed_terms = [] + resolved_contexts = [] + for item in terms: + if isinstance(item, event_search.AggregateFilter): + resolved_term, resolved_context = self.resolve_aggregate_term(item) + parsed_terms.append(resolved_term) + resolved_contexts.extend(resolved_context) + + if len(parsed_terms) > 1: + return ( + AggregationFilter(and_filter=AggregationAndFilter(filters=parsed_terms)), + resolved_contexts, + ) + elif len(parsed_terms) == 1: + return parsed_terms[0], resolved_contexts + return None, [] + + def resolve_virtual_context_term( + self, + term: str, + raw_value: str | list[str], + resolved_column: ResolvedAttribute, + context: VirtualColumnDefinition, + ) -> list[str] | str: + # Convert the term to the expected values + final_raw_value: str | list[str] = [] + resolved_context = context.constructor(self.params, self) + reversed_context = {v: k for k, v in resolved_context.value_map.items()} + if isinstance(raw_value, list): + new_value = [] + for raw_iterable in raw_value: + if context.default_value and context.default_value == raw_iterable: + # Avoiding this for now, while this could work with the Unknown:"" mapping + # But that won't work once we use the VirtualColumnContext.default_value + raise InvalidSearchQuery( + f"Using {raw_iterable} in an IN filter is not currently supported" + ) + elif raw_iterable not in reversed_context: + valid_values = list(reversed_context.keys())[:5] + if len(valid_values) > 5: + valid_values.append("...") + raise InvalidSearchQuery( + constants.REVERSE_CONTEXT_ERROR.format( + raw_value, term, ", ".join(valid_values) + ) + ) + else: + new_value.append(reversed_context[raw_iterable]) + final_raw_value = new_value + elif raw_value in reversed_context: + final_raw_value = reversed_context[raw_value] + elif context.default_value and context.default_value == raw_value: + # Avoiding this for now, while this could work with the Unknown:"" mapping + # But that won't work once we use the VirtualColumnContext.default_value + raise InvalidSearchQuery(f"Using {raw_value} is not currently supported") + else: + valid_values = list(reversed_context.keys())[:5] + if len(valid_values) > 5: + valid_values.append("...") + raise InvalidSearchQuery( + constants.REVERSE_CONTEXT_ERROR.format( + raw_value, term, ", ".join(list(reversed_context.keys())[:5]) + ) + ) + return final_raw_value + + def convert_term(self, term: event_search.SearchFilter) -> list[event_search.SearchFilter]: + name = term.key.name + + converter = self.definitions.filter_aliases.get(name) + if converter is not None: + if term.value.is_regex: + # The converters resolve values against Sentry models, so they would treat the + # pattern as a literal rather than matching against it + raise InvalidSearchQuery(f"Cannot use regular expressions with {name}") + return converter(self.params, term, self) + + return [term] + + def resolve_term( + self, term: event_search.SearchFilter + ) -> tuple[list[TraceItemFilter], list[VirtualColumnDefinition | None]]: + terms = self.convert_term(term) + + resolved_terms = [] + resolved_contexts = [] + + for t in terms: + resolved_term, resolved_context = self._resolve_term(t) + resolved_terms.append(resolved_term) + resolved_contexts.append(resolved_context) + + return resolved_terms, resolved_contexts + + def _resolve_term( + self, term: event_search.SearchFilter + ) -> tuple[TraceItemFilter, VirtualColumnDefinition | None]: + resolved_column, context_definitions = self.resolve_column(term.key.name) + non_none_definitions = [defn for defn in context_definitions if defn is not None] + if len(non_none_definitions) > 1: + raise ValueError(f"{term.key.name} resolved with multiple contexts") + elif len(non_none_definitions) == 1: + context_definition = non_none_definitions[0] + else: + context_definition = None + + self._raise_if_hidden_api_attribute(term.key.name, resolved_column) + + if term.value.is_regex: + return self._resolve_regex_term(term, resolved_column, context_definition), None + + if context_definition is not None and term.value.is_wildcard(): + raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}") + + value = term.value.value + if self.params.is_timeseries_request and context_definition is not None: + resolved_column, value = self.map_search_term_context_to_original_column( + term, context_definition + ) + self._raise_if_hidden_api_attribute(term.key.name, resolved_column) + context_definition = None + + if not isinstance(resolved_column.proto_definition, AttributeKey): + raise ValueError(f"{term.key.name} is not valid search term") + + if term.value.is_wildcard(): + is_list = False + if term.operator == "=": + operator = ComparisonFilter.OP_LIKE + elif term.operator == "!=": + operator = ComparisonFilter.OP_NOT_LIKE + elif term.operator == "IN": + operator = ComparisonFilter.OP_LIKE + is_list = True + elif term.operator == "NOT IN": + operator = ComparisonFilter.OP_NOT_LIKE + is_list = True + else: + raise InvalidSearchQuery(f"Cannot use operator: {term.operator} with wildcards") + + if is_list: + raw_value = cast(list[str], term.value.raw_value) + filters = [ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=self._resolve_search_value( + resolved_column, + ( + "=" if operator == ComparisonFilter.OP_LIKE else "!=" + ), # tell this function the single operator since its being ORed + event_search.translate_wildcard_as_clickhouse_pattern(str(value)), + ), + ) + ) + for value in raw_value + ] + return ( + ( + TraceItemFilter(or_filter=OrFilter(filters=filters)) + if term.operator == "IN" + else TraceItemFilter(and_filter=AndFilter(filters=filters)) + ), + context_definition, + ) + else: + value = str(term.value.raw_value) + value = event_search.translate_wildcard_as_clickhouse_pattern(value) + elif term.operator in constants.OPERATOR_MAP: + operator = constants.OPERATOR_MAP[term.operator] + else: + raise InvalidSearchQuery(f"Unknown operator: {term.operator}") + + if value is None: + exists_filter = TraceItemFilter( + exists_filter=ExistsFilter( + key=resolved_column.proto_definition, + ) + ) + if term.operator == "=": + not_exists_filter = TraceItemFilter(not_filter=NotFilter(filters=[exists_filter])) + return not_exists_filter, context_definition + elif term.operator == "!=": + return exists_filter, context_definition + else: + raise InvalidSearchQuery(f"Unsupported operator for None {term.operator}") + + # Handle lists containing None for IN/NOT IN operators + if isinstance(value, list) and None in value and term.operator in constants.IN_OPERATORS: + non_none_values = [v for v in value if v is not None] + + exists_filter = TraceItemFilter( + exists_filter=ExistsFilter( + key=resolved_column.proto_definition, + ) + ) + + if term.operator == "IN": + filters = [] + filters.append(TraceItemFilter(not_filter=NotFilter(filters=[exists_filter]))) + + if non_none_values: + filters.append( + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=self._resolve_search_value( + resolved_column, term.operator, non_none_values + ), + ignore_case=self.params.case_insensitive + and resolved_column.search_type == "string", + ) + ) + ) + + return ( + TraceItemFilter(or_filter=OrFilter(filters=filters)), + context_definition, + ) + else: # NOT IN + filters = [exists_filter] + + if non_none_values: + filters.append( + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=self._resolve_search_value( + resolved_column, term.operator, non_none_values + ), + ignore_case=self.params.case_insensitive + and resolved_column.search_type == "string", + ) + ) + ) + + return ( + TraceItemFilter(and_filter=AndFilter(filters=filters)), + context_definition, + ) + + if value == "" and context_definition is None: + exists_filter = TraceItemFilter( + exists_filter=ExistsFilter( + key=resolved_column.proto_definition, + ) + ) + # Attributes that reject the empty string can never hold it, and comparing against it + # fails in snuba for the ones backed by a typed column (eg. trace id parsed as a UUID). + compare_to_empty_string = ( + resolved_column.proto_definition.type == constants.STRING + and resolved_column.allows_value(value) + ) + if term.operator == "!=": + filters = [exists_filter] + if compare_to_empty_string: + filters.append( + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=AttributeValue(val_str=value), + ) + ) + ) + return ( + TraceItemFilter(and_filter=AndFilter(filters=filters)), + context_definition, + ) + elif term.operator == "=": + filters = [TraceItemFilter(not_filter=NotFilter(filters=[exists_filter]))] + if compare_to_empty_string: + filters.append( + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=AttributeValue(val_str=value), + ) + ) + ) + return ( + TraceItemFilter(or_filter=OrFilter(filters=filters)), + context_definition, + ) + else: + raise InvalidSearchQuery(f"Unsupported operator for empty strings {term.operator}") + + if not self.params.is_timeseries_request and context_definition: + value = self.remap_value_using_context_definition(context_definition, value) + + return ( + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=operator, + value=self._resolve_search_value(resolved_column, term.operator, value), + ignore_case=self.params.case_insensitive + and resolved_column.search_type == "string", + ) + ), + context_definition, + ) + + def map_context_to_original_column( + self, + context_definition: VirtualColumnDefinition, + ) -> ResolvedAttribute: + """ + Time series request do not support virtual column contexts, so we have to remap the value back to the original column. + (see https://github.com/getsentry/eap-planning/issues/236) + """ + context = context_definition.constructor(self.params, self) + + is_number_column = ( + context.from_column_name in SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"] + ) + + public_alias = ( + SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"].get(context.from_column_name) + if is_number_column + else context.from_column_name + ) + + if public_alias is None: + raise InvalidSearchQuery(f"Cannot map {context.from_column_name} to a public alias") + + resolved_column, _ = self.resolve_column(public_alias) + + if not isinstance(resolved_column.proto_definition, AttributeKey): + raise ValueError(f"{resolved_column.public_alias} is not valid search term") + + self._raise_if_hidden_api_attribute(context.to_column_name, resolved_column) + + return resolved_column + + def map_search_term_context_to_original_column( + self, + term: event_search.SearchFilter, + context_definition: VirtualColumnDefinition, + ) -> tuple[ResolvedAttribute, str | int | list[str]]: + """ + Time series request do not support virtual column contexts, so we have to remap the value back to the original column. + (see https://github.com/getsentry/eap-planning/issues/236) + """ + context = context_definition.constructor(self.params, self) + is_number_column = ( + context.from_column_name in SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"] + ) + + resolved_column = self.map_context_to_original_column(context_definition) + + value = term.value.value + + inverse_value_map: dict[str, list[str]] = {} + for key, val in context.value_map.items(): + inverse_value_map[val] = inverse_value_map.get(val, []) + [key] + + def remap_value(old_value: str) -> list[str]: + if old_value in inverse_value_map: + return inverse_value_map[old_value] + elif old_value in context.value_map: + return [old_value] + elif context.default_value: + return [context.default_value] + else: + raise InvalidSearchQuery(f"Unknown value {old_value}") + + final_value: list[str] = [] + if isinstance(value, list): + value_set: set[str] = set() + for v in value: + for mapped_values in remap_value(v): + value_set.add(mapped_values) + + final_value = list(value_set) + + else: + final_value = remap_value(value) + + if len(final_value) == 1 and not isinstance(value, list): + if is_number_column: + return resolved_column, int(final_value[0]) + return resolved_column, final_value[0] + + return resolved_column, final_value + + def resolve_aggregate_term( + self, term: event_search.AggregateFilter + ) -> tuple[AggregationFilter, list[VirtualColumnDefinition | None]]: + resolved_column, context = self.resolve_column(term.key.name) + self._raise_if_hidden_api_attribute(term.key.name, resolved_column) + proto_definition = resolved_column.proto_definition + + if not isinstance( + proto_definition, + (AttributeAggregation, AttributeConditionalAggregation, Column.BinaryFormula), + ): + raise ValueError(f"{term.key.name} is not valid search term") + + # TODO: Handle different units properly + value = term.value.value + + if term.operator in constants.OPERATOR_MAP: + operator = constants.AGGREGATION_OPERATOR_MAP[term.operator] + else: + raise InvalidSearchQuery(f"Unknown operator: {term.operator}") + + kwargs = {"op": operator, "val": value} + if isinstance(proto_definition, AttributeAggregation): + aggregation_key = "aggregation" + elif isinstance(proto_definition, AttributeConditionalAggregation): + aggregation_key = "conditional_aggregation" + elif isinstance(proto_definition, Column.BinaryFormula): + aggregation_key = "formula" + else: + raise InvalidSearchQuery(f"{term.key.name} is not a valid search") + + kwargs[aggregation_key] = proto_definition + return ( + AggregationFilter( + comparison_filter=AggregationComparisonFilter(**kwargs), + ), + context, + ) + + def _resolve_regex_term( + self, + term: event_search.SearchFilter, + resolved_column: ResolvedAttribute | ResolvedFunction, + context_definition: VirtualColumnDefinition | None, + ) -> TraceItemFilter: + if context_definition is not None: + raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") + + key = resolved_column.proto_definition + if not isinstance(key, AttributeKey) or key.type not in constants.REGEXP_ATTRIBUTE_TYPES: + raise InvalidSearchQuery( + f"Cannot use regular expressions with {term.key.name}, it is not a string attribute" + ) + + # Snuba's `ignore_case` lowercases the pattern along with the value, rewriting `[A-Z]` + # and inverting escapes like `\D`. RE2's inline flag leaves the pattern intact. + prefix = "(?i)" if self.params.case_insensitive else "" + match = TraceItemFilter( + comparison_filter=ComparisonFilter( + key=key, + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str=f"{prefix}{term.value.raw_value}"), + ) + ) + + if term.operator == "!=": + # There is no OP_NOT_REGEXP, so negation is expressed by wrapping the match + return TraceItemFilter(not_filter=NotFilter(filters=[match])) + return match + + def _resolve_search_value( + self, + column: ResolvedAttribute, + operator: str, + value: str | float | datetime | Sequence[float] | Sequence[str], + ) -> AttributeValue: + column.validate(value) + value = column.normalize(value) + if isinstance(column.proto_definition, AttributeKey): + column_type = column.proto_definition.type + if column_type == constants.STRING: + if operator in constants.IN_OPERATORS: + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return AttributeValue(val_str_array=StrArray(values=value)) + else: + raise InvalidSearchQuery( + f"{value} is not a valid value for doing an IN filter" + ) + else: + return AttributeValue(val_str=str(value)) + elif column_type == constants.INT: + # The search parser will always convert a value to a float, so we need to cast back to an int + if operator in constants.IN_OPERATORS: + if isinstance(value, list): + return AttributeValue( + val_int_array=IntArray(values=[int(val) for val in value]) + ) + else: + raise InvalidSearchQuery( + f"{value} is not a valid value for doing an IN filter" + ) + elif isinstance(value, (float, int)): + return AttributeValue(val_int=int(value)) + elif column_type == constants.DOUBLE: + if operator in constants.IN_OPERATORS: + if isinstance(value, list): + return AttributeValue( + val_double_array=DoubleArray( + values=[ + val.timestamp() if isinstance(val, datetime) else val + for val in value + ] + ) + ) + else: + raise InvalidSearchQuery( + f"{value} is not a valid value for doing an IN filter" + ) + elif isinstance(value, datetime): + return AttributeValue(val_double=value.timestamp()) + elif isinstance(value, float): + return AttributeValue(val_double=value) + elif column_type == constants.BOOLEAN: + if operator in constants.IN_OPERATORS: + raise InvalidSearchQuery( + f"{column.public_alias} cannot be used with an IN filter" + ) + elif isinstance(value, str): + lowered_value = value.lower() + if lowered_value not in constants.BOOLEAN_VALUES: + raise InvalidSearchQuery( + f"{value} is not a valid boolean value, expecting true or false" + ) + bool_value = lowered_value in constants.TRUTHY_VALUES + return AttributeValue(val_bool=bool_value) + elif isinstance(value, bool): + return AttributeValue(val_bool=value) + elif column_type == constants.ARRAY: + # Only scalar value membership in an array is allowed. + # Allowed operators: =,!=, LIKE, NOT_LIKE. + # TODO: Add support for scalar: >, < for numbers + if operator in constants.IN_OPERATORS: + raise InvalidSearchQuery( + f"{column.public_alias} (array) cannot be used with an IN filter; " + f"use {column.public_alias}[*]:value for membership" + ) + # All primitive types are converted to strings on EAP before comparison. + return AttributeValue(val_str=str(value)) + raise InvalidSearchQuery( + f"{value} is not a valid filter value for {column.public_alias}, expecting {constants.TYPE_TO_STRING_MAP[column_type]}, but got a {type(value)}" + ) + else: + raise NotImplementedError("Aggregate Queries not implemented yet") + + def resolve_contexts( + self, context_definitions: list[VirtualColumnDefinition | None] + ) -> list[VirtualColumnContext]: + """Given a list of contexts that may have None in them, remove the Nones and remove the dupes""" + final_contexts = [] + existing_target_columns = set() + for context_definition in context_definitions: + if context_definition is None: + continue + context = context_definition.constructor(self.params, self) + if context is None or context.to_column_name in existing_target_columns: + continue + else: + existing_target_columns.add(context.to_column_name) + final_contexts.append(context) + return final_contexts + + @traces.trace + def resolve_columns( + self, selected_columns: list[str], has_aggregates: bool = False + ) -> tuple[ + list[ResolvedAttribute | ResolvedFunction], + list[VirtualColumnDefinition | None], + ]: + """Given a list of columns resolve them and get their context if applicable + + This function will also dedupe the virtual column contexts if necessary + """ + span = traces.get_current_span() + resolved_columns = [] + resolved_contexts = [] + stripped_columns = [column.strip() for column in selected_columns] + if span: + span.set_attribute("SearchResolver.selected_columns", stripped_columns) + for column in stripped_columns: + match = fields.is_function(column) + has_aggregates = has_aggregates or match is not None + try: + resolved_column, context = self.resolve_column(column, match) + except HiddenApiAttribute: + continue + if isinstance(resolved_column, ResolvedAttribute) and self._should_hide_api_attribute( + column, resolved_column + ): + continue + if ( + self.config.disable_array_attributes + and isinstance(resolved_column, ResolvedAttribute) + and resolved_column.internal_type == constants.ARRAY + ): + continue + resolved_columns.append(resolved_column) + resolved_contexts.extend(context) + + if self.config.auto_fields: + # Ensure fields we require to build a functioning interface are present. + if not has_aggregates and "id" not in stripped_columns: + id_column, id_context = self.resolve_column("id") + resolved_columns.append(id_column) + resolved_contexts.extend(id_context) + stripped_columns.append("id") + if "id" in stripped_columns and "project.id" not in stripped_columns: + project_column, project_context = self.resolve_column("project.name") + resolved_columns.append(project_column) + resolved_contexts.extend(project_context) + + return resolved_columns, resolved_contexts + + def resolve_column( + self, + column: str, + match: Match[str] | None = None, + public_alias_override: str | None = None, + default_value: float | None = None, + ) -> tuple[ + ResolvedAttribute | ResolvedFunction, + list[VirtualColumnDefinition | None], + ]: + """Column is either an attribute or an aggregate, this function will determine which it is and call the relevant + resolve function""" + match = fields.is_function(column) + if match: + return self.resolve_function( + column, match, public_alias_override, default_value=default_value + ) + else: + return self.resolve_attribute(column, public_alias_override) + + def get_field_type(self, column: str) -> str: + resolved_column, _ = self.resolve_column(column) + return resolved_column.search_type + + @traces.trace + def resolve_attributes( + self, columns: list[str] + ) -> tuple[list[ResolvedAttribute], list[VirtualColumnDefinition | None]]: + """Helper function to resolve a list of attributes instead of 1 attribute at a time""" + resolved_columns = [] + resolved_contexts = [] + for column in columns: + col, context = self.resolve_attribute(column) + self._raise_if_hidden_api_attribute(column, col) + if self.config.disable_array_attributes and col.internal_type == constants.ARRAY: + continue + resolved_columns.append(col) + resolved_contexts.extend(context) + return resolved_columns, resolved_contexts + + def should_hide_api_column( + self, column: str, resolved_column: ResolvedAttribute | ResolvedFunction + ) -> bool: + if not isinstance(resolved_column, ResolvedAttribute): + return False + return self._should_hide_api_attribute(column, resolved_column) + + def _should_hide_api_attribute( + self, column: str, resolved_attribute: ResolvedAttribute + ) -> bool: + if self.config.api_attribute_visibility_item_type is None: + return False + + from sentry.search.eap.utils import can_expose_attribute_to_api + + item_type = SupportedTraceItemType(self.config.api_attribute_visibility_item_type) + if column in self.definitions.contexts and resolved_attribute.internal_name != column: + visibility_attribute = resolved_attribute.internal_name + elif column in self.definitions.contexts or column in self.definitions.columns: + visibility_attribute = column + else: + visibility_attribute = resolved_attribute.internal_name + return not can_expose_attribute_to_api( + visibility_attribute, + item_type, + include_internal=self.config.api_attribute_visibility_include_internal, + include_internal_convention_attributes=self.config.api_attribute_visibility_include_internal_convention_attributes, + ) + + def _raise_if_hidden_api_attribute( + self, column: str, resolved_column: ResolvedAttribute | ResolvedFunction + ) -> None: + if isinstance(resolved_column, ResolvedAttribute) and self._should_hide_api_attribute( + column, resolved_column + ): + raise HiddenApiAttribute(f"Could not parse {column}") + + def resolve_attribute( + self, column: str, public_alias_override: str | None = None + ) -> tuple[ResolvedAttribute, list[VirtualColumnDefinition | None]]: + """Attributes are columns that aren't 'functions' or 'aggregates', usually this means string or numeric + attributes (aka. tags), but can also refer to fields like span.description""" + # If a virtual context is defined the column definition is always the same + if column in self._resolved_attribute_cache: + return self._resolved_attribute_cache[column] + + alias = column + if public_alias_override is not None: + alias = public_alias_override + + if column in self.definitions.contexts: + column_context = self.definitions.contexts[column] + column_definition = ResolvedAttribute( + public_alias=alias, + internal_name=column, + search_type="string", + processor=column_context.processor, + ) + elif column in self.definitions.columns: + column_context = None + column_definition = self.definitions.columns[column] + if column_definition.private and column not in self.config.fields_acl.attributes: + raise InvalidSearchQuery(f"The field {column} is not allowed for this query") + # Need to override the ResolvedAttribute entirely + if public_alias_override: + column_definition = ResolvedAttribute( + public_alias=public_alias_override, + internal_name=column_definition.internal_name, + search_type=column_definition.search_type, + ) + elif (internal_match := self._find_column_by_internal_name(column)) is not None: + column_context = None + column_definition = ResolvedAttribute( + public_alias=alias, + internal_name=internal_match.internal_name, + search_type=internal_match.search_type, + internal_type=internal_match.internal_type, + validator=internal_match.validator, + normalizer=internal_match.normalizer, + processor=internal_match.processor, + ) + else: + if len(column) > qb_constants.MAX_TAG_KEY_LENGTH: + raise InvalidSearchQuery( + f"{column} is too long, can be a maximum of 200 characters" + ) + + tag_match = qb_constants.TYPED_TAG_KEY_RE.search(column) + if tag_match is None: + tag_match = qb_constants.TAG_KEY_RE.search(column) + field_type = "string" + else: + field_type = None + # make sure to remove surrounding quotes if it's a tag + field = tag_match.group("tag") if tag_match else column + if field is None: + raise InvalidSearchQuery(f"Could not parse {column}") + # Assume string if a type isn't passed. eg. tags[foo] + if field_type is None: + field_type = tag_match.group("type") if tag_match else None + + if field_type not in constants.TYPE_MAP: + raise InvalidSearchQuery(f"Unsupported type {field_type} in {column}") + + if column.startswith("sentry_tags"): + field = f"sentry.{field}" + + if self.definitions.alias_to_column is not None: + mapped_column = self.definitions.alias_to_column(field) + if mapped_column is not None: + field = mapped_column + + column_definition = ResolvedAttribute( + public_alias=alias, internal_name=field, search_type=field_type + ) + column_context = None + + if column_definition: + self._resolved_attribute_cache[column] = (column_definition, [column_context]) + return self._resolved_attribute_cache[column] + else: + raise InvalidSearchQuery(f"Could not parse {column}") + + @traces.trace + def resolve_functions( + self, columns: list[str] + ) -> tuple[ + list[ResolvedFunction], + list[VirtualColumnDefinition | None], + ]: + """Helper function to resolve a list of functions instead of 1 attribute at a time""" + resolved_functions, resolved_contexts = [], [] + for column in columns: + try: + function, contexts = self.resolve_function(column) + except HiddenApiAttribute: + continue + resolved_functions.append(function) + resolved_contexts.extend(contexts) + return resolved_functions, resolved_contexts + + def resolve_function( + self, + column: str, + match: Match[str] | None = None, + public_alias_override: str | None = None, + default_value: float | None = None, + ) -> tuple[ResolvedFunction, list[VirtualColumnDefinition | None]]: + if match is None: + match = fields.is_function(column) + if match is None: + raise InvalidSearchQuery(f"{column} is not a function") + + function_name = match.group("function") + columns = match.group("columns") + # Alias defaults to the name of the function + alias = match.group("alias") or column + if public_alias_override is not None: + alias = public_alias_override + + # Don't use cache if default_value is passed + if alias in self._resolved_function_cache and default_value is None: + return self._resolved_function_cache[alias] + # Check if the column looks like a function (matches a pattern), parse the function name and args out + + if self.config.saved_formulas is not None: + if function_name in self.config.saved_formulas: + resolved_column, contexts = self.resolve_formula( + self.config.saved_formulas[function_name], columns, alias + ) + if not isinstance(resolved_column, ResolvedFunction): + raise InvalidSearchQuery( + f"The formula {function_name} must resolve to a function or equation" + ) + return resolved_column, contexts + + function_definition = self.get_function_definition(function_name) + if ( + self.definitions.aggregate_deprecations + and function_name in self.definitions.aggregate_deprecations + ): + deprecated_definition = self.definitions.aggregate_deprecations[function_name] + else: + deprecated_definition = None + + # Temporary while we get rid of the old _if combinators + if deprecated_definition: + try: + return self._resolve_function( + function_definition, function_name, alias, columns, default_value + ) + except Exception: + logger.warning( + "resolver.deprecated_function_used", + extra={ + "organization_id": self.params.organization_id, + "function_name": function_name, + "column": column, + "query_string": self.params.query_string, + }, + ) + if self.config.disable_deprecated_functions: + raise + return self._resolve_function( + deprecated_definition, function_name, alias, columns, default_value + ) + else: + return self._resolve_function( + function_definition, function_name, alias, columns, default_value + ) + + def _resolve_function( + self, + function_definition: FormulaDefinition | AggregateDefinition, + function_name: str, + alias: str, + columns: str, + default_value: float | None = None, + ) -> tuple[ResolvedFunction, list[VirtualColumnDefinition | None]]: + if function_definition.private and function_name not in self.config.fields_acl.functions: + raise InvalidSearchQuery(f"The function {function_name} is not allowed for this query") + + parsed_args: list[ResolvedAttribute | str | int | float | TraceItemFilter] = [] + + # Parse the arguments + arguments = fields.parse_arguments(function_name, columns) + if len(arguments) < len(function_definition.required_arguments): + raise InvalidSearchQuery( + f"Invalid number of arguments for {function_name}, was expecting {len(function_definition.required_arguments)} arguments" + ) + + missing_args = len(function_definition.arguments) - len(arguments) + argument_index = 0 + + for argument_definition in function_definition.arguments: + if argument_definition.ignored: + continue + + # If there are missing arguments, and the argument definition has a default arg, use the default arg + # this assumes the missing args are at the beginning or end of the arguments list + if missing_args > 0 and argument_definition.default_arg is not None: + if isinstance(argument_definition, ValueArgumentDefinition): + parsed_args.append(argument_definition.default_arg) + else: + parsed_argument, _ = self.resolve_attribute(argument_definition.default_arg) + self._raise_if_hidden_api_attribute( + argument_definition.default_arg, parsed_argument + ) + parsed_args.append(parsed_argument) + missing_args -= 1 + continue + + if argument_index < len(arguments): + argument = arguments[argument_index] + argument_index += 1 + if argument_definition.validator is not None: + if not argument_definition.validator(argument): + raise InvalidSearchQuery( + f"{argument} is not a valid argument for {function_name}" + ) + if isinstance(argument_definition, AttributeArgumentDefinition): + parsed_argument, _ = self.resolve_attribute(argument) + self._raise_if_hidden_api_attribute(argument, parsed_argument) + parsed_args.append(parsed_argument) + else: + if argument_definition.argument_types is None: + parsed_args.append(argument) # assume it's a string + continue + for arg_type in argument_definition.argument_types: + if arg_type == "integer": + parsed_args.append(int(argument)) + elif arg_type == "number": + parsed_args.append(float(argument)) + elif arg_type == "query": + # Only TraceItemFilter currently supported + trace_item_filters = self.resolve_query(argument[1:-1])[0] + if trace_item_filters is None: + raise InvalidSearchQuery( + "The if combinator requires non-aggregate filters" + ) + parsed_args.append(trace_item_filters) + else: + parsed_args.append(argument) + continue + else: + raise InvalidSearchQuery( + f"Invalid number of arguments for {function_name}, was expecting {len(function_definition.required_arguments)} arguments" + ) + + if isinstance(argument_definition, AttributeArgumentDefinition) and ( + argument_definition.attribute_types is not None + and parsed_argument.search_type not in argument_definition.attribute_types + ): + if argument_definition.field_allowlist is not None: + if parsed_argument.public_alias not in argument_definition.field_allowlist: + raise InvalidSearchQuery( + f"{parsed_argument.public_alias} is invalid for parameter {argument_index} in {function_name}." + ) + else: + raise InvalidSearchQuery( + f"{parsed_argument.public_alias} is invalid for parameter {argument_index} in {function_name}. Its a {parsed_argument.search_type} type field, but it must be one of these types: {argument_definition.attribute_types}" + ) + + resolved_arguments: ResolvedArguments = [] + for parsed_arg in parsed_args: + resolved_argument: ResolvedArgument + if not isinstance(parsed_arg, ResolvedAttribute): + resolved_argument = parsed_arg + search_type = function_definition.default_search_type + else: + resolved_argument = parsed_arg.proto_definition + resolved_arguments.append(resolved_argument) + + if len(parsed_args) == 0 or not isinstance(parsed_args[0], ResolvedAttribute): + search_type = function_definition.default_search_type + else: + # unless infer_search_type_from_arguments is passed we assume the first argument is the search_type + search_type = ( + parsed_args[0].search_type + if function_definition.infer_search_type_from_arguments + else function_definition.default_search_type + ) + + resolved_function = function_definition.resolve( + alias=alias, + search_type=search_type, + resolved_arguments=resolved_arguments, + snuba_params=self.params, + query_result_cache=self._query_result_cache, + search_config=self.config, + default_value=default_value, + ) + + resolved_context: list[VirtualColumnDefinition | None] = [None] + if default_value is None: + self._resolved_function_cache[alias] = (resolved_function, resolved_context) + return self._resolved_function_cache[alias] + else: + return resolved_function, resolved_context + + def resolve_equations( + self, equations: list[str] + ) -> tuple[ + list[ResolvedColumn], + list[VirtualColumnDefinition | None], + ]: + formulas = [] + contexts = [] + for equation in equations: + try: + formula, context = self.resolve_equation(equation) + except HiddenApiAttribute: + continue + formulas.append(formula) + contexts.extend(context) + return formulas, contexts + + def resolve_equation( + self, equation: str, alias: str | None = None + ) -> tuple[ + ResolvedColumn, + list[VirtualColumnDefinition | None], + ]: + """Resolve an equation creating a ResolvedEquation object, we don't just return a Column.BinaryFormula since + it'll help callers with extra information, like the existence of aggregates and the search type + """ + operation, fields, functions = arithmetic.parse_arithmetic( + equation, definitions=self.definitions + ) + # Handle the case where the equation is just a single term + if isinstance(operation, str): + # Resolve the column, and turn it into a RPC Column so it can be used in a BinaryFormula + col, context = self.resolve_column( + operation, public_alias_override=f"equation|{equation}" if alias is None else alias + ) + if isinstance(col, ResolvedAttribute): + self._raise_if_hidden_api_attribute(operation, col) + return col, context + elif isinstance(operation, float): + return ( + ResolvedLiteral( + public_alias=f"equation|{equation}" if alias is None else alias, + search_type="number", + value=operation, + ), + [], + ) + # Fold equations that are only literals so Snuba does not evaluate them. + # Field equations such as `span.duration / 1000` still go through the RPC. + elif len(fields) == 0 and len(functions) == 0: + try: + value = arithmetic.resolve_arithmetic(operation) + except ZeroDivisionError: + raise arithmetic.ArithmeticValidationError("division by 0 is not allowed") + return ( + ResolvedLiteral( + public_alias=f"equation|{equation}" if alias is None else alias, + search_type="number", + value=value, + ), + [], + ) + + lhs, lhs_contexts = ( + self._resolve_operation(operation.lhs) if operation.lhs is not None else (None, []) + ) + rhs, rhs_contexts = ( + self._resolve_operation(operation.rhs) if operation.rhs is not None else (None, []) + ) + has_aggregates = False + for function in functions: + resolved_function, _ = self.resolve_function(function) + if resolved_function.is_aggregate: + has_aggregates = True + break + return ( + ResolvedEquation( + public_alias=f"equation|{equation}" if alias is None else alias, + # Type of equations can become complex very quickly. We could try to make sure all the columns have the + # same type and return that, but then ratios would have types eg. (p75/p50), as well managing multiple + # column types is strange too (p75+count). Keeping this as just `number` for now + search_type="number", + operator=constants.ARITHMETIC_OPERATOR_MAP[operation.operator], + lhs=lhs, + rhs=rhs, + is_aggregate=has_aggregates, + ), + lhs_contexts + rhs_contexts, + ) + + def _resolve_operation( + self, operation: arithmetic.OperandType + ) -> tuple[ + Column, + list[VirtualColumnDefinition | None], + ]: + """This function is to recursively step into the branches of the arithmetic to resolve branches to Columns for + the RPC, but we can't only use this since we want the resolver to return a ResolvedEquation so the resolver API + matches between resolved arithmetic and columns + """ + if isinstance(operation, arithmetic.Operation): + lhs, lhs_contexts = ( + self._resolve_operation(operation.lhs) if operation.lhs is not None else (None, []) + ) + rhs, rhs_contexts = ( + self._resolve_operation(operation.rhs) if operation.rhs is not None else (None, []) + ) + vcc = [] + if lhs_contexts: + vcc += lhs_contexts + if rhs_contexts: + vcc += rhs_contexts + return ( + Column( + formula=Column.BinaryFormula( + op=constants.ARITHMETIC_OPERATOR_MAP[operation.operator], + left=lhs, + right=rhs, + ) + ), + vcc, + ) + elif isinstance(operation, float): + return Column(literal=LiteralValue(val_double=operation)), [] + + # Resolve the column, and turn it into a RPC Column so it can be used in a BinaryFormula + # Columns in equations must pass default_value=0 otherwise they may become a null and ruin the entire formula + col, context = self.resolve_column(operation, default_value=0) + if isinstance(col, ResolvedAttribute): + self._raise_if_hidden_api_attribute(operation, col) + proto_definition = col.proto_definition + + if isinstance(proto_definition, AttributeKey): + return Column(key=proto_definition), context + + if isinstance(proto_definition, AttributeAggregation): + return Column(aggregation=proto_definition), context + + if isinstance(proto_definition, AttributeConditionalAggregation): + return Column(conditional_aggregation=proto_definition), context + + if isinstance(proto_definition, Column.BinaryFormula): + return Column(formula=proto_definition), context + + raise TypeError(f"Unsupported proto definition type: {type(proto_definition)}") + + def resolve_dataset_conditions( + self, + selected_columns: list[str] | None, + equations: list[str] | None, + ) -> TraceItemFilter | None: + extra_conditions = self.config.extra_conditions(self, selected_columns, equations) + + return and_trace_item_filters(extra_conditions) + + def remap_value_using_context_definition( + self, context_definition: VirtualColumnDefinition, value: str | int | list[str] | Any + ) -> str | int | list[str] | Any: + context = context_definition.constructor(self.params, self) + + # if the value passed is one of the potential values, then it's expected + # and we should pass it through as is + for val in context.value_map.values(): + if val == value: + return value + + # if the value passed is one of the potential keys, then it should before + # remapped to the value + if isinstance(value, str) and value in context.value_map: + value = context.value_map[value] + + # now that we've checked all potentially allowed values, we should fall back + # to using the default + if context.default_value: + return context.default_value + + return value + + def resolve_formula( + self, formula: ExploreSavedFormula, columns: str, alias: str + ) -> tuple[ResolvedColumn, list[VirtualColumnDefinition | None]]: + """Given a formula object, parse the arguments, then resolve it as an equation + + Like equations these aren't cached + """ + from sentry.search.eap.utils import resolve_and_parse_formula + + arguments = fields.parse_arguments(formula.name, columns) + + equation = resolve_and_parse_formula(formula, arguments, self) + return self.resolve_equation(equation, alias) diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml new file mode 100644 index 000000000..8c5d61662 --- /dev/null +++ b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml @@ -0,0 +1,96 @@ +# Dispatch a request to getsentry to run getsentry test suites +name: getsentry dispatcher + +on: + # XXX: We are using `pull_request_target` instead of `pull_request` because we want + # this to run on forks. It allows forks to access secrets safely by + # only running workflows from the main branch. Prefer to use `pull_request` when possible. + # + # See https://github.com/getsentry/sentry/pull/21600 for more details + pull_request_target: + types: [labeled, opened, reopened, synchronize] + +# disable all other special privileges +permissions: + # needed for `actions/checkout` to clone the code + contents: read + # needed to remove the pull-request label + pull-requests: write + +jobs: + dispatch: + if: "github.event.action != 'labeled' || github.event.label.name == 'Trigger: getsentry tests'" + name: getsentry dispatch + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + persist-credentials: false + + - name: permissions + run: | + python3 -uS .github/workflows/scripts/getsentry-dispatch-setup \ + --repo-id ${{ github.event.repository.id }} \ + --pr ${{ github.event.number }} \ + --event ${{ github.event.action }} \ + --username "$ARG_USERNAME" \ + --label-names "$ARG_LABEL_NAMES" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # these can contain special characters + ARG_USERNAME: ${{ github.event.pull_request.user.login }} + ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} + + - name: Check for file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Get changed files for selective testing + id: changed-files + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api repos/${{ github.repository }}/pulls/${{ github.event.number }}/files \ + --paginate | python3 .github/workflows/scripts/parse-pr-files.py >> "$GITHUB_OUTPUT" + + - name: getsentry token + uses: getsentry/action-github-app-token@5c1e90706fe007857338ac1bfbd7a4177db2f789 # v4.0.0 + id: getsentry + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - name: Wait for PR merge commit + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + id: mergecommit + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + const { waitForMergeCommit } = await import(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit.js`); + await waitForMergeCommit({ + github, + context, + core, + }); + + - name: Dispatch getsentry tests + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + SENTRY_CHANGED_FILES: ${{ steps.changed-files.outputs.files }} + SENTRY_PREVIOUS_FILENAMES: ${{ steps.changed-files.outputs.previous-filenames }} + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + const { dispatch } = await import(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/getsentry-dispatch.js`); + await dispatch({ + github, + context, + core, + mergeCommitSha: '${{ steps.mergecommit.outputs.mergeCommitSha }}', + fileChanges: ${{ toJson(steps.changes.outputs) }}, + sentryChangedFiles: process.env.SENTRY_CHANGED_FILES, + sentryPreviousFilenames: process.env.SENTRY_PREVIOUS_FILENAMES, + }); diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py new file mode 100644 index 000000000..a027c0c03 --- /dev/null +++ b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Parse the GitHub PR files API response into changed files and previous filenames. + +Reads JSON from stdin (the output of `gh api .../pulls/N/files --paginate`). +Note: --paginate can emit multiple JSON arrays (one per page), so we handle +concatenated arrays by decoding incrementally. + +Outputs two lines (suitable for appending to $GITHUB_OUTPUT): + files= + previous-filenames= + +Usage: + gh api repos/OWNER/REPO/pulls/N/files --paginate \ + | python3 parse-pr-files.py >> "$GITHUB_OUTPUT" +""" + +from __future__ import annotations + +import json +import sys + + +def main() -> None: + raw = sys.stdin.read() + decoder = json.JSONDecoder() + files = [] + idx = 0 + while idx < len(raw): + while idx < len(raw) and raw[idx].isspace(): + idx += 1 + if idx >= len(raw): + break + obj, end = decoder.raw_decode(raw, idx) + if isinstance(obj, list): + files.extend(obj) + idx = end + + changed = [f["filename"] for f in files] + previous = [ + f["previous_filename"] + for f in files + if f.get("status") == "renamed" and f.get("previous_filename") + ] + + print(f"files={' '.join(changed)}") + print(f"previous-filenames={' '.join(previous)}") + + +if __name__ == "__main__": + main() diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py new file mode 100644 index 000000000..92a29227b --- /dev/null +++ b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py @@ -0,0 +1,695 @@ +import dataclasses +import logging +import re +from collections import Counter, OrderedDict, defaultdict +from collections.abc import Sequence +from ipaddress import ip_address, ip_interface, ip_network +from typing import Any, Callable + +from sentry.utils import metrics + +logger = logging.getLogger("sentry.events.grouping") + + +# Counter for logging a set amount of example data. Not meant to be used directly. (Use the +# `_log_example_data` helper instead.) +LOGGING_COUNTER: Counter[str] = Counter() + +# Function parameterization regexes can specify to provide a customized replacement string. Can also +# be used to do conditional replacement, by returning the original value in cases where replacement +# shouldn't happen. +ParameterizationReplacementFunction = Callable[[str], str] + + +# Log examples, up to the given limit. +def _log_example_data( + key: str, # Key used for tracking log count and in logger `event` string + extra: dict[str, Any], # Extra data to add to the log (should include example data) + limit: int = 100, # Number of logs to be gathered per deployment +) -> None: + # Note: In a multi-threaded environment, it's possible to run into a race condition where + # multiple threads are simultaneously logging what should theoretically be the last example. As + # result, we may end up logging a few more examples than the given limit. To fix this, we'd need + # to wrap everything here in a lock, but given that a few extra logs don't hurt anything, it's + # not worth blocking ingest by doing so. + if LOGGING_COUNTER[key] < limit: + logger.info(f"grouping.parameterization.{key}", extra=extra) + LOGGING_COUNTER[key] += 1 + + +@dataclasses.dataclass +class ParameterizationRegex: + name: str # name of the pattern (also used as group name in combined regex) + raw_pattern: str # regex pattern w/o matching group name + raw_pattern_experimental: str | None = None + # Function which takes the matched value and returns the replacement value. + replacement_callback: ParameterizationReplacementFunction | None = None + + # These need to be used with `(?x)`, to tell the regex compiler to ignore comments + # and unescaped whitespace, so we can use newlines and indentation for better legibility. + + @property + def pattern(self) -> str: + return self._get_pattern(self.raw_pattern) + + @property + def experimental_pattern(self) -> str | None: + if not self.raw_pattern_experimental: + return None + return self._get_pattern(self.raw_pattern_experimental) + + def _get_pattern(self, raw_pattern: str) -> str: + """ + Returns the regex pattern inside of a named matching group. + """ + return rf"(?P<{self.name}>{raw_pattern})" + + +def is_valid_ip(maybe_ip_str: str) -> bool: + # Validate the string by attempting to pass it to the three built-in factory functions for + # creating different types of ip address objects. If any of them succeeds, it's a valid IP. If + # all three raise an error, it's not. + for fn, kwargs in ( + (ip_address, {}), + (ip_interface, {}), + (ip_network, {"strict": False}), # `strict: False` allows host bits + ): + try: + fn(maybe_ip_str, **kwargs) + except ValueError: + pass + else: + return True + + return False + + +# fmt: off +TLDS = {"com", "net", "org", "jp", "de", "uk", "fr", "br", "it", "ru", "es", "me", "gov", "pl", "ca", "au", "cn", "co", "in", "nl", "edu", "info", "eu", "ch", "id", "at", "kr", "cz", "mx", "be", "tv", "se", "tr", "tw", "al", "ua", "ir", "vn", "cl", "sk", "ly", "cc", "to", "no", "fi", "us", "pt", "dk", "ar", "hu", "tk", "gr", "il", "news", "ro", "my", "biz", "ie", "za", "nz", "sg", "ee", "th", "io", "xyz", "pe", "bg", "hk", "rs", "lt", "link", "ph", "club", "si", "site", "mobi", "by", "cat", "wiki", "la", "ga", "xxx", "cf", "hr", "ng", "jobs", "online", "kz", "ug", "gq", "ae", "is", "lv", "pro", "fm", "tips", "ms", "sa", "app"} +# fmt: on + + +def is_valid_hostname(maybe_hostname_str: str) -> bool: + # By spec, hostnames have a max length of 255 + if len(maybe_hostname_str) > 255: + return False + + # Almost anything is an allowable top-level domain since ICANN opened it up, but that means that + # paths like `some.path.to.a.module` and attribute access like `someobj.someproperty` and + # filenames like `somefile.txt` all would match as hostnames, which isn't what we want. So, at + # the risk of occasional false negatives, we check against the top 99 TLDs and if it doesn't + # match one of those, we don't count it as a hostname. (Fortunately there's not a ton of overlap + # between popular TLDs and common file extensions, and most of them aren't actual words, so + # they're also less likely to be used as module or property names.) + maybe_tld = maybe_hostname_str.split(".")[-1] + if maybe_tld.lower() not in TLDS: + return False + + return True + + +DEFAULT_PARAMETERIZATION_REGEXES = [ + ParameterizationRegex( + name="email", + raw_pattern=r"""[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]{1,254}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*""", + ), + ParameterizationRegex( + name="url", + raw_pattern=r""" + # Scheme - by spec, must start with a letter, but after that can contain anything + # alphanumeric in addition to plus, minus (has to be escaped so it's clear it's not part + # of a range), and dot + [a-zA-Z][a-zA-Z0-9+\-.]* + # The normal separator + :// + # First character of the domain (or path, if the domain is empty) - must be a valid URL + # character (so no quotes, backticks, backslashes, angle brackets, sqiggly brackets, + # pipes, carets, or whitespace) and must be allowable in the first spot (no starting the + # querystring right away, for example) + [^'"`\\<>{}|\^\s$.?#] + # The rest of the URL is technically optional + ( + [^'"`\\<>{}|\^\s]* # Any number of copies of anything not globally invalid + [^'"`\\<>{}|\^\s.,;] # Final character - must be both valid in general and allowable + # in the last spot (so no trailing punctuation) + )? + """, + ), + ParameterizationRegex( + name="hostname", + # This regex is intentionally loose, in that it doesn't encode the hostname spec's overall + # length restriction, and in that it catches things which are technically valid hostnames + # but which we don't want to parameterize that way (module paths, filenames with extensions, + # etc.). Max length is complicated to bake into a regex, and depending on how you do it, + # using the regex to narrow matches down to the most popular top-level domains is either + # slow or too aggressive. So here we use the regex to find hostname-like strings, and then + # validate them using a callback. + raw_pattern=r""" + # The overall pattern here expresses "2 to 128 dot-separated segments, each segment + # consisting of up to 63 letters/numbers/dashes, as long as no segment starts or ends + # with a dash." Individual parts labeled below. + \b + # All segments but the final one, each followed by a dot + ( + # Lookahead guaranteeing at least one letter + (?= [a-zA-Z0-9\-]* [a-zA-Z]) + # Segment body - either all letters/numbers (no dashes), or with interior-only dashes + ( + [a-zA-Z0-9]{1,63} + | + [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] + ) + \. + ){1,127} + # Final segment has no dot + ( + [a-zA-Z0-9]{1,63} + | + [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] + ) + \b + """, + # Validate that the matched string actually is a valid hostname before replacing it. If not, + # leave it alone. + replacement_callback=lambda orig_value: "" + if is_valid_hostname(orig_value) + else orig_value, + ), + ParameterizationRegex( + name="traceparent", + raw_pattern=r""" + # https://www.w3.org/TR/trace-context/#traceparent-header + (\b00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]\b) | + + # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html#request-tracing-syntax + (\b1-[0-9a-f]{8}-[0-9a-f]{24}\b) + """, + ), + ParameterizationRegex( + name="uuid", + raw_pattern=r"""\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b""", + ), + ParameterizationRegex(name="sha1", raw_pattern=r"""\b[0-9a-fA-F]{40}\b"""), + ParameterizationRegex(name="md5", raw_pattern=r"""\b[0-9a-fA-F]{32}\b"""), + ParameterizationRegex( + name="date", + raw_pattern=r""" + # No word boundaries required around dates. Should there be? + # RFC822, RFC1123, RFC1123Z + ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s\d{1,2}\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2,4}\s\d{1,2}:\d{1,2}(:\d{1,2})?\s([-\+][\d]{2}[0-5][\d]|(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z]))) + | + # Similar to RFC822, but "Mon Jan 02, 1999", "Jan 02, 1999" + (((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-3]\d,\s\d{2,4}) + | + # RFC850 + ((?:Sun|Sunday|Mon|Monday|Tue|Tuesday|Wed|Wednesday|Thu|Thursday|Fri|Friday|Sat|Saturday),\s\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}\s\d{2}:\d{2}:\d{2}\s(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z])) + | + # RFC3339, RFC3339Nano + (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?([+-]?\d{2}:\d{2})?) + | + # JavaScript + ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2}\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT[+-]\d{4}(?:\s\([^)]+\))?) + | + # Datetime with timezone offset (Z=UTC): + # (Note: These come before plain datetimes so the offset is seen as part of the time and + # not a separate int value.) + ( + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+([+-][0-2]\d:?[0-5]\d|Z))| # decimal seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z))| # seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z)) # no seconds + ) + | + # Datetime + ( + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+)| # decimal seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d)| # seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d) # no seconds + ) + | + # Kitchen, 12-hr + ( + (?) + (?.5) + (?!\.\d) + \b + ) + | + # IPv6-like strings + # + # Note: We can't use word boundaries here as we did with IPv4s because IPv6s contain + # non-word characters. Instead, we use a negative lookbehind and a negative lookahead, + # respectively, to specify the beginning and end of the pattern. These protect against + # two things: + # - Cases where the initial or end characters are all valid, but there are too many of + # them. (IOW, we don't want to match `2345:...` inside of an otherwise-invalid IP + # like `12345:...`, and the same applies to `...:1234` inside of `...:12345`. + # Similar logic applies to initial and final colons - we don't want `:::1` to turn + # into `:` because it's matching on just the second and third colons.) + # - Cases where `::` (which is a valid IPv6 address) appears inside of expressions + # like `SomeClass::someMethod()`, especially when the characters bordering the `::` + # are valid hex, like `Space::explore()`. + # This doesn't fix edge cases like `Fee::add()`, where it's all hex and also fewer than + # 5 characters on either side, but those are presumably pretty rare. + (?" if is_valid_ip(orig_value) else orig_value, + ), + ParameterizationRegex( + name="swift_txn_id", + raw_pattern=r""" + # OpenStack Swift transaction IDs: "tx" + 21 hex chars + "-" + 10 hex chars, + # optionally followed by a suffix (e.g. cluster ID or client-provided extra) + # https://docs.openstack.org/api-ref/object-store/ + (\btx[0-9a-f]{21}-[0-9a-f]{10}\S*) + """, + ), + ParameterizationRegex( + name="hex", + raw_pattern=r""" + # Hex value with `0x/0X` prefix (any length, with any mix of numbers and/or letters - + # the prefix pretty much guarantees it's hex). + (\b0[xX][0-9a-fA-F]+\b) | + + # Hex value without `0x/0X` prefix (at least 4 characters long, containing both a letter + # and number if shorter than 8 characters, and either all uppercase or all lowercase - + # we're more conservative here in order to reduce false positives). + ( + # For positive values, a negative lookbehind to create a word boundary but with + # underscores allowed (to permit things like `some_file_31d12.py`) + (? 96% of 7-digit + # hex values are mixed, so that's a tradeoff we're okay with.) Also, it only includes + # lowercase letters, since git shas are always expressed that way. + (\b(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7}\b) + """, + ), + # This pattern must come after the hex-based patterns so it doesn't catch strings which happen + # to contain only letters between A and F. + ParameterizationRegex( + name="random_id", + raw_pattern=r""" + \b + # Random nonsense alphanumeric id strings. To avoid false positives, we require the + # following: + # - A mix of uppercase letters, lowercase letters, and numbers + # - A minimum of 4 characters, with at least a certain level of "mixed-up-ness". For + # our purposes, that means the string must switch back and forth between letters and + # numbers at least 3 times. This rules out human-readable strings like `dogNumber1` + # (1 switch) and `bball4lyfe` (2 switches), while catching strings like `aKj8XLr2`. + # + # Lookahead guaranteeing at least one uppercase letter + (?= [a-z0-9]* [A-Z]) + # Lookahead guaranteeing at least one lowercase letter + (?= [A-Z0-9]* [a-z]) + # Lookahead enforcing letter/number switches. Two versions depending on whether the + # string starts with a letter or number. This also takes care of the "contains a number" + # requirement. + (?= + \d+ [a-zA-Z]+ \d+ [a-zA-Z]+ + | + [a-zA-Z]+ \d+ [a-zA-Z]+ \d+ + ) + # The pattern itself + [a-zA-Z0-9]{4,128} + \b + """, + ), + ParameterizationRegex( + name="float", + raw_pattern=r""" + ( + # For positive values, in addition to the wordbreak, a negative lookbehind to ensure + # this isn't part of a longer set of dot-delimited ints (IOW, to prevent `1.2.3` + # from matching as `1.`) + ( + \b + (?.3) + (?!\.\d) + \b + """, + ), + ParameterizationRegex( + name="int", + raw_pattern=r""" + ( + # For positive values, a negative lookbehind to create a word boundary but with + # underscores allowed (to permit things like `some_file_1121.py`) + (?=`-type key-value + # pairs, rather than all quoted strings + (?<=[=]) + ( + '([^']+)' | + "([^"]+)" + ) + """, + ), + ParameterizationRegex( + name="bool", + raw_pattern=r""" + # Lookbehind to ensure we'll only match the value half of `=`-type key-value + # pairs, rather than all instances of the words 'true' and 'false' + (?<=[=]) + ( + True | true | False | false + ) + """, + ), +] + + +class Parameterizer: + # Inputs longer than this are not parameterized, as a defense against ReDoS + MAX_INPUT_LENGTH = 8192 + + def __init__( + self, + # List of `ParameterizationRegex` objects defining the regexes to use. If nothing is passed, + # the default set will be used. + regexes: Sequence[ParameterizationRegex] = DEFAULT_PARAMETERIZATION_REGEXES, + *, + # List of `ParameterizationRegex.name` values, used to selectively enable pattern types. To + # use all available parameterization, omit this argument. + regex_keys: Sequence[str] | None = None, + # Whether to use experimental patterns, if available. (Pattern types without an experimental + # pattern will fall back to the standard pattern.) + use_experimental_regexes: bool = False, + ): + # Filter regexes by the specified keys, if given + if regex_keys: + regexes = [r for r in regexes if r.name in regex_keys] + + self.is_experimental = ( + use_experimental_regexes + # Only mark the parameterizer as experimental if there are actually any experiments + # running. If there aren't, then both parameterizers use the default regex patterns, so + # the "experimental" parameterizer isn't actually experimental. + and any(r.experimental_pattern is not None for r in regexes) + ) + + if self.is_experimental: + # This uses an `OrderedDict` to guarantee we apply the patterns in the given order. + patterns_by_name = OrderedDict( + ( + r.name, + r.experimental_pattern if r.experimental_pattern else r.pattern, + ) + for r in regexes + ) + else: + patterns_by_name = OrderedDict((r.name, r.pattern) for r in regexes) + + # Combine the individual patterns into one giant regex to check against. (This is faster + # than checking each pattern individually because it entails less overhead.) + self.combined_regex = re.compile( + # The `(?x)` tells the regex compiler to ignore comments and unescaped whitespace, so we + # can use newlines and indentation for better legibility when defining regexes + rf"(?x){'|'.join(patterns_by_name.values())}" + ) + + # Collect replacement callbacks, if any + self.replacement_functions = { + r.name: r.replacement_callback for r in regexes if r.replacement_callback + } + + # Store the individual regexes in case we need to handle replacement function false + # positives by manually looping through the patterns. This uses an `OrderedDict` to + # guarantee we apply them in the same order as in the main/combined regex. + self.compiled_regexes_by_name = OrderedDict( + (name, re.compile(rf"(?x){pattern}")) for name, pattern in patterns_by_name.items() + ) + + def parameterize(self, input_str: str) -> str: + """ + Replace all regex matches in the input string with placeholder strings, using the regexes + with which the parameterizer was initialized. + + For example, turn "Error with order #1231" into "Error with order #". + """ + + if len(input_str) > self.MAX_INPUT_LENGTH: + metrics.incr("grouping.parameterization_skipped_long_input") + return input_str + + replacement_counts: defaultdict[str, int] = defaultdict(int) + # Track whether any regex matches don't lead to a replacement + found_false_positive = False + # Flag allowing us to only count false positives during the main parameterization, not the + # fallback run + emit_false_positive_metric = True + + def _handle_regex_match(match: re.Match[str]) -> str: + # Ensure we're dealing with the flag from the outer scope, rather than shadowing it + nonlocal found_false_positive + + # This handler gets called on two types of regexes: + # - The main/combination regex, which consists of a bunch of named capturing groups + # (with no nested named groups) separated by `|`. + # - Individual regexes for each pattern type, each consisting of one of the + # alternatives in the main regex. + # In the former case, only one alternative can have been matched, so its group name will + # be the only (and therefore last) matched group name. In the latter case, there is only + # one named group to match, so its name will automatically be the only/last matched + # group name. Thus `lastgroup` should give us the group name in either case. + matched_key = match.lastgroup + orig_value = match.groupdict().get( + matched_key or "" # Empty string for mypy appeasment + ) + + if not matched_key or not orig_value: # Insurance - shouldn't happen IRL + return "" + + replacement_callback = self.replacement_functions.get(matched_key) + replacement_string = ( + replacement_callback(orig_value) if replacement_callback else f"<{matched_key}>" + ) + + # The replacement callback might return the original value, if it determines it's not + # something which should be replaced, and we don't want to count that + if replacement_string != orig_value: + replacement_counts[matched_key] += 1 + else: + found_false_positive = True + + # This is only true during the main combo-regex parameterization, not during + # fallback, so that we don't double-count these occurrences + if emit_false_positive_metric: + # Track the number of false positive matches, and what pattern produced them. We + # can compare this to the same key's `grouping.value_parameterized` metric below + # to see how often our maybe-matches pan out to be actual matches. + metrics.incr( + "grouping.parameterization_false_positive", tags={"key": matched_key} + ) + # TODO: Remove this once we have enough sample data + _log_example_data( + "ip_false_positive", extra={"input_str": input_str, "value": orig_value} + ) + + return replacement_string + + with metrics.timer( + "grouping.parameterize", tags={"experimental": self.is_experimental} + ) as metric_tags: + parameterized = self.combined_regex.sub(_handle_regex_match, input_str) + + # Our big combo regex will short-circuit when it finds a match, which is great for + # performance but problematic in cases where a replacement callback declines to + # parameterize a value, because then the value never gets checked against later + # patterns. To protect against that, in those cases we cycle through all patterns + # individually, which is slower but ensures we check them all. + if found_false_positive: + metric_tags["false_positive"] = True + + # Reset values before applying the patterns again + replacement_counts = defaultdict(int) + parameterized = input_str + + # Prevent double-counting of false positives + emit_false_positive_metric = False + + # Apply patterns one by one, with no short-circuiting + for regex_key, regex in self.compiled_regexes_by_name.items(): + parameterized = regex.sub(_handle_regex_match, parameterized) + + metric_tags["changed"] = parameterized != input_str + + for regex_key, count in replacement_counts.items(): + # Track the kinds of replacements being made + metrics.incr("grouping.value_parameterized", amount=count, tags={"key": regex_key}) + + return parameterized + + +parameterizer = Parameterizer(use_experimental_regexes=False) +experimental_parameterizer = Parameterizer(use_experimental_regexes=True) diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py new file mode 100644 index 000000000..48139acaf --- /dev/null +++ b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py @@ -0,0 +1,696 @@ +import dataclasses +import logging +import re +from collections import Counter, OrderedDict, defaultdict +from collections.abc import Sequence +from ipaddress import ip_address, ip_interface, ip_network +from typing import Any, Callable + +from sentry.utils import metrics + +logger = logging.getLogger("sentry.events.grouping") + + +# Counter for logging a set amount of example data. Not meant to be used directly. (Use the +# `_log_example_data` helper instead.) +LOGGING_COUNTER: Counter[str] = Counter() + +# Function parameterization regexes can specify to provide a customized replacement string. Can also +# be used to do conditional replacement, by returning the original value in cases where replacement +# shouldn't happen. +ParameterizationReplacementFunction = Callable[[str], str] + + +# Log examples, up to the given limit. +def _log_example_data( + key: str, # Key used for tracking log count and in logger `event` string + extra: dict[str, Any], # Extra data to add to the log (should include example data) + limit: int = 100, # Number of logs to be gathered per deployment +) -> None: + # Note: In a multi-threaded environment, it's possible to run into a race condition where + # multiple threads are simultaneously logging what should theoretically be the last example. As + # result, we may end up logging a few more examples than the given limit. To fix this, we'd need + # to wrap everything here in a lock, but given that a few extra logs don't hurt anything, it's + # not worth blocking ingest by doing so. + if LOGGING_COUNTER[key] < limit: + logger.info(f"grouping.parameterization.{key}", extra=extra) + LOGGING_COUNTER[key] += 1 + + +@dataclasses.dataclass +class ParameterizationRegex: + name: str # name of the pattern (also used as group name in combined regex) + raw_pattern: str # regex pattern w/o matching group name + raw_pattern_experimental: str | None = None + # Function which takes the matched value and returns the replacement value. + replacement_callback: ParameterizationReplacementFunction | None = None + + # These need to be used with `(?x)`, to tell the regex compiler to ignore comments + # and unescaped whitespace, so we can use newlines and indentation for better legibility. + + @property + def pattern(self) -> str: + return self._get_pattern(self.raw_pattern) + + @property + def experimental_pattern(self) -> str | None: + if not self.raw_pattern_experimental: + return None + return self._get_pattern(self.raw_pattern_experimental) + + def _get_pattern(self, raw_pattern: str) -> str: + """ + Returns the regex pattern inside of a named matching group. + """ + return rf"(?P<{self.name}>{raw_pattern})" + + +def is_valid_ip(maybe_ip_str: str) -> bool: + # Validate the string by attempting to pass it to the three built-in factory functions for + # creating different types of ip address objects. If any of them succeeds, it's a valid IP. If + # all three raise an error, it's not. + for fn, kwargs in ( + (ip_address, {}), + (ip_interface, {}), + (ip_network, {"strict": False}), # `strict: False` allows host bits + ): + try: + fn(maybe_ip_str, **kwargs) + except ValueError: + pass + else: + return True + + return False + + +# fmt: off +TLDS = {"com", "net", "org", "jp", "de", "uk", "fr", "br", "it", "ru", "es", "me", "gov", "pl", "ca", "au", "cn", "co", "in", "nl", "edu", "info", "eu", "ch", "id", "at", "kr", "cz", "mx", "be", "tv", "se", "tr", "tw", "al", "ua", "ir", "vn", "cl", "sk", "ly", "cc", "to", "no", "fi", "us", "pt", "dk", "ar", "hu", "tk", "gr", "il", "news", "ro", "my", "biz", "ie", "za", "nz", "sg", "ee", "th", "io", "xyz", "pe", "bg", "hk", "rs", "lt", "link", "ph", "club", "si", "site", "mobi", "by", "cat", "wiki", "la", "ga", "xxx", "cf", "hr", "ng", "jobs", "online", "kz", "ug", "gq", "ae", "is", "lv", "pro", "fm", "tips", "ms", "sa", "app"} +# fmt: on + + +def is_valid_hostname(maybe_hostname_str: str) -> bool: + # By spec, hostnames have a max length of 255 + if len(maybe_hostname_str) > 255: + return False + + # Almost anything is an allowable top-level domain since ICANN opened it up, but that means that + # paths like `some.path.to.a.module` and attribute access like `someobj.someproperty` and + # filenames like `somefile.txt` all would match as hostnames, which isn't what we want. So, at + # the risk of occasional false negatives, we check against the top 99 TLDs and if it doesn't + # match one of those, we don't count it as a hostname. (Fortunately there's not a ton of overlap + # between popular TLDs and common file extensions, and most of them aren't actual words, so + # they're also less likely to be used as module or property names.) + maybe_tld = maybe_hostname_str.split(".")[-1] + if maybe_tld.lower() not in TLDS: + return False + + return True + + +DEFAULT_PARAMETERIZATION_REGEXES = [ + ParameterizationRegex( + name="email", + raw_pattern=r"""[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]{1,254}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*""", + ), + ParameterizationRegex( + name="url", + raw_pattern=r""" + # Scheme - by spec, must start with a letter, but after that can contain anything + # alphanumeric in addition to plus, minus (has to be escaped so it's clear it's not part + # of a range), and dot. The length is bounded (real schemes are short) to prevent + # quadratic backtracking on long runs of scheme-valid characters that lack a `://`. + [a-zA-Z][a-zA-Z0-9+\-.]{0,32} + # The normal separator + :// + # First character of the domain (or path, if the domain is empty) - must be a valid URL + # character (so no quotes, backticks, backslashes, angle brackets, sqiggly brackets, + # pipes, carets, or whitespace) and must be allowable in the first spot (no starting the + # querystring right away, for example) + [^'"`\\<>{}|\^\s$.?#] + # The rest of the URL is technically optional + ( + [^'"`\\<>{}|\^\s]* # Any number of copies of anything not globally invalid + [^'"`\\<>{}|\^\s.,;] # Final character - must be both valid in general and allowable + # in the last spot (so no trailing punctuation) + )? + """, + ), + ParameterizationRegex( + name="hostname", + # This regex is intentionally loose, in that it doesn't encode the hostname spec's overall + # length restriction, and in that it catches things which are technically valid hostnames + # but which we don't want to parameterize that way (module paths, filenames with extensions, + # etc.). Max length is complicated to bake into a regex, and depending on how you do it, + # using the regex to narrow matches down to the most popular top-level domains is either + # slow or too aggressive. So here we use the regex to find hostname-like strings, and then + # validate them using a callback. + raw_pattern=r""" + # The overall pattern here expresses "2 to 128 dot-separated segments, each segment + # consisting of up to 63 letters/numbers/dashes, as long as no segment starts or ends + # with a dash." Individual parts labeled below. + \b + # All segments but the final one, each followed by a dot + ( + # Lookahead guaranteeing at least one letter + (?= [a-zA-Z0-9\-]* [a-zA-Z]) + # Segment body - either all letters/numbers (no dashes), or with interior-only dashes + ( + [a-zA-Z0-9]{1,63} + | + [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] + ) + \. + ){1,127} + # Final segment has no dot + ( + [a-zA-Z0-9]{1,63} + | + [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] + ) + \b + """, + # Validate that the matched string actually is a valid hostname before replacing it. If not, + # leave it alone. + replacement_callback=lambda orig_value: "" + if is_valid_hostname(orig_value) + else orig_value, + ), + ParameterizationRegex( + name="traceparent", + raw_pattern=r""" + # https://www.w3.org/TR/trace-context/#traceparent-header + (\b00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]\b) | + + # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html#request-tracing-syntax + (\b1-[0-9a-f]{8}-[0-9a-f]{24}\b) + """, + ), + ParameterizationRegex( + name="uuid", + raw_pattern=r"""\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b""", + ), + ParameterizationRegex(name="sha1", raw_pattern=r"""\b[0-9a-fA-F]{40}\b"""), + ParameterizationRegex(name="md5", raw_pattern=r"""\b[0-9a-fA-F]{32}\b"""), + ParameterizationRegex( + name="date", + raw_pattern=r""" + # No word boundaries required around dates. Should there be? + # RFC822, RFC1123, RFC1123Z + ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s\d{1,2}\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2,4}\s\d{1,2}:\d{1,2}(:\d{1,2})?\s([-\+][\d]{2}[0-5][\d]|(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z]))) + | + # Similar to RFC822, but "Mon Jan 02, 1999", "Jan 02, 1999" + (((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-3]\d,\s\d{2,4}) + | + # RFC850 + ((?:Sun|Sunday|Mon|Monday|Tue|Tuesday|Wed|Wednesday|Thu|Thursday|Fri|Friday|Sat|Saturday),\s\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}\s\d{2}:\d{2}:\d{2}\s(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z])) + | + # RFC3339, RFC3339Nano + (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?([+-]?\d{2}:\d{2})?) + | + # JavaScript + ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2}\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT[+-]\d{4}(?:\s\([^)]+\))?) + | + # Datetime with timezone offset (Z=UTC): + # (Note: These come before plain datetimes so the offset is seen as part of the time and + # not a separate int value.) + ( + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+([+-][0-2]\d:?[0-5]\d|Z))| # decimal seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z))| # seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z)) # no seconds + ) + | + # Datetime + ( + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+)| # decimal seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d)| # seconds + (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d) # no seconds + ) + | + # Kitchen, 12-hr + ( + (?) + (?.5) + (?!\.\d) + \b + ) + | + # IPv6-like strings + # + # Note: We can't use word boundaries here as we did with IPv4s because IPv6s contain + # non-word characters. Instead, we use a negative lookbehind and a negative lookahead, + # respectively, to specify the beginning and end of the pattern. These protect against + # two things: + # - Cases where the initial or end characters are all valid, but there are too many of + # them. (IOW, we don't want to match `2345:...` inside of an otherwise-invalid IP + # like `12345:...`, and the same applies to `...:1234` inside of `...:12345`. + # Similar logic applies to initial and final colons - we don't want `:::1` to turn + # into `:` because it's matching on just the second and third colons.) + # - Cases where `::` (which is a valid IPv6 address) appears inside of expressions + # like `SomeClass::someMethod()`, especially when the characters bordering the `::` + # are valid hex, like `Space::explore()`. + # This doesn't fix edge cases like `Fee::add()`, where it's all hex and also fewer than + # 5 characters on either side, but those are presumably pretty rare. + (?" if is_valid_ip(orig_value) else orig_value, + ), + ParameterizationRegex( + name="swift_txn_id", + raw_pattern=r""" + # OpenStack Swift transaction IDs: "tx" + 21 hex chars + "-" + 10 hex chars, + # optionally followed by a suffix (e.g. cluster ID or client-provided extra) + # https://docs.openstack.org/api-ref/object-store/ + (\btx[0-9a-f]{21}-[0-9a-f]{10}\S*) + """, + ), + ParameterizationRegex( + name="hex", + raw_pattern=r""" + # Hex value with `0x/0X` prefix (any length, with any mix of numbers and/or letters - + # the prefix pretty much guarantees it's hex). + (\b0[xX][0-9a-fA-F]+\b) | + + # Hex value without `0x/0X` prefix (at least 4 characters long, containing both a letter + # and number if shorter than 8 characters, and either all uppercase or all lowercase - + # we're more conservative here in order to reduce false positives). + ( + # For positive values, a negative lookbehind to create a word boundary but with + # underscores allowed (to permit things like `some_file_31d12.py`) + (? 96% of 7-digit + # hex values are mixed, so that's a tradeoff we're okay with.) Also, it only includes + # lowercase letters, since git shas are always expressed that way. + (\b(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7}\b) + """, + ), + # This pattern must come after the hex-based patterns so it doesn't catch strings which happen + # to contain only letters between A and F. + ParameterizationRegex( + name="random_id", + raw_pattern=r""" + \b + # Random nonsense alphanumeric id strings. To avoid false positives, we require the + # following: + # - A mix of uppercase letters, lowercase letters, and numbers + # - A minimum of 4 characters, with at least a certain level of "mixed-up-ness". For + # our purposes, that means the string must switch back and forth between letters and + # numbers at least 3 times. This rules out human-readable strings like `dogNumber1` + # (1 switch) and `bball4lyfe` (2 switches), while catching strings like `aKj8XLr2`. + # + # Lookahead guaranteeing at least one uppercase letter + (?= [a-z0-9]* [A-Z]) + # Lookahead guaranteeing at least one lowercase letter + (?= [A-Z0-9]* [a-z]) + # Lookahead enforcing letter/number switches. Two versions depending on whether the + # string starts with a letter or number. This also takes care of the "contains a number" + # requirement. + (?= + \d+ [a-zA-Z]+ \d+ [a-zA-Z]+ + | + [a-zA-Z]+ \d+ [a-zA-Z]+ \d+ + ) + # The pattern itself + [a-zA-Z0-9]{4,128} + \b + """, + ), + ParameterizationRegex( + name="float", + raw_pattern=r""" + ( + # For positive values, in addition to the wordbreak, a negative lookbehind to ensure + # this isn't part of a longer set of dot-delimited ints (IOW, to prevent `1.2.3` + # from matching as `1.`) + ( + \b + (?.3) + (?!\.\d) + \b + """, + ), + ParameterizationRegex( + name="int", + raw_pattern=r""" + ( + # For positive values, a negative lookbehind to create a word boundary but with + # underscores allowed (to permit things like `some_file_1121.py`) + (?=`-type key-value + # pairs, rather than all quoted strings + (?<=[=]) + ( + '([^']+)' | + "([^"]+)" + ) + """, + ), + ParameterizationRegex( + name="bool", + raw_pattern=r""" + # Lookbehind to ensure we'll only match the value half of `=`-type key-value + # pairs, rather than all instances of the words 'true' and 'false' + (?<=[=]) + ( + True | true | False | false + ) + """, + ), +] + + +class Parameterizer: + # Inputs longer than this are not parameterized, as a defense against ReDoS + MAX_INPUT_LENGTH = 8192 + + def __init__( + self, + # List of `ParameterizationRegex` objects defining the regexes to use. If nothing is passed, + # the default set will be used. + regexes: Sequence[ParameterizationRegex] = DEFAULT_PARAMETERIZATION_REGEXES, + *, + # List of `ParameterizationRegex.name` values, used to selectively enable pattern types. To + # use all available parameterization, omit this argument. + regex_keys: Sequence[str] | None = None, + # Whether to use experimental patterns, if available. (Pattern types without an experimental + # pattern will fall back to the standard pattern.) + use_experimental_regexes: bool = False, + ): + # Filter regexes by the specified keys, if given + if regex_keys: + regexes = [r for r in regexes if r.name in regex_keys] + + self.is_experimental = ( + use_experimental_regexes + # Only mark the parameterizer as experimental if there are actually any experiments + # running. If there aren't, then both parameterizers use the default regex patterns, so + # the "experimental" parameterizer isn't actually experimental. + and any(r.experimental_pattern is not None for r in regexes) + ) + + if self.is_experimental: + # This uses an `OrderedDict` to guarantee we apply the patterns in the given order. + patterns_by_name = OrderedDict( + ( + r.name, + r.experimental_pattern if r.experimental_pattern else r.pattern, + ) + for r in regexes + ) + else: + patterns_by_name = OrderedDict((r.name, r.pattern) for r in regexes) + + # Combine the individual patterns into one giant regex to check against. (This is faster + # than checking each pattern individually because it entails less overhead.) + self.combined_regex = re.compile( + # The `(?x)` tells the regex compiler to ignore comments and unescaped whitespace, so we + # can use newlines and indentation for better legibility when defining regexes + rf"(?x){'|'.join(patterns_by_name.values())}" + ) + + # Collect replacement callbacks, if any + self.replacement_functions = { + r.name: r.replacement_callback for r in regexes if r.replacement_callback + } + + # Store the individual regexes in case we need to handle replacement function false + # positives by manually looping through the patterns. This uses an `OrderedDict` to + # guarantee we apply them in the same order as in the main/combined regex. + self.compiled_regexes_by_name = OrderedDict( + (name, re.compile(rf"(?x){pattern}")) for name, pattern in patterns_by_name.items() + ) + + def parameterize(self, input_str: str) -> str: + """ + Replace all regex matches in the input string with placeholder strings, using the regexes + with which the parameterizer was initialized. + + For example, turn "Error with order #1231" into "Error with order #". + """ + + if len(input_str) > self.MAX_INPUT_LENGTH: + metrics.incr("grouping.parameterization_skipped_long_input") + return input_str + + replacement_counts: defaultdict[str, int] = defaultdict(int) + # Track whether any regex matches don't lead to a replacement + found_false_positive = False + # Flag allowing us to only count false positives during the main parameterization, not the + # fallback run + emit_false_positive_metric = True + + def _handle_regex_match(match: re.Match[str]) -> str: + # Ensure we're dealing with the flag from the outer scope, rather than shadowing it + nonlocal found_false_positive + + # This handler gets called on two types of regexes: + # - The main/combination regex, which consists of a bunch of named capturing groups + # (with no nested named groups) separated by `|`. + # - Individual regexes for each pattern type, each consisting of one of the + # alternatives in the main regex. + # In the former case, only one alternative can have been matched, so its group name will + # be the only (and therefore last) matched group name. In the latter case, there is only + # one named group to match, so its name will automatically be the only/last matched + # group name. Thus `lastgroup` should give us the group name in either case. + matched_key = match.lastgroup + orig_value = match.groupdict().get( + matched_key or "" # Empty string for mypy appeasment + ) + + if not matched_key or not orig_value: # Insurance - shouldn't happen IRL + return "" + + replacement_callback = self.replacement_functions.get(matched_key) + replacement_string = ( + replacement_callback(orig_value) if replacement_callback else f"<{matched_key}>" + ) + + # The replacement callback might return the original value, if it determines it's not + # something which should be replaced, and we don't want to count that + if replacement_string != orig_value: + replacement_counts[matched_key] += 1 + else: + found_false_positive = True + + # This is only true during the main combo-regex parameterization, not during + # fallback, so that we don't double-count these occurrences + if emit_false_positive_metric: + # Track the number of false positive matches, and what pattern produced them. We + # can compare this to the same key's `grouping.value_parameterized` metric below + # to see how often our maybe-matches pan out to be actual matches. + metrics.incr( + "grouping.parameterization_false_positive", tags={"key": matched_key} + ) + # TODO: Remove this once we have enough sample data + _log_example_data( + "ip_false_positive", extra={"input_str": input_str, "value": orig_value} + ) + + return replacement_string + + with metrics.timer( + "grouping.parameterize", tags={"experimental": self.is_experimental} + ) as metric_tags: + parameterized = self.combined_regex.sub(_handle_regex_match, input_str) + + # Our big combo regex will short-circuit when it finds a match, which is great for + # performance but problematic in cases where a replacement callback declines to + # parameterize a value, because then the value never gets checked against later + # patterns. To protect against that, in those cases we cycle through all patterns + # individually, which is slower but ensures we check them all. + if found_false_positive: + metric_tags["false_positive"] = True + + # Reset values before applying the patterns again + replacement_counts = defaultdict(int) + parameterized = input_str + + # Prevent double-counting of false positives + emit_false_positive_metric = False + + # Apply patterns one by one, with no short-circuiting + for regex_key, regex in self.compiled_regexes_by_name.items(): + parameterized = regex.sub(_handle_regex_match, parameterized) + + metric_tags["changed"] = parameterized != input_str + + for regex_key, count in replacement_counts.items(): + # Track the kinds of replacements being made + metrics.incr("grouping.value_parameterized", amount=count, tags={"key": regex_key}) + + return parameterized + + +parameterizer = Parameterizer(use_experimental_regexes=False) +experimental_parameterizer = Parameterizer(use_experimental_regexes=True) diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py new file mode 100644 index 000000000..76ab87d9c --- /dev/null +++ b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py @@ -0,0 +1,71 @@ +from django.conf import settings +from rest_framework.request import Request +from rest_framework.response import Response + +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import Endpoint, all_silo_endpoint +from sentry.api.permissions import SuperuserPermission +from sentry.conf.server import SENTRY_EARLY_FEATURES +from sentry.runner.settings import configure, discover_configs + + +@all_silo_endpoint +class InternalFeatureFlagsEndpoint(Endpoint): + permission_classes = (SuperuserPermission,) + owner = ApiOwner.HYBRID_CLOUD + publish_status = { + "GET": ApiPublishStatus.PRIVATE, + "PUT": ApiPublishStatus.PRIVATE, + } + + def get(self, request: Request) -> Response: + if not settings.SENTRY_SELF_HOSTED: + return Response("You are not self-hosting Sentry.", status=403) + + result = {} + for key in SENTRY_EARLY_FEATURES: + result[key] = { + "value": settings.SENTRY_FEATURES.get(key, False), + "description": SENTRY_EARLY_FEATURES[key], + } + + return Response(result) + + def put(self, request: Request) -> Response: + if not settings.SENTRY_SELF_HOSTED: + return Response("You are not self-hosting Sentry.", status=403) + + data = request.data.keys() + valid_feature_flags = [flag for flag in data if SENTRY_EARLY_FEATURES.get(flag, False)] + _, py, yml = discover_configs() + # Open the file for reading and writing + with open(py, "r+") as file: + lines = file.readlines() + # print(lines) + for valid_flag in valid_feature_flags: + match_found = False + new_string = ( + f'\nSENTRY_FEATURES["{valid_flag}"]={request.data.get(valid_flag, False)}\n' + ) + # Search for the string match and update lines + for i, line in enumerate(lines): + if valid_flag in line: + match_found = True + lines[i] = new_string + + break + + # If no match found, append a new line + if not match_found: + lines.append(new_string) + + # Move the file pointer to the beginning and truncate the file + file.seek(0) + file.truncate() + + # Write modified lines back to the file + file.writelines(lines) + configure(None, py, yml) + + return Response(status=200) diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py new file mode 100644 index 000000000..47e09f7dd --- /dev/null +++ b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py @@ -0,0 +1,117 @@ +from collections.abc import Mapping + +from django.conf import settings +from rest_framework.request import Request +from rest_framework.response import Response + +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import Endpoint, all_silo_endpoint +from sentry.api.permissions import SuperuserPermission +from sentry.conf.server import SENTRY_EARLY_FEATURES +from sentry.runner.settings import configure, discover_configs + + +def _coerce_early_feature_flag_value(value: object) -> bool | None: + """ + Map API input to a bool for SENTRY_FEATURES. Accepts JSON booleans, 0/1 + (common in scripts), and the strings "true"/"false" (case-insensitive). + Returns None if the value cannot be interpreted safely. + """ + if isinstance(value, bool): + return value + if isinstance(value, int): + if value == 1: + return True + if value == 0: + return False + return None + if isinstance(value, str): + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + return None + return None + + +@all_silo_endpoint +class InternalFeatureFlagsEndpoint(Endpoint): + permission_classes = (SuperuserPermission,) + owner = ApiOwner.HYBRID_CLOUD + publish_status = { + "GET": ApiPublishStatus.PRIVATE, + "PUT": ApiPublishStatus.PRIVATE, + } + + def get(self, request: Request) -> Response: + if not settings.SENTRY_SELF_HOSTED: + return Response("You are not self-hosting Sentry.", status=403) + + result = {} + for key in SENTRY_EARLY_FEATURES: + result[key] = { + "value": settings.SENTRY_FEATURES.get(key, False), + "description": SENTRY_EARLY_FEATURES[key], + } + + return Response(result) + + def put(self, request: Request) -> Response: + if not settings.SENTRY_SELF_HOSTED: + return Response("You are not self-hosting Sentry.", status=403) + + payload: object = request.data + if not isinstance(payload, Mapping): + return Response( + {"detail": "Feature flag updates must be a JSON object."}, + status=400, + ) + + valid_feature_flags = [flag for flag in payload if flag in SENTRY_EARLY_FEATURES] + coerced_values: dict[str, bool] = {} + for valid_flag in valid_feature_flags: + coerced = _coerce_early_feature_flag_value(payload.get(valid_flag)) + if coerced is None: + return Response( + { + "detail": ( + f'Feature flag "{valid_flag}" must be true or false ' + f"(boolean, 0 or 1, or the string true or false)." + ) + }, + status=400, + ) + coerced_values[valid_flag] = coerced + + _, py, yml = discover_configs() + # Open the file for reading and writing + with open(py, "r+") as file: + lines = file.readlines() + # print(lines) + for valid_flag in valid_feature_flags: + match_found = False + python_bool = "True" if coerced_values[valid_flag] else "False" + new_string = f'\nSENTRY_FEATURES["{valid_flag}"]={python_bool}\n' + # Search for the string match and update lines + for i, line in enumerate(lines): + if valid_flag in line: + match_found = True + lines[i] = new_string + + break + + # If no match found, append a new line + if not match_found: + lines.append(new_string) + + # Move the file pointer to the beginning and truncate the file + file.seek(0) + file.truncate() + + # Write modified lines back to the file + file.writelines(lines) + configure(None, py, yml) + + return Response(status=200) diff --git a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py new file mode 100644 index 000000000..bddc4209e --- /dev/null +++ b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py @@ -0,0 +1,142 @@ +import logging + +from django.conf import settings +from django.db import router, transaction +from rest_framework.request import Request +from rest_framework.response import Response + +import sentry +from sentry import options +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import Endpoint, all_silo_endpoint +from sentry.api.permissions import SuperuserPermission +from sentry.options.store import Key +from sentry.utils.email import is_smtp_enabled + +logger = logging.getLogger("sentry") + +SYSTEM_OPTIONS_ALLOWLIST = frozenset( + [ + # Used during setup before the superadmin role with the options.admin permission is authed + "system.admin-email" + ] +) + + +def _is_secret(k: Key) -> bool: + keywords = ["secret", "private", "token"] + return ((k.flags & options.FLAG_CREDENTIAL) != 0) or any( + [keyword in k.name for keyword in keywords] + ) + + +@all_silo_endpoint +class SystemOptionsEndpoint(Endpoint): + publish_status = { + "GET": ApiPublishStatus.PRIVATE, + "PUT": ApiPublishStatus.PRIVATE, + } + owner = ApiOwner.HYBRID_CLOUD + permission_classes = (SuperuserPermission,) + + def get(self, request: Request) -> Response: + query = request.GET.get("query") + if query == "is:required": + option_list = options.filter(flag=options.FLAG_REQUIRED) + elif query: + return Response(f"{query} is not a supported search query", status=400) + else: + option_list = options.all() + + smtp_disabled = not is_smtp_enabled() + + results = {} + for k in option_list: + disabled, disabled_reason = False, None + + if smtp_disabled and k.name[:5] == "mail.": + disabled_reason, disabled = "smtpDisabled", True + elif bool( + k.flags & options.FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(k.name) + ): + # TODO(mattrobenolt): Expose this as a property on Key. + disabled_reason, disabled = "diskPriority", True + + # TODO(mattrobenolt): help, placeholder, title, type + results[k.name] = { + "value": options.get(k.name) if not _is_secret(k) else "[redacted]", + "field": { + "default": k.default(), + "required": bool(k.flags & options.FLAG_REQUIRED), + "disabled": disabled, + "disabledReason": disabled_reason, + "isSet": options.isset(k.name), + "allowEmpty": bool(k.flags & options.FLAG_ALLOW_EMPTY), + }, + } + + return Response(results) + + def has_permission(self, request: Request) -> bool: + if settings.SENTRY_SELF_HOSTED and request.user.is_superuser: + return True + if not request.access.has_permission("options.admin"): + # We ignore options.admin permission if all keys in the update match the allowlist. + return bool(request.data) and all( + [k in SYSTEM_OPTIONS_ALLOWLIST for k in request.data.keys()] + ) + + return True + + def put(self, request: Request) -> Response: + if not self.has_permission(request): + return Response(status=403) + + # TODO(dcramer): this should validate options before saving them + for k, v in request.data.items(): + if v and isinstance(v, str): + v = v.strip() + try: + option = options.lookup_key(k) + except options.UnknownOption: + # TODO(dcramer): unify API errors + return Response( + {"error": "unknown_option", "errorDetail": {"option": k}}, status=400 + ) + + try: + with transaction.atomic(router.db_for_write(options.default_store.model)): + if not (option.flags & options.FLAG_ALLOW_EMPTY) and not v: + options.delete(k) + else: + options.set(k, v, channel=options.UpdateChannel.APPLICATION) + + logger.info( + "options.update", + extra={ + "ip_address": request.META["REMOTE_ADDR"], + "user_id": request.user.id, + "option_key": k, + "option_value": "[redacted]" if _is_secret(option) else v, + }, + ) + except (TypeError, AssertionError) as e: + # TODO(chadwhitacre): Use a custom exception for the + # immutability case, especially since asserts disappear with + # `python -O`. + return Response( + { + "error": "invalid_type" if type(e) is TypeError else "immutable_option", + "errorDetail": {"option": k, "message": str(e)}, + }, + status=400, + ) + # TODO(dcramer): this has nothing to do with configuring options and + # should not be set here + options.set( + "sentry:version-configured", + sentry.get_version(), + channel=options.UpdateChannel.APPLICATION, + ) + return Response(status=200) diff --git a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py new file mode 100644 index 000000000..af2fd93ec --- /dev/null +++ b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py @@ -0,0 +1,141 @@ +import logging +from typing import Any + +from django.conf import settings +from django.db import router, transaction +from rest_framework.request import Request +from rest_framework.response import Response + +import sentry +from sentry import options +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import Endpoint, all_silo_endpoint +from sentry.api.permissions import SuperuserPermission +from sentry.utils.email import is_smtp_enabled + +logger = logging.getLogger("sentry") + +SYSTEM_OPTIONS_ALLOWLIST = frozenset( + [ + # Used during setup before the superadmin role with the options.admin permission is authed + "system.admin-email" + ] +) + + +@all_silo_endpoint +class SystemOptionsEndpoint(Endpoint): + publish_status = { + "GET": ApiPublishStatus.PRIVATE, + "PUT": ApiPublishStatus.PRIVATE, + } + owner = ApiOwner.HYBRID_CLOUD + permission_classes = (SuperuserPermission,) + + def get(self, request: Request) -> Response: + query = request.GET.get("query") + if query == "is:required": + option_list = options.filter(flag=options.FLAG_REQUIRED) + elif query: + return Response(f"{query} is not a supported search query", status=400) + else: + option_list = options.all() + + smtp_disabled = not is_smtp_enabled() + + results = {} + for k in option_list: + disabled, disabled_reason = False, None + + if smtp_disabled and k.name[:5] == "mail.": + disabled_reason, disabled = "smtpDisabled", True + elif bool( + k.flags & options.FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(k.name) + ): + # TODO(mattrobenolt): Expose this as a property on Key. + disabled_reason, disabled = "diskPriority", True + + # TODO(mattrobenolt): help, placeholder, title, type + results[k.name] = { + "value": options.get(k.name) if not self.__is_secret(k) else "[redacted]", + "field": { + "default": k.default(), + "required": bool(k.flags & options.FLAG_REQUIRED), + "disabled": disabled, + "disabledReason": disabled_reason, + "isSet": options.isset(k.name), + "allowEmpty": bool(k.flags & options.FLAG_ALLOW_EMPTY), + }, + } + + return Response(results) + + def __is_secret(self, k: Any) -> bool: + keywords = ["secret", "private", "token"] + return (k.flags & options.FLAG_CREDENTIAL) or any( + [keyword in k.name for keyword in keywords] + ) + + def has_permission(self, request: Request) -> bool: + if settings.SENTRY_SELF_HOSTED and request.user.is_superuser: + return True + if not request.access.has_permission("options.admin"): + # We ignore options.admin permission if all keys in the update match the allowlist. + return bool(request.data) and all( + [k in SYSTEM_OPTIONS_ALLOWLIST for k in request.data.keys()] + ) + + return True + + def put(self, request: Request) -> Response: + if not self.has_permission(request): + return Response(status=403) + + # TODO(dcramer): this should validate options before saving them + for k, v in request.data.items(): + if v and isinstance(v, str): + v = v.strip() + try: + option = options.lookup_key(k) + except options.UnknownOption: + # TODO(dcramer): unify API errors + return Response( + {"error": "unknown_option", "errorDetail": {"option": k}}, status=400 + ) + + try: + with transaction.atomic(router.db_for_write(options.default_store.model)): + if not (option.flags & options.FLAG_ALLOW_EMPTY) and not v: + options.delete(k) + else: + options.set(k, v, channel=options.UpdateChannel.APPLICATION) + + logger.info( + "options.update", + extra={ + "ip_address": request.META["REMOTE_ADDR"], + "user_id": request.user.id, + "option_key": k, + "option_value": v, + }, + ) + except (TypeError, AssertionError) as e: + # TODO(chadwhitacre): Use a custom exception for the + # immutability case, especially since asserts disappear with + # `python -O`. + return Response( + { + "error": "invalid_type" if type(e) is TypeError else "immutable_option", + "errorDetail": {"option": k, "message": str(e)}, + }, + status=400, + ) + # TODO(dcramer): this has nothing to do with configuring options and + # should not be set here + options.set( + "sentry:version-configured", + sentry.get_version(), + channel=options.UpdateChannel.APPLICATION, + ) + return Response(status=200) diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py new file mode 100644 index 000000000..05f089e24 --- /dev/null +++ b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py @@ -0,0 +1,303 @@ +import datetime +import logging +from datetime import timedelta +from urllib.parse import urljoin +from urllib.robotparser import RobotFileParser + +from dateutil.parser import parse as parse_datetime +from django.utils import timezone + +from sentry import features, options +from sentry.exceptions import RestrictedIPAddress +from sentry.http import safe_urlopen +from sentry.locks import locks +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.tasks.base import instrumented_task +from sentry.taskworker.namespaces import uptime_tasks +from sentry.uptime.autodetect.ranking import ( + delete_candidate_projects_for_org, + delete_candidate_urls_for_project, + delete_organization_bucket, + get_candidate_projects_for_org, + get_candidate_urls_for_project, + get_organization_bucket, + should_autodetect_for_organization, + should_autodetect_for_project, +) +from sentry.uptime.subscriptions.subscriptions import ( + create_uptime_detector, + delete_uptime_detector, + get_auto_monitored_detectors_for_project, + is_url_auto_monitored_for_project, +) +from sentry.uptime.types import DEFAULT_2XX_STATUS_ASSERTION, UptimeMonitorMode +from sentry.uptime.utils import get_cluster +from sentry.utils import metrics +from sentry.utils.hashlib import md5_text +from sentry.utils.locking import UnableToAcquireLock +from sentry.workflow_engine.models.detector import Detector + +UPTIME_USER_AGENT = "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)" +LAST_PROCESSED_KEY = "uptime_detector_last_processed" +SCHEDULER_LOCK_KEY = "uptime_detector_scheduler_lock" +FAILED_URL_RETRY_FREQ = timedelta(days=7) +URL_MIN_TIMES_SEEN = 5 +URL_MIN_PERCENT = 0.05 +# Default value for how often we should run these subscriptions when onboarding them +ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS = int(timedelta(minutes=60).total_seconds()) +# Default timeout for auto-detected uptime monitors +ONBOARDING_SUBSCRIPTION_TIMEOUT_MS = 10_000 + +logger = logging.getLogger("sentry.uptime-url-autodetection") + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.schedule_detections", + namespace=uptime_tasks, + processing_deadline_duration=60, +) +def schedule_autodetections(): + """ + Runs regularly and fires off a task for each autodetection bucket that needs to be run since + this task last ran. + """ + lock = locks.get( + SCHEDULER_LOCK_KEY, + duration=60, + name="uptime.detection.schedule_detections", + ) + try: + with lock.acquire(): + cluster = get_cluster() + last_processed = cluster.get(LAST_PROCESSED_KEY) + if last_processed is None: + last_processed = timezone.now().replace(second=0, microsecond=0) + else: + last_processed = datetime.datetime.fromtimestamp( + int(last_processed), tz=datetime.UTC + ) + + minutes_since_last_processed = int( + (timezone.now() - last_processed) / timedelta(minutes=1) + ) + for _ in range(minutes_since_last_processed): + metrics.incr("uptime.detectors.scheduler.scheduled_bucket") + last_processed = last_processed + timedelta(minutes=1) + process_autodetection_bucket.delay(last_processed.isoformat()) + + cluster.set(LAST_PROCESSED_KEY, int(last_processed.timestamp()), timedelta(hours=1)) + except UnableToAcquireLock: + # If we can't acquire the lock it just means another task is already handling scheduling, + # so just exit + metrics.incr("uptime.detectors.scheduler.unable_to_acquire_lock") + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.process_detection_bucket", + namespace=uptime_tasks, +) +def process_autodetection_bucket(bucket: str): + """ + Schedules url autodetection for all projects in this time bucket that saw promising urls. + """ + date_bucket = parse_datetime(bucket) + + for organization_id in get_organization_bucket(date_bucket): + metrics.incr("uptime.detectors.scheduler.scheduled_organization") + process_organization_url_ranking.delay(organization_id) + delete_organization_bucket(date_bucket) + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.process_organization_url_ranking", + namespace=uptime_tasks, + processing_deadline_duration=20, +) +def process_organization_url_ranking(organization_id: int): + org = Organization.objects.get_from_cache(id=organization_id) + logger.info( + "uptime.process_organization", + extra={"organization_id": org.id}, + ) + should_autodetect = should_autodetect_for_organization(org) + + for project_id, project_count in get_candidate_projects_for_org(org): + project = Project.objects.get_from_cache(id=project_id) + if not should_autodetect: + # We still want to clear up these urls even if we're no longer autodetecting + delete_candidate_urls_for_project(project) + else: + if process_project_url_ranking(project, project_count): + metrics.incr("uptime.detectors.scheduler.detected_url_for_organization") + should_autodetect = False + + delete_candidate_projects_for_org(org) + + +def process_project_url_ranking(project: Project, project_url_count: int) -> bool: + """ + Looks at candidate urls for a project and determines whether we should start monitoring them + """ + logger.info( + "uptime.process_project", + extra={ + "project_id": project.id, + "project_url_count": project_url_count, + }, + ) + if not should_autodetect_for_project(project): + metrics.incr("uptime.detectors.project_detection_skipped") + return False + + found_url = False + + for url, url_count in get_candidate_urls_for_project(project, limit=5): + if process_candidate_url(project, project_url_count, url, url_count): + found_url = True + break + else: + # TODO: If we don't find any urls to monitor, we want to increment a counter in redis and check the value. + # After a number of failures, we want to stop checking for this project for a while. + pass + + delete_candidate_urls_for_project(project) + return found_url + + +def process_candidate_url( + project: Project, project_url_count: int, url: str, url_count: int +) -> bool: + """ + Takes a candidate url for a project and determines whether we should create an uptime subscription for it. + Checks that: + - URL has been seen at least `URL_MIN_TIMES_SEEN` times and is seen in at least `URL_MIN_PERCENT` of events with urls + - URL hasn't already been checked and failed recently + - Whether we are already monitoring this url for this project + - Whether the url's robots.txt will allow us to monitor this url + + If the url passes, and we don't already have a subscription for it, then create a new remote subscription for the + url and delete any existing automated monitors. + """ + logger.info( + "uptime.process_project_url", + extra={ + "project_id": project.id, + "project_url_count": project_url_count, + "url": url, + "url_count": url_count, + }, + ) + # The url has to be seen a minimum number of times, and make up at least + # a certain percentage of all urls seen in this project + if url_count < URL_MIN_TIMES_SEEN or url_count / project_url_count < URL_MIN_PERCENT: + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "below_thresholds"}) + return False + + # See if we're already auto monitoring this url on this project + if is_url_auto_monitored_for_project(project, url): + # Just mark this successful so `process_project_url_ranking` will choose to not process urls for this project + # for a week + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "already_monitored"}) + return True + + # Check whether we've recently attempted to monitor this url recently and failed. + if is_failed_url(url): + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "previously_failed"}) + return False + + # Check robots.txt to see if it's ok for us to attempt to monitor this url + if not check_url_robots_txt(url): + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "robots_txt"}) + logger.info( + "uptime.url_failed_robots_txt_check", + extra={ + "url": url, + "project": project.id, + }, + ) + set_failed_url(url) + return False + + logger.info( + "uptime.url_autodetected", + extra={ + "url": url, + "project": project.id, + }, + ) + if options.get("uptime.automatic-subscription-creation") and features.has( + "organizations:uptime", project.organization + ): + # If we hit this point, then the url looks worth monitoring. Create an uptime subscription in monitor mode. + monitor_url_for_project(project, url) + # Disable autodetection on this project and organization now that we've successfully found a hostname + project.update_option("sentry:uptime_autodetection", False) + project.organization.update_option("sentry:uptime_autodetection", False) + + metrics.incr("uptime.detectors.candidate_url.succeeded", sample_rate=1.0) + return True + + +def monitor_url_for_project(project: Project, url: str) -> Detector: + """ + Start monitoring a url for a project. Creates a subscription using our onboarding interval and links the project to + it. Also deletes any other auto-detected monitors since this one should replace them. + """ + for uptime_detector in get_auto_monitored_detectors_for_project(project): + delete_uptime_detector(uptime_detector) + metrics.incr("uptime.detectors.candidate_url.monitor_created", sample_rate=1.0) + return create_uptime_detector( + project, + # TODO(epurkhiser): This is where we would put the environment object + # from autodetection if we decide to do that. + environment=None, + url=url, + interval_seconds=ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS, + timeout_ms=ONBOARDING_SUBSCRIPTION_TIMEOUT_MS, + name=f"Uptime Monitoring for {url}", + mode=UptimeMonitorMode.AUTO_DETECTED_ONBOARDING, + assertion=DEFAULT_2XX_STATUS_ASSERTION, + ) + + +def is_failed_url(url: str) -> bool: + key = get_failed_url_key(url) + return get_cluster().exists(key) == 1 + + +def set_failed_url(url: str) -> None: + """ + If we failed to monitor a url for some reason, skip processing it for FAILED_URL_RETRY_FREQ + """ + key = get_failed_url_key(url) + # TODO: Jitter the expiry here, so we don't retry all at the same time. + get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ) + + +def get_failed_url_key(url: str) -> str: + return f"f:u:{md5_text(url).hexdigest()}" + + +def check_url_robots_txt(url: str) -> bool: + try: + return get_robots_txt_parser(url).can_fetch(UPTIME_USER_AGENT, url) + except RestrictedIPAddress: + logger.warning("Blocked robots.txt fetch to restricted IP", exc_info=True) + return False + except Exception: + logger.warning("Failed to check robots.txt", exc_info=True) + return True + + +def get_robots_txt_parser(url: str) -> RobotFileParser: + robots_url = urljoin(url, "robots.txt") + response = safe_urlopen(robots_url, timeout=10, allow_redirects=True) + robot_parser = RobotFileParser(url=robots_url) + if response.status_code in (401, 403): + robot_parser.parse(["User-agent: *", "Disallow: /"]) + elif response.status_code >= 400: + robot_parser.parse([]) + else: + robot_parser.parse(response.text.splitlines()) + return robot_parser diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md new file mode 100644 index 000000000..f97b1d1f6 --- /dev/null +++ b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md @@ -0,0 +1,105 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2008-2024 Functional Software, Inc. dba Sentry + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the Apache License, Version 2.0 that is effective on the second anniversary of +the date we make the Software available. On or after that date, you may use the +Software under the Apache License, Version 2.0, in which case the following +will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py new file mode 100644 index 000000000..9a30cfe54 --- /dev/null +++ b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py @@ -0,0 +1,291 @@ +import datetime +import logging +from datetime import timedelta +from urllib.parse import urljoin +from urllib.robotparser import RobotFileParser + +from dateutil.parser import parse as parse_datetime +from django.utils import timezone + +from sentry import features, options +from sentry.locks import locks +from sentry.models.organization import Organization +from sentry.models.project import Project +from sentry.tasks.base import instrumented_task +from sentry.taskworker.namespaces import uptime_tasks +from sentry.uptime.autodetect.ranking import ( + delete_candidate_projects_for_org, + delete_candidate_urls_for_project, + delete_organization_bucket, + get_candidate_projects_for_org, + get_candidate_urls_for_project, + get_organization_bucket, + should_autodetect_for_organization, + should_autodetect_for_project, +) +from sentry.uptime.subscriptions.subscriptions import ( + create_uptime_detector, + delete_uptime_detector, + get_auto_monitored_detectors_for_project, + is_url_auto_monitored_for_project, +) +from sentry.uptime.types import DEFAULT_2XX_STATUS_ASSERTION, UptimeMonitorMode +from sentry.uptime.utils import get_cluster +from sentry.utils import metrics +from sentry.utils.hashlib import md5_text +from sentry.utils.locking import UnableToAcquireLock +from sentry.workflow_engine.models.detector import Detector + +UPTIME_USER_AGENT = "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)" +LAST_PROCESSED_KEY = "uptime_detector_last_processed" +SCHEDULER_LOCK_KEY = "uptime_detector_scheduler_lock" +FAILED_URL_RETRY_FREQ = timedelta(days=7) +URL_MIN_TIMES_SEEN = 5 +URL_MIN_PERCENT = 0.05 +# Default value for how often we should run these subscriptions when onboarding them +ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS = int(timedelta(minutes=60).total_seconds()) +# Default timeout for auto-detected uptime monitors +ONBOARDING_SUBSCRIPTION_TIMEOUT_MS = 10_000 + +logger = logging.getLogger("sentry.uptime-url-autodetection") + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.schedule_detections", + namespace=uptime_tasks, + processing_deadline_duration=60, +) +def schedule_autodetections(): + """ + Runs regularly and fires off a task for each autodetection bucket that needs to be run since + this task last ran. + """ + lock = locks.get( + SCHEDULER_LOCK_KEY, + duration=60, + name="uptime.detection.schedule_detections", + ) + try: + with lock.acquire(): + cluster = get_cluster() + last_processed = cluster.get(LAST_PROCESSED_KEY) + if last_processed is None: + last_processed = timezone.now().replace(second=0, microsecond=0) + else: + last_processed = datetime.datetime.fromtimestamp( + int(last_processed), tz=datetime.UTC + ) + + minutes_since_last_processed = int( + (timezone.now() - last_processed) / timedelta(minutes=1) + ) + for _ in range(minutes_since_last_processed): + metrics.incr("uptime.detectors.scheduler.scheduled_bucket") + last_processed = last_processed + timedelta(minutes=1) + process_autodetection_bucket.delay(last_processed.isoformat()) + + cluster.set(LAST_PROCESSED_KEY, int(last_processed.timestamp()), timedelta(hours=1)) + except UnableToAcquireLock: + # If we can't acquire the lock it just means another task is already handling scheduling, + # so just exit + metrics.incr("uptime.detectors.scheduler.unable_to_acquire_lock") + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.process_detection_bucket", + namespace=uptime_tasks, +) +def process_autodetection_bucket(bucket: str): + """ + Schedules url autodetection for all projects in this time bucket that saw promising urls. + """ + date_bucket = parse_datetime(bucket) + + for organization_id in get_organization_bucket(date_bucket): + metrics.incr("uptime.detectors.scheduler.scheduled_organization") + process_organization_url_ranking.delay(organization_id) + delete_organization_bucket(date_bucket) + + +@instrumented_task( + name="sentry.uptime.detectors.tasks.process_organization_url_ranking", + namespace=uptime_tasks, + processing_deadline_duration=20, +) +def process_organization_url_ranking(organization_id: int): + org = Organization.objects.get_from_cache(id=organization_id) + logger.info( + "uptime.process_organization", + extra={"organization_id": org.id}, + ) + should_autodetect = should_autodetect_for_organization(org) + + for project_id, project_count in get_candidate_projects_for_org(org): + project = Project.objects.get_from_cache(id=project_id) + if not should_autodetect: + # We still want to clear up these urls even if we're no longer autodetecting + delete_candidate_urls_for_project(project) + else: + if process_project_url_ranking(project, project_count): + metrics.incr("uptime.detectors.scheduler.detected_url_for_organization") + should_autodetect = False + + delete_candidate_projects_for_org(org) + + +def process_project_url_ranking(project: Project, project_url_count: int) -> bool: + """ + Looks at candidate urls for a project and determines whether we should start monitoring them + """ + logger.info( + "uptime.process_project", + extra={ + "project_id": project.id, + "project_url_count": project_url_count, + }, + ) + if not should_autodetect_for_project(project): + metrics.incr("uptime.detectors.project_detection_skipped") + return False + + found_url = False + + for url, url_count in get_candidate_urls_for_project(project, limit=5): + if process_candidate_url(project, project_url_count, url, url_count): + found_url = True + break + else: + # TODO: If we don't find any urls to monitor, we want to increment a counter in redis and check the value. + # After a number of failures, we want to stop checking for this project for a while. + pass + + delete_candidate_urls_for_project(project) + return found_url + + +def process_candidate_url( + project: Project, project_url_count: int, url: str, url_count: int +) -> bool: + """ + Takes a candidate url for a project and determines whether we should create an uptime subscription for it. + Checks that: + - URL has been seen at least `URL_MIN_TIMES_SEEN` times and is seen in at least `URL_MIN_PERCENT` of events with urls + - URL hasn't already been checked and failed recently + - Whether we are already monitoring this url for this project + - Whether the url's robots.txt will allow us to monitor this url + + If the url passes, and we don't already have a subscription for it, then create a new remote subscription for the + url and delete any existing automated monitors. + """ + logger.info( + "uptime.process_project_url", + extra={ + "project_id": project.id, + "project_url_count": project_url_count, + "url": url, + "url_count": url_count, + }, + ) + # The url has to be seen a minimum number of times, and make up at least + # a certain percentage of all urls seen in this project + if url_count < URL_MIN_TIMES_SEEN or url_count / project_url_count < URL_MIN_PERCENT: + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "below_thresholds"}) + return False + + # See if we're already auto monitoring this url on this project + if is_url_auto_monitored_for_project(project, url): + # Just mark this successful so `process_project_url_ranking` will choose to not process urls for this project + # for a week + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "already_monitored"}) + return True + + # Check whether we've recently attempted to monitor this url recently and failed. + if is_failed_url(url): + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "previously_failed"}) + return False + + # Check robots.txt to see if it's ok for us to attempt to monitor this url + if not check_url_robots_txt(url): + metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "robots_txt"}) + logger.info( + "uptime.url_failed_robots_txt_check", + extra={ + "url": url, + "project": project.id, + }, + ) + set_failed_url(url) + return False + + logger.info( + "uptime.url_autodetected", + extra={ + "url": url, + "project": project.id, + }, + ) + if options.get("uptime.automatic-subscription-creation") and features.has( + "organizations:uptime", project.organization + ): + # If we hit this point, then the url looks worth monitoring. Create an uptime subscription in monitor mode. + monitor_url_for_project(project, url) + # Disable autodetection on this project and organization now that we've successfully found a hostname + project.update_option("sentry:uptime_autodetection", False) + project.organization.update_option("sentry:uptime_autodetection", False) + + metrics.incr("uptime.detectors.candidate_url.succeeded", sample_rate=1.0) + return True + + +def monitor_url_for_project(project: Project, url: str) -> Detector: + """ + Start monitoring a url for a project. Creates a subscription using our onboarding interval and links the project to + it. Also deletes any other auto-detected monitors since this one should replace them. + """ + for uptime_detector in get_auto_monitored_detectors_for_project(project): + delete_uptime_detector(uptime_detector) + metrics.incr("uptime.detectors.candidate_url.monitor_created", sample_rate=1.0) + return create_uptime_detector( + project, + # TODO(epurkhiser): This is where we would put the environment object + # from autodetection if we decide to do that. + environment=None, + url=url, + interval_seconds=ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS, + timeout_ms=ONBOARDING_SUBSCRIPTION_TIMEOUT_MS, + name=f"Uptime Monitoring for {url}", + mode=UptimeMonitorMode.AUTO_DETECTED_ONBOARDING, + assertion=DEFAULT_2XX_STATUS_ASSERTION, + ) + + +def is_failed_url(url: str) -> bool: + key = get_failed_url_key(url) + return get_cluster().exists(key) == 1 + + +def set_failed_url(url: str) -> None: + """ + If we failed to monitor a url for some reason, skip processing it for FAILED_URL_RETRY_FREQ + """ + key = get_failed_url_key(url) + # TODO: Jitter the expiry here, so we don't retry all at the same time. + get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ) + + +def get_failed_url_key(url: str) -> str: + return f"f:u:{md5_text(url).hexdigest()}" + + +def check_url_robots_txt(url: str) -> bool: + try: + return get_robots_txt_parser(url).can_fetch(UPTIME_USER_AGENT, url) + except Exception: + logger.warning("Failed to check robots.txt", exc_info=True) + return True + + +def get_robots_txt_parser(url: str) -> RobotFileParser: + robot_parser = RobotFileParser(url=urljoin(url, "robots.txt")) + robot_parser.read() + return robot_parser diff --git a/packages/evals/src/index.test.ts b/packages/evals/src/index.test.ts index 392ec4631..e234e5d75 100644 --- a/packages/evals/src/index.test.ts +++ b/packages/evals/src/index.test.ts @@ -232,6 +232,9 @@ describe('standalone scenario files', () => { ...discoverEvalScenarioFiles('security-review', evalsDir), ...discoverEvalScenarioFiles('verification', evalsDir), ...discoverEvalScenarioFiles('wrdn-authz', evalsDir), + ...discoverEvalScenarioFiles('wrdn-code-execution', evalsDir), + ...discoverEvalScenarioFiles('wrdn-data-exfil', evalsDir), + ...discoverEvalScenarioFiles('wrdn-dos-review', evalsDir), ]; const missingMetadata: string[] = []; diff --git a/packages/evals/src/security-matrix.eval.ts b/packages/evals/src/security-matrix.eval.ts index 0240ec13d..fa9b8d87c 100644 --- a/packages/evals/src/security-matrix.eval.ts +++ b/packages/evals/src/security-matrix.eval.ts @@ -1,5 +1,5 @@ import { join } from 'node:path'; -import { expect } from 'vitest'; +import { afterAll, expect } from 'vitest'; import { describeEval } from 'vitest-evals'; import { resolveSkillAsync } from '@sentry/warden'; import { @@ -10,6 +10,12 @@ import { import { discoverEvalScenarios } from './index.js'; import { formatEvalId, formatEvalTestName } from './names.js'; import { resolveEvalProvider } from './provider.js'; +import { + matrixScenarioKind, + summarizeSecurityMatrix, + type MatrixLane, + type MatrixResult, +} from './security-matrix.js'; import type { EvalMeta } from './types.js'; /** @@ -18,30 +24,41 @@ import type { EvalMeta } from './types.js'; */ const WARDEN_SKILLS_REMOTE = 'getsentry/warden-skills@daf01f970a28450627a8d2d61d9915a742a735e2'; -/** Remote security skill under test. Its scenarios live in the category of the same name. */ -const WRDN_SKILL = 'wrdn-authz'; +/** Remote security skills under test. Each skill's scenarios live in the category of the same name. */ +const MATRIX_SKILLS = ['wrdn-authz', 'wrdn-code-execution', 'wrdn-data-exfil', 'wrdn-dos-review'] as const; const SECURITY_REVIEW_SKILL = '../warden/src/builtin-skills/security-review/SKILL.md'; const MATRIX_RUN_TIMEOUT_MS = 240_000; const MATRIX_EVAL_TIMEOUT_MS = MATRIX_RUN_TIMEOUT_MS + 60_000; const provider = resolveEvalProvider(); -let wrdnSkillPath: Promise | undefined; +const remoteSkillPaths = new Map>(); +const results: MatrixResult[] = []; // Fetch lazily inside tests so suites without credentials never touch the network. -function resolveWrdnSkillPath(): Promise { - wrdnSkillPath ??= resolveSkillAsync(WRDN_SKILL, undefined, { remote: WARDEN_SKILLS_REMOTE }).then((skill) => { - if (!skill.rootDir) { - throw new Error(`Remote skill ${WRDN_SKILL} resolved without a rootDir`); - } - return join(skill.rootDir, 'SKILL.md'); - }); - return wrdnSkillPath; +function resolveRemoteSkillPath(name: string): Promise { + let path = remoteSkillPaths.get(name); + if (!path) { + path = resolveSkillAsync(name, undefined, { remote: WARDEN_SKILLS_REMOTE }).then((skill) => { + if (!skill.rootDir) { + throw new Error(`Remote skill ${name} resolved without a rootDir`); + } + return join(skill.rootDir, 'SKILL.md'); + }); + remoteSkillPaths.set(name, path); + } + return path; } -function describeLane(lane: string, evals: EvalMeta[], resolveMeta: (meta: EvalMeta) => Promise): void { +function describeLane( + skill: string, + lane: MatrixLane, + evals: EvalMeta[], + resolveMeta: (meta: EvalMeta) => Promise, +): void { + const laneSkill = lane === 'skill' ? skill : 'security-review'; describeEval( - lane, + `security-matrix/${skill}/${laneSkill}`, { harness: createWardenEvalHarness({ apiKey: provider.apiKey, @@ -61,13 +78,26 @@ function describeLane(lane: string, evals: EvalMeta[], resolveMeta: (meta: EvalM formatEvalTestName(scenario), { timeout: MATRIX_EVAL_TIMEOUT_MS }, async ({ run }) => { + const record = (passed: boolean) => results.push({ + skill, + scenario: scenario.name, + kind: matrixScenarioKind(scenario), + lane, + passed, + }); const meta = await resolveMeta(scenario); - const result = await run(meta); - const output = WardenEvalOutputSchema.parse(result.output); - - expect(output.name).toBe(formatEvalId(meta)); - console.log(`\n${output.summary}`); - console.log(` Findings: ${output.findings.length}`); + try { + // run() throws when the judge score misses the threshold. + const result = await run(meta); + const output = WardenEvalOutputSchema.parse(result.output); + expect(output.name).toBe(formatEvalId(meta)); + record(true); + console.log(`\n${output.summary}`); + console.log(` Findings: ${output.findings.length}`); + } catch (error) { + record(false); + throw error; + } }, ); } @@ -76,17 +106,26 @@ function describeLane(lane: string, evals: EvalMeta[], resolveMeta: (meta: EvalM } // Each scenario runs under security-review and under the wrdn skill that owns the -// category. A scenario only the wrdn lane passes is coverage security-review lacks. -const evals = discoverEvalScenarios({ - category: WRDN_SKILL, - skill: SECURITY_REVIEW_SKILL, - runtime: 'pi', - model: 'anthropic/claude-sonnet-4-6', -}); +// category. A bug only the wrdn lane catches is coverage security-review lacks. +for (const skill of MATRIX_SKILLS) { + const evals = discoverEvalScenarios({ + category: skill, + skill: SECURITY_REVIEW_SKILL, + runtime: 'pi', + model: 'anthropic/claude-sonnet-4-6', + }); -describeLane(`security-matrix/${WRDN_SKILL}/security-review`, evals, async (meta) => meta); -describeLane( - `security-matrix/${WRDN_SKILL}/${WRDN_SKILL}`, - evals.map((meta) => ({ ...meta, skillName: WRDN_SKILL })), - async (meta) => ({ ...meta, skillPath: await resolveWrdnSkillPath() }), -); + describeLane(skill, 'security-review', evals, async (meta) => meta); + describeLane( + skill, + 'skill', + evals.map((meta) => ({ ...meta, skillName: skill })), + async (meta) => ({ ...meta, skillPath: await resolveRemoteSkillPath(skill) }), + ); +} + +afterAll(() => { + if (results.length > 0) { + console.log(`\n${summarizeSecurityMatrix(results)}`); + } +}); diff --git a/packages/evals/src/security-matrix.test.ts b/packages/evals/src/security-matrix.test.ts new file mode 100644 index 000000000..747e2f4ee --- /dev/null +++ b/packages/evals/src/security-matrix.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { matrixScenarioKind, summarizeSecurityMatrix, type MatrixResult } from './security-matrix.js'; + +function result( + scenario: string, + lane: MatrixResult['lane'], + passed: boolean, + kind: MatrixResult['kind'] = 'bug', + skill = 'wrdn-authz', +): MatrixResult { + return { skill, scenario, kind, lane, passed }; +} + +describe('matrixScenarioKind', () => { + it('treats scenarios without a required should_find as clean', () => { + expect(matrixScenarioKind({ should_find: [{ finding: 'real bug', required: true }] })).toBe('bug'); + expect(matrixScenarioKind({ should_find: [{ finding: 'no bugs expected', required: false }] })).toBe('clean'); + }); +}); + +describe('summarizeSecurityMatrix', () => { + it('counts bugs only the wrdn skill caught and clean fixtures each lane flagged', () => { + const summary = summarizeSecurityMatrix([ + result('scim', 'skill', true), + result('scim', 'security-review', false), + result('projects', 'skill', true), + result('projects', 'security-review', true), + result('workflow', 'skill', false), + result('workflow', 'security-review', true), + result('fixed-scim', 'skill', false, 'clean'), + result('fixed-scim', 'security-review', true, 'clean'), + ]); + + expect(summary).toBe([ + 'Security matrix', + '', + 'wrdn-authz: 1 of 3 bugs caught only by wrdn-authz (scim)', + ' caught: wrdn-authz 2/3, security-review 2/3', + ' caught only by security-review: 1 (workflow)', + ' clean fixtures flagged: wrdn-authz 1/1, security-review 0/1', + ].join('\n')); + }); + + it('reports zero unique catches per skill and names scenarios missing a lane', () => { + const summary = summarizeSecurityMatrix([ + result('ssrf', 'skill', true, 'bug', 'wrdn-data-exfil'), + result('ssrf', 'security-review', true, 'bug', 'wrdn-data-exfil'), + result('logs', 'skill', false, 'bug', 'wrdn-data-exfil'), + ]); + + expect(summary).toContain('wrdn-data-exfil: 0 of 2 bugs caught only by wrdn-data-exfil'); + expect(summary).toContain(' incomplete, missing a lane: logs'); + }); +}); diff --git a/packages/evals/src/security-matrix.ts b/packages/evals/src/security-matrix.ts new file mode 100644 index 000000000..a94ec6e10 --- /dev/null +++ b/packages/evals/src/security-matrix.ts @@ -0,0 +1,71 @@ +import type { EvalMeta } from './types.js'; + +/** Which skill ran a matrix scenario: builtin security-review or the wrdn skill that owns it. */ +export type MatrixLane = 'security-review' | 'skill'; + +/** A bug scenario expects a finding. A clean scenario only guards against false positives. */ +export type MatrixScenarioKind = 'bug' | 'clean'; + +/** One judged run of one scenario in one lane. */ +export interface MatrixResult { + skill: string; + scenario: string; + kind: MatrixScenarioKind; + lane: MatrixLane; + passed: boolean; +} + +/** Classify a scenario: clean when it has no required `should_find`. */ +export function matrixScenarioKind(meta: Pick): MatrixScenarioKind { + return meta.should_find.some((assertion) => assertion.required) ? 'bug' : 'clean'; +} + +function passedIn(results: MatrixResult[], scenario: string, lane: MatrixLane): boolean | undefined { + return results.find((result) => result.scenario === scenario && result.lane === lane)?.passed; +} + +function formatList(names: string[]): string { + return names.length > 0 ? ` (${names.join(', ')})` : ''; +} + +/** + * Render the per-skill coverage verdict. The headline number is bugs the wrdn + * skill caught that security-review missed; zero across a real fixture set is + * the evidence that the skill adds no coverage. + */ +export function summarizeSecurityMatrix(results: MatrixResult[]): string { + const skills = [...new Set(results.map((result) => result.skill))].sort(); + const lines: string[] = ['Security matrix']; + + for (const skill of skills) { + const skillResults = results.filter((result) => result.skill === skill); + const scenarios = [...new Set(skillResults.map((result) => result.scenario))].sort(); + const bugs = scenarios.filter((name) => skillResults.some((r) => r.scenario === name && r.kind === 'bug')); + const clean = scenarios.filter((name) => !bugs.includes(name)); + const incomplete = scenarios.filter( + (name) => passedIn(skillResults, name, 'skill') === undefined + || passedIn(skillResults, name, 'security-review') === undefined, + ); + + const caughtBy = (lane: MatrixLane) => bugs.filter((name) => passedIn(skillResults, name, lane) === true); + const flaggedBy = (lane: MatrixLane) => clean.filter((name) => passedIn(skillResults, name, lane) === false); + const skillCaught = caughtBy('skill'); + const reviewCaught = caughtBy('security-review'); + const onlySkill = skillCaught.filter((name) => !reviewCaught.includes(name)); + const onlyReview = reviewCaught.filter((name) => !skillCaught.includes(name)); + + lines.push( + '', + `${skill}: ${onlySkill.length} of ${bugs.length} bugs caught only by ${skill}${formatList(onlySkill)}`, + ` caught: ${skill} ${skillCaught.length}/${bugs.length}, security-review ${reviewCaught.length}/${bugs.length}`, + ` caught only by security-review: ${onlyReview.length}${formatList(onlyReview)}`, + ` clean fixtures flagged: ${skill} ${flaggedBy('skill').length}/${clean.length}, ` + + `security-review ${flaggedBy('security-review').length}/${clean.length}`, + ); + if (incomplete.length > 0) { + lines.push(` incomplete, missing a lane: ${incomplete.join(', ')}`); + } + } + + return lines.join('\n'); +} diff --git a/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json b/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json new file mode 100644 index 000000000..2ade58af5 --- /dev/null +++ b/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json @@ -0,0 +1,30 @@ +{ + "given": "styled-components history analysis script builds git shell commands with template strings that include the current branch name", + "files": [ + "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts" + ], + "should_find": [ + { + "finding": "originalBranch (the current git branch name) is interpolated into child_process.execSync shell command strings such as git show and git checkout, so a branch name containing shell metacharacters runs arbitrary commands" + } + ], + "should_not_find": [ + "slow synchronous git calls", + "analysis accuracy of styled-components detection" + ], + "supporting_files": [ + "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-061)", + "repository": "getsentry/sentry", + "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source_files": [ + { + "fixturePath": "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts", + "sourcePath": "scripts/analyze-styled.ts", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + } + ] + } +} diff --git a/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json b/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json new file mode 100644 index 000000000..4acf4c6f4 --- /dev/null +++ b/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json @@ -0,0 +1,36 @@ +{ + "given": "PR workflow writes changed filenames from the GitHub API into $GITHUB_OUTPUT with a script that does not guard against newlines", + "files": [ + "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml", + "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py" + ], + "should_find": [ + { + "finding": "parse-pr-files.py prints files= into $GITHUB_OUTPUT; PR filenames are attacker-controlled and can contain newlines, so a fork PR can inject or override step outputs that the workflow forwards to the getsentry dispatch" + } + ], + "should_not_find": [ + "pinned action SHAs", + "token scope of GITHUB_TOKEN for the permissions step" + ], + "supporting_files": [ + "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-060)", + "repository": "getsentry/sentry", + "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source_files": [ + { + "fixturePath": "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml", + "sourcePath": ".github/workflows/getsentry-dispatch.yml", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + }, + { + "fixturePath": "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py", + "sourcePath": ".github/workflows/scripts/parse-pr-files.py", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + } + ] + } +} diff --git a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json new file mode 100644 index 000000000..801cab3b1 --- /dev/null +++ b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json @@ -0,0 +1,30 @@ +{ + "given": "self-hosted feature flag endpoint writes request values into sentry.conf.py as Python source and reloads the config", + "files": [ + "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py" + ], + "should_find": [ + { + "finding": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server" + } + ], + "should_not_find": [ + "GET leaking the list of early features", + "missing CSRF protection" + ], + "supporting_files": [ + "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (pre-fix parent, getsentry/sentry#112980)", + "repository": "getsentry/sentry", + "source_ref": "101823f59fee6d71221feceb0cbcb28cc6024606", + "source_files": [ + { + "fixturePath": "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py", + "sourcePath": "src/sentry/api/endpoints/internal/feature_flags.py", + "ref": "101823f59fee6d71221feceb0cbcb28cc6024606" + } + ] + } +} diff --git a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json new file mode 100644 index 000000000..2a01a9e5b --- /dev/null +++ b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json @@ -0,0 +1,31 @@ +{ + "given": "feature flag endpoint after the fix: request values are normalized to booleans before being written to sentry.conf.py", + "files": [ + "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py" + ], + "should_find": [ + { + "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", + "required": false + } + ], + "should_not_find": [ + "code injection into sentry.conf.py through feature flag values", + "arbitrary Python executed by configure() from request data" + ], + "supporting_files": [ + "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (fixed state)", + "repository": "getsentry/sentry", + "source_ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866", + "source_files": [ + { + "fixturePath": "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py", + "sourcePath": "src/sentry/api/endpoints/internal/feature_flags.py", + "ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866" + } + ] + } +} diff --git a/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json b/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json new file mode 100644 index 000000000..f9ad8368e --- /dev/null +++ b/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json @@ -0,0 +1,42 @@ +{ + "given": "data forwarder API serializer returns the stored provider config, which holds decrypted Segment, AWS SQS, and Splunk credentials", + "files": [ + "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py" + ], + "should_find": [ + { + "finding": "DataForwarderSerializer returns obj.config (and project effectiveConfig) in API responses; config is an EncryptedJSONField decrypted on read, so stored provider secrets such as Segment write keys, AWS secret access keys, and Splunk tokens are exposed to API callers" + } + ], + "should_not_find": [ + "N+1 queries when loading project configs", + "missing pagination on enrolled projects" + ], + "supporting_files": [ + "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py", + "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py", + "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-052)", + "repository": "getsentry/sentry", + "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source_files": [ + { + "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py", + "sourcePath": "src/sentry/integrations/api/serializers/models/data_forwarder.py", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + }, + { + "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py", + "sourcePath": "src/sentry/integrations/models/data_forwarder.py", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + }, + { + "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py", + "sourcePath": "src/sentry/integrations/api/endpoints/data_forwarding_index.py", + "ref": "7f41cc502faa1088108e896565409be72d92bc38" + } + ] + } +} diff --git a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json new file mode 100644 index 000000000..c2189d3e4 --- /dev/null +++ b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json @@ -0,0 +1,31 @@ +{ + "given": "system options endpoint after the fix: secret option values are redacted before logging", + "files": [ + "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py" + ], + "should_find": [ + { + "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", + "required": false + } + ], + "should_not_find": [ + "secret option values logged in cleartext", + "credentials written to logs by options.update" + ], + "supporting_files": [ + "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (fixed state)", + "repository": "getsentry/sentry", + "source_ref": "36ae5236a01ac174b5d3a432ac350fd469f05441", + "source_files": [ + { + "fixturePath": "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py", + "sourcePath": "src/sentry/api/endpoints/system_options.py", + "ref": "36ae5236a01ac174b5d3a432ac350fd469f05441" + } + ] + } +} diff --git a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json new file mode 100644 index 000000000..e6797ccb1 --- /dev/null +++ b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json @@ -0,0 +1,30 @@ +{ + "given": "PUT /system/options/ logs every updated option key and value, including credential options, in cleartext", + "files": [ + "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py" + ], + "should_find": [ + { + "finding": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext" + } + ], + "should_not_find": [ + "superuser permission checks on the endpoint", + "option channel handling" + ], + "supporting_files": [ + "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (pre-fix parent, getsentry/sentry#114187)", + "repository": "getsentry/sentry", + "source_ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a", + "source_files": [ + { + "fixturePath": "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py", + "sourcePath": "src/sentry/api/endpoints/system_options.py", + "ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a" + } + ] + } +} diff --git a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json new file mode 100644 index 000000000..14faa0efc --- /dev/null +++ b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json @@ -0,0 +1,31 @@ +{ + "given": "uptime autodetect after the fix: robots.txt is fetched with safe_urlopen and parsed from the response body", + "files": [ + "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py" + ], + "should_find": [ + { + "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", + "required": false + } + ], + "should_not_find": [ + "SSRF in the robots.txt fetch", + "robots.txt fetched through raw urllib or RobotFileParser.read" + ], + "supporting_files": [ + "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (fixed state)", + "repository": "getsentry/sentry", + "source_ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b", + "source_files": [ + { + "fixturePath": "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py", + "sourcePath": "src/sentry/uptime/autodetect/tasks.py", + "ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b" + } + ] + } +} diff --git a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json new file mode 100644 index 000000000..ad1463d52 --- /dev/null +++ b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json @@ -0,0 +1,30 @@ +{ + "given": "uptime autodetect checks robots.txt for URLs pulled from customer event data with urllib.robotparser, which fetches through raw urllib instead of Sentry's SSRF-safe HTTP stack", + "files": [ + "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py" + ], + "should_find": [ + { + "finding": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)" + } + ], + "should_not_find": [ + "uptime monitor ranking logic", + "metrics tag cardinality" + ], + "supporting_files": [ + "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (pre-fix parent, getsentry/sentry#118358)", + "repository": "getsentry/sentry", + "source_ref": "86006042764d70705ca51df36e3f215effd3113d", + "source_files": [ + { + "fixturePath": "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py", + "sourcePath": "src/sentry/uptime/autodetect/tasks.py", + "ref": "86006042764d70705ca51df36e3f215effd3113d" + } + ] + } +} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json new file mode 100644 index 000000000..551876863 --- /dev/null +++ b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json @@ -0,0 +1,31 @@ +{ + "given": "EAP item serialization after the fix: recursive encoding stringifies values beyond a fixed depth", + "files": [ + "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py" + ], + "should_find": [ + { + "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", + "required": false + } + ], + "should_not_find": [ + "unbounded recursion in _encode_value", + "missing depth limit on nested event attributes" + ], + "supporting_files": [ + "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (fixed state)", + "repository": "getsentry/sentry", + "source_ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e", + "source_files": [ + { + "fixturePath": "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py", + "sourcePath": "src/sentry/eventstream/item_helpers.py", + "ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e" + } + ] + } +} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json new file mode 100644 index 000000000..7b4574a47 --- /dev/null +++ b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json @@ -0,0 +1,30 @@ +{ + "given": "occurrence-to-EAP item serialization recursively encodes nested event arrays and dicts with no depth limit", + "files": [ + "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py" + ], + "should_find": [ + { + "finding": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event" + } + ], + "should_not_find": [ + "float precision when encoding numbers", + "attribute key naming" + ], + "supporting_files": [ + "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (pre-fix parent, getsentry/sentry#108555)", + "repository": "getsentry/sentry", + "source_ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5", + "source_files": [ + { + "fixturePath": "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py", + "sourcePath": "src/sentry/eventstream/item_helpers.py", + "ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5" + } + ] + } +} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json b/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json new file mode 100644 index 000000000..2476fd673 --- /dev/null +++ b/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json @@ -0,0 +1,30 @@ +{ + "given": "EAP search resolver recursively resolves user-supplied boolean and parenthesized query terms with no depth limit", + "files": [ + "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py" + ], + "should_find": [ + { + "finding": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request" + } + ], + "should_not_find": [ + "attribute type resolution", + "aggregate filter ordering" + ], + "supporting_files": [ + "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080 (pre-fix parent, getsentry/sentry#128180)", + "repository": "getsentry/sentry", + "source_ref": "975da7ea97f7c9f964d6238229338fc02896661e", + "source_files": [ + { + "fixturePath": "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py", + "sourcePath": "src/sentry/search/eap/resolver.py", + "ref": "975da7ea97f7c9f964d6238229338fc02896661e" + } + ] + } +} diff --git a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json new file mode 100644 index 000000000..5edd3879b --- /dev/null +++ b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json @@ -0,0 +1,30 @@ +{ + "given": "message parameterization URL regex has an unbounded scheme quantifier that backtracks on long runs of scheme-valid characters", + "files": [ + "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py" + ], + "should_find": [ + { + "finding": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)" + } + ], + "should_not_find": [ + "IPv6 parameterization accuracy", + "hostname validation correctness" + ], + "supporting_files": [ + "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (pre-fix parent, getsentry/sentry#118512)", + "repository": "getsentry/sentry", + "source_ref": "f7c64397f494fe2da5c7775baa3943085998fa5a", + "source_files": [ + { + "fixturePath": "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py", + "sourcePath": "src/sentry/grouping/parameterization.py", + "ref": "f7c64397f494fe2da5c7775baa3943085998fa5a" + } + ] + } +} diff --git a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json new file mode 100644 index 000000000..842cf3578 --- /dev/null +++ b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json @@ -0,0 +1,31 @@ +{ + "given": "message parameterization after the fix: the URL scheme quantifier is bounded to 32 characters", + "files": [ + "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py" + ], + "should_find": [ + { + "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", + "required": false + } + ], + "should_not_find": [ + "catastrophic regex backtracking in the URL scheme pattern", + "unbounded scheme quantifier ReDoS" + ], + "supporting_files": [ + "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md" + ], + "notes": { + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (fixed state)", + "repository": "getsentry/sentry", + "source_ref": "a111b7473cc002875a38a7ffd3792ee1d395f362", + "source_files": [ + { + "fixturePath": "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py", + "sourcePath": "src/sentry/grouping/parameterization.py", + "ref": "a111b7473cc002875a38a7ffd3792ee1d395f362" + } + ] + } +} From a883b3eb01b84dfd9d87758f507a1c9fffcca28f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:49:27 +0000 Subject: [PATCH 03/11] feat(evals): Move security matrix into its own worktree-backed package Replace the vitest matrix in packages/evals and its vendored sentry fixtures with packages/security-evals. Skills now scan corpus and case targets inside a full getsentry/sentry worktree at the pinned SHA, so they can trace callers and permission code the way they do in prod. - one scenario per corpus entry plus 15 hand-picked cases (10 bugs, 5 clean) that record repo, ref, and targets - one Warden pass per SHA with every selected skill on the same context; findings in target files are scored by the corpus rule - per-lane recall, unique catches, misses by security-review, clean flags, cost, and a diffable results JSON - committed security-review baseline from the two published production-config corpus runs - runs only via pnpm evals:security or the manual Security Evals workflow, never from pnpm evals or the PR Evals workflow Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .github/workflows/ci.yml | 2 + .github/workflows/security-evals.yml | 85 + AGENTS.md | 1 + package.json | 8 +- packages/evals/INTERNAL.md | 51 - .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/scripts/analyze-styled.ts | 1655 ---------------- .../github/getsentry/sentry/LICENSE.md | 105 -- .../api/endpoints/data_forwarding_index.py | 101 - .../api/serializers/models/data_forwarder.py | 119 -- .../integrations/models/data_forwarder.py | 38 - .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/eventstream/item_helpers.py | 116 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/eventstream/item_helpers.py | 106 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/src/sentry/search/eap/resolver.py | 1668 ----------------- .../.github/workflows/getsentry-dispatch.yml | 96 - .../workflows/scripts/parse-pr-files.py | 50 - .../github/getsentry/sentry/LICENSE.md | 105 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/grouping/parameterization.py | 695 ------- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/grouping/parameterization.py | 696 ------- .../github/getsentry/sentry/LICENSE.md | 105 -- .../api/endpoints/internal/feature_flags.py | 71 - .../github/getsentry/sentry/LICENSE.md | 105 -- .../api/endpoints/internal/feature_flags.py | 117 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/core/endpoints/scim/utils.py | 274 --- .../github/getsentry/sentry/LICENSE.md | 105 -- .../organization_seer_workflow_configs.py | 627 ------- .../src/sentry/seer/workflows/workflow.py | 430 ----- .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/src/sentry/auth/access.py | 1533 --------------- .../organization_member_team_details.py | 593 ------ .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/src/sentry/auth/access.py | 1537 --------------- .../core/endpoints/organization_projects.py | 483 ----- .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/api/endpoints/system_options.py | 142 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../sentry/api/endpoints/system_options.py | 141 -- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/uptime/autodetect/tasks.py | 303 --- .../github/getsentry/sentry/LICENSE.md | 105 -- .../src/sentry/uptime/autodetect/tasks.py | 291 --- packages/evals/src/index.test.ts | 4 - packages/evals/src/security-matrix.eval.ts | 131 -- packages/evals/src/security-matrix.test.ts | 54 - packages/evals/src/security-matrix.ts | 71 - .../sentry-scim-restriction-revert.json | 30 - ...kflow-put-drops-inaccessible-projects.json | 36 - ...rvice-account-all-projects-team-write.json | 36 - ...-service-account-org-projects-listing.json | 36 - ...analyze-styled-branch-shell-injection.json | 30 - ...ntry-dispatch-github-output-injection.json | 36 - ...ternal-feature-flags-config-injection.json | 30 - ...ernal-feature-flags-restricted-values.json | 31 - ...forwarder-serializer-decrypted-config.json | 42 - .../sentry-system-options-logs-redacted.json | 31 - ...try-system-options-logs-secret-values.json | 30 - .../sentry-uptime-robots-txt-safe-fetch.json | 31 - .../sentry-uptime-robots-txt-ssrf.json | 30 - .../sentry-eap-item-encode-depth-limited.json | 31 - ...y-eap-item-encode-unbounded-recursion.json | 30 - ...-eap-search-boolean-nesting-recursion.json | 30 - ...entry-grouping-url-regex-backtracking.json | 30 - ...try-grouping-url-regex-bounded-scheme.json | 31 - packages/security-evals/.gitignore | 1 + packages/security-evals/README.md | 107 ++ packages/security-evals/baseline/results.json | 1292 +++++++++++++ .../sentry-scim-restriction-revert.json | 10 + ...kflow-put-drops-inaccessible-projects.json | 10 + ...rvice-account-all-projects-team-write.json | 11 + ...-service-account-org-projects-listing.json | 10 + ...ternal-feature-flags-config-injection.json | 10 + ...ernal-feature-flags-restricted-values.json | 10 + .../sentry-system-options-logs-redacted.json | 10 + ...try-system-options-logs-secret-values.json | 10 + .../sentry-uptime-robots-txt-safe-fetch.json | 10 + .../sentry-uptime-robots-txt-ssrf.json | 10 + .../sentry-eap-item-encode-depth-limited.json | 10 + ...y-eap-item-encode-unbounded-recursion.json | 10 + ...-eap-search-boolean-nesting-recursion.json | 10 + ...entry-grouping-url-regex-backtracking.json | 10 + ...try-grouping-url-regex-bounded-scheme.json | 10 + packages/security-evals/package.json | 23 + packages/security-evals/roach.json | 84 + .../scripts/import-benchmark.ts | 19 + .../scripts/prepare-sentry-clone.sh | 14 + packages/security-evals/scripts/run.ts | 76 + packages/security-evals/src/baseline.test.ts | 56 + packages/security-evals/src/baseline.ts | 72 + packages/security-evals/src/entries.test.ts | 57 + packages/security-evals/src/entries.ts | 121 ++ packages/security-evals/src/options.test.ts | 39 + packages/security-evals/src/options.ts | 97 + packages/security-evals/src/results.test.ts | 101 + packages/security-evals/src/results.ts | 213 +++ packages/security-evals/src/score.test.ts | 78 + packages/security-evals/src/score.ts | 79 + packages/security-evals/src/security.eval.ts | 82 + packages/security-evals/src/setup.ts | 14 + packages/security-evals/src/shard.ts | 116 ++ packages/security-evals/src/worktrees.test.ts | 86 + packages/security-evals/src/worktrees.ts | 68 + packages/security-evals/tsconfig.json | 23 + packages/security-evals/vitest.config.ts | 21 + packages/security-evals/vitest.unit.config.ts | 15 + pnpm-lock.yaml | 28 + 111 files changed, 3227 insertions(+), 14666 deletions(-) create mode 100644 .github/workflows/security-evals.yml delete mode 100644 packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts delete mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py delete mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py delete mode 100644 packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py delete mode 100644 packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py delete mode 100644 packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py delete mode 100644 packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py delete mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml delete mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py delete mode 100644 packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py delete mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py delete mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py delete mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py delete mode 100644 packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py delete mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py delete mode 100644 packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py delete mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py delete mode 100644 packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py delete mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py delete mode 100644 packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py delete mode 100644 packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py delete mode 100644 packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py delete mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py delete mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md delete mode 100644 packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py delete mode 100644 packages/evals/src/security-matrix.eval.ts delete mode 100644 packages/evals/src/security-matrix.test.ts delete mode 100644 packages/evals/src/security-matrix.ts delete mode 100644 packages/evals/wrdn-authz/sentry-scim-restriction-revert.json delete mode 100644 packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json delete mode 100644 packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json delete mode 100644 packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json delete mode 100644 packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json delete mode 100644 packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json delete mode 100644 packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json delete mode 100644 packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json delete mode 100644 packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json delete mode 100644 packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json delete mode 100644 packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json delete mode 100644 packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json delete mode 100644 packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json delete mode 100644 packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json delete mode 100644 packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json delete mode 100644 packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json delete mode 100644 packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json delete mode 100644 packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json create mode 100644 packages/security-evals/.gitignore create mode 100644 packages/security-evals/README.md create mode 100644 packages/security-evals/baseline/results.json create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json create mode 100644 packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json create mode 100644 packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json create mode 100644 packages/security-evals/package.json create mode 100644 packages/security-evals/roach.json create mode 100644 packages/security-evals/scripts/import-benchmark.ts create mode 100755 packages/security-evals/scripts/prepare-sentry-clone.sh create mode 100644 packages/security-evals/scripts/run.ts create mode 100644 packages/security-evals/src/baseline.test.ts create mode 100644 packages/security-evals/src/baseline.ts create mode 100644 packages/security-evals/src/entries.test.ts create mode 100644 packages/security-evals/src/entries.ts create mode 100644 packages/security-evals/src/options.test.ts create mode 100644 packages/security-evals/src/options.ts create mode 100644 packages/security-evals/src/results.test.ts create mode 100644 packages/security-evals/src/results.ts create mode 100644 packages/security-evals/src/score.test.ts create mode 100644 packages/security-evals/src/score.ts create mode 100644 packages/security-evals/src/security.eval.ts create mode 100644 packages/security-evals/src/setup.ts create mode 100644 packages/security-evals/src/shard.ts create mode 100644 packages/security-evals/src/worktrees.test.ts create mode 100644 packages/security-evals/src/worktrees.ts create mode 100644 packages/security-evals/tsconfig.json create mode 100644 packages/security-evals/vitest.config.ts create mode 100644 packages/security-evals/vitest.unit.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33036a89..8c930c221 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - run: pnpm lint - name: Service and React dashboard tests run: pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test + - name: Security eval suite unit tests + run: pnpm -C packages/security-evals test - name: Test env: VITEST_JUNIT: test-results.junit.xml diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml new file mode 100644 index 000000000..aaab2d48c --- /dev/null +++ b/.github/workflows/security-evals.yml @@ -0,0 +1,85 @@ +name: Security Evals + +# Manual only. Runs security-review and the wrdn-* security skills over the +# sentry vulnerability corpus and hand-picked cases. See packages/security-evals. +on: + workflow_dispatch: + inputs: + skills: + description: Comma-separated lanes (empty runs all six) + default: '' + entries: + description: Comma-separated entry ids or prefixes ending in * (empty runs all) + default: '' + repeats: + description: Passes per shard + default: '1' + model: + description: Analysis model + default: openrouter/x-ai/grok-4.5 + effort: + description: Analysis effort + default: high + aux_model: + description: Auxiliary model + default: openrouter/openai/gpt-5.6-luna + judge_model: + description: Scoring model + default: openrouter/anthropic/claude-sonnet-4.6 + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + security-evals: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + SENTRY_CLONE: ${{ github.workspace }}/../sentry + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: pnpm/action-setup@02f6c237bd2518259fed6c71566509edfb3f2b74 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24' + cache: 'pnpm' + + - run: pnpm install --frozen-lockfile + + # Keyed on the files that pin the scanned SHAs. Older clones still restore; + # the suite fetches whatever SHA they lack. + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ env.SENTRY_CLONE }} + key: sentry-blobless-${{ hashFiles('packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json', 'packages/security-evals/cases/**') }} + restore-keys: sentry-blobless- + + - name: Prepare sentry clone + run: packages/security-evals/scripts/prepare-sentry-clone.sh + + - name: Run security evals + env: + WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} + ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} + WARDEN_SECURITY_EVALS_SENTRY_REPO: ${{ env.SENTRY_CLONE }} + WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} + WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} + WARDEN_SECURITY_EVALS_REPEATS: ${{ inputs.repeats }} + WARDEN_SECURITY_EVALS_MODEL: ${{ inputs.model }} + WARDEN_SECURITY_EVALS_EFFORT: ${{ inputs.effort }} + WARDEN_SECURITY_EVALS_AUX_MODEL: ${{ inputs.aux_model }} + WARDEN_SECURITY_EVALS_JUDGE_MODEL: ${{ inputs.judge_model }} + # Each sentry worktree is large; drop it once its shard finishes. + WARDEN_SECURITY_EVALS_REMOVE_WORKTREES: '1' + run: pnpm evals:security + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: ${{ !cancelled() }} + with: + name: security-evals-results + path: packages/security-evals/.cache/results.json + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 2235d5023..2e7457109 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,7 @@ packages/ │ └── examples/ # Example configurations ├── docs/ # Astro docs site (warden-docs), deployed via Vercel ├── evals/ # Private eval package: runner, scenarios, fixtures, test skills +├── security-evals/ # Optional security skill matrix over the sentry corpus (pnpm evals:security) ``` ## Key Conventions diff --git a/package.json b/package.json index e8a26a960..531b3d887 100644 --- a/package.json +++ b/package.json @@ -11,13 +11,14 @@ "dev": "pnpm --filter @sentry/warden dev", "lint": "oxlint . && pnpm --filter warden-service-app lint:types", "lint:fix": "oxlint --fix . && pnpm --filter warden-service-app lint:types", - "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test", + "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test && pnpm -C packages/security-evals test", "test:coverage": "pnpm --filter @sentry/warden test:coverage", "test:watch": "pnpm --filter @sentry/warden test:watch", "test:examples": "pnpm --filter @sentry/warden test:examples", "evals": "pnpm -C packages/evals evals", "evals:scaffold": "pnpm -C packages/evals scaffold", - "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck", + "evals:security": "pnpm -C packages/security-evals evals:security", + "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck && pnpm -C packages/security-evals typecheck", "generate:jsonl-schema": "pnpm --filter @sentry/warden generate:jsonl-schema", "docs": "pnpm --filter warden-docs dev", "docs:check": "pnpm -C packages/docs check", @@ -40,6 +41,9 @@ "packages/evals/*.ts": [ "oxlint --fix" ], + "packages/security-evals/**/*.ts": [ + "oxlint --fix" + ], "packages/docs/**/*.astro": [ "pnpm -C packages/docs build" ], diff --git a/packages/evals/INTERNAL.md b/packages/evals/INTERNAL.md index 45ed12ea2..31999ee1e 100644 --- a/packages/evals/INTERNAL.md +++ b/packages/evals/INTERNAL.md @@ -59,62 +59,11 @@ source finding is not actually a reachable bug under the target skill. - `packages/evals/src/code-review.eval.ts`: code-review correctness benchmark scenarios. - `packages/evals/src/security-review.eval.ts`: security-review benchmark scenarios. - `packages/evals/src/verify.eval.ts`: verifier-only scenarios. -- `packages/evals/wrdn-*/*.json`: one scenario per file for a remote - `getsentry/warden-skills` security skill. -- `packages/evals/src/security-matrix.eval.ts`: runs each `wrdn-*` scenario - under both builtin security-review and the owning `wrdn-*` skill. Eval names should read as `/`. Runtime and model belong to the suite configuration, not the case identity. Avoid category names that hide the real skill under test. -## Security Coverage - -Security accuracy has two layers. Neither replaces the other. - -**PR-fixture matrix (CI).** `security-matrix.eval.ts` answers one question: -does a `wrdn-*` skill catch bugs that security-review misses? Each scenario in -`packages/evals//` runs in two lanes, `security-matrix//security-review` -and `security-matrix//`. A case only the `wrdn-*` lane passes is -coverage that skill adds. A skill whose cases security-review also passes is -redundant on that evidence. - -- After the lanes finish, the suite prints a `Security matrix` block. Per - skill, the first line is the number of bugs caught only by that skill. Zero - across a real fixture set is the evidence that the skill adds nothing - security-review does not already catch. -- Bug scenarios come from real code: the parent of a fix commit, a corpus - entry, or a finding only the `wrdn-*` skill posted on a PR. Write - `should_find` for the real bug even when a lane misses it today. -- Clean scenarios use the fixed state of the same file, with no required - `should_find` and the fixed bug in `should_not_find`. They count false - positives per lane. -- Fixtures are whole files, so a lane sees the vulnerable code but not the - diff that introduced it. A bug that only shows as a removed check is harder - to catch here than on the real PR. -- The matrix ignores `warden.toml` path filters. A catch in `scripts/` or - `.github/` proves the skill can find it, not that the production config - would run the skill there. -- Remote skills load from the pinned `WARDEN_SKILLS_REMOTE` ref through - Warden's normal remote skill cache. Bump the SHA on purpose and rerun. -- Some cases are expected to fail in one lane. That lowers the aggregate eval - score by design; read the matrix block, not the average. -- Roach records them like any other eval. Add `run-evals` to record new cases. - -**Vulnerability corpus (repeated benchmark).** The -[Sentry vulnerability corpus](../docs/src/content/docs/benchmarking/running.mdx) -is the regression bar for security-review model, effort, and harness changes. -It is not a CI eval: runs cost real money and are scored by hand. - -- One run is noise. Two identical Grok 4.5 runs agreed on only 32 of the 46 - entries either one found. Compare configurations on at least 3 runs each. -- Report per-entry hit rate across runs, not one found count. An entry found - 3 of 3 times that drops to 0 of 3 is a regression; a 1 of 3 entry moving is - not evidence. -- The corpus is mostly authorization bugs, with a few webhook, crypto, - injection, and secret-logging entries, and no DoS entries. It only scores - security-review, so `wrdn-*` coverage claims come from the matrix. - ## Adding Full-Pipeline Evals 1. Add or scaffold a scenario JSON file under `packages/evals//`. diff --git a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts b/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts deleted file mode 100644 index 2dafee92c..000000000 --- a/packages/evals/fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts +++ /dev/null @@ -1,1655 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -import * as child_process from 'node:child_process'; -import * as fs from 'node:fs'; -import * as path from 'node:path'; - -import * as ts from 'typescript'; - -// Main execution -// analyze-styled.ts is a script that analyzes styled components in TypeScript React files. -// It can be invoked via node analyze-styled.ts and accepts the following options: - -// Execute with: node scripts/analyze-styled.ts - -// -- : Analyze a specific .tsx file instead of searching directories -// -n : Show top N most styled components (default: 10) -// -c : Filter analysis to specific components (comma-separated) -// -l : Show file locations for styled components (requires -c) -// -debug : Enable debug logging -// -g : Use glob pattern to analyze only a subset of files -// --out csv : Output results in CSV format -// : Directory to search for .tsx files (default: './static/app') - -// Examples: -// node scripts/analyze-styled.ts -- ./path/to/file.tsx -// node scripts/analyze-styled.ts -n 20 -c div,span ./static/app -// node scripts/analyze-styled.ts -l -c div ./static/app -// node scripts/analyze-styled.ts -g 'static/app/components/core/**/*.tsx' -// node scripts/analyze-styled.ts --out csv ./static/app - -class Logger { - private debugEnabled: boolean; - - constructor(debugEnabled: boolean) { - this.debugEnabled = debugEnabled; - } - - log(...args: any[]): void { - console.log(...args); - } - - debug(...args: any[]): void { - if (this.debugEnabled) { - console.log(...args); - } - } - - error(...args: any[]): void { - console.error(...args); - } - - fatal(...args: any[]): void { - console.error(...args); - } - - table(...args: any[]): void { - console.table(...args); - } - - setDebugEnabled(enabled: boolean): void { - this.debugEnabled = enabled; - } -} - -// Global logger instance, initialized with debug disabled -const logger = new Logger(false); - -// Metadata collection interfaces -interface Metadata { - analyzedFiles: number; - gitBranch: string | null; - gitCommit: string | null; - searchDirectory: string; - timestamp: string; - totalFiles: number; -} - -// Metadata collection functions -function getGitInfo(): { - branch: string | null; - commit: string | null; - status: string | null; -} { - try { - const commit = child_process - .execSync('git rev-parse HEAD', {encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe']}) - .trim(); - - const branch = child_process - .execSync('git rev-parse --abbrev-ref HEAD', { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'], - }) - .trim(); - - const status = child_process - .execSync('git status --porcelain', { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'], - }) - .trim(); - - return {commit, branch, status: status || 'clean'}; - } catch (error) { - logger.debug('❌ Error getting git info:', (error as Error).message); - return {commit: null, branch: null, status: null}; - } -} - -function collectMetadata( - searchDir: string, - totalFiles: number, - analyzedFiles: number -): Metadata { - const gitInfo = getGitInfo(); - - return { - timestamp: new Date().toISOString(), - gitCommit: gitInfo.commit, - gitBranch: gitInfo.branch, - totalFiles, - analyzedFiles, - searchDirectory: searchDir, - }; -} - -function outputMetadata(metadata: Metadata): void { - if (config.outputFormat === 'csv') { - logger.log('=== METADATA ==='); - const metadataData = [ - {Key: 'Timestamp', Value: metadata.timestamp}, - {Key: 'GitCommit', Value: metadata.gitCommit || 'unknown'}, - {Key: 'GitBranch', Value: metadata.gitBranch || 'unknown'}, - {Key: 'TotalFiles', Value: metadata.totalFiles || 0}, - {Key: 'AnalyzedFiles', Value: metadata.analyzedFiles || 0}, - ]; - logger.log(arrayToCSV(metadataData)); - logger.log(); - } else { - logger.log('\n📋 Analysis Metadata:\n'); - logger.log(` • Timestamp: ${metadata.timestamp}`); - logger.log(` • Git Commit: ${metadata.gitCommit || 'unknown'}`); - logger.log(` • Git Branch: ${metadata.gitBranch || 'unknown'}`); - logger.log(` • Total Files: ${metadata.totalFiles || 0}`); - logger.log(` • Analyzed Files: ${metadata.analyzedFiles || 0}`); - } -} - -// Git history functions -interface GitCommit { - date: string; - hash: string; - timestamp: number; -} - -function getCommitsInDateRange(startDate: string, intervalDays: number): GitCommit[] { - // Precompute interval dates to avoid processing too many commits - const intervalDates: string[] = []; - const start = new Date(startDate); - const now = new Date(); - - let currentDate = new Date(start); - - while (currentDate <= now) { - intervalDates.push(currentDate.toISOString().split('T')[0]!); - currentDate = new Date(currentDate.getTime() + intervalDays * 24 * 60 * 60 * 1000); - } - - logger.log(`Getting first commit for ${intervalDates.length} interval dates...`); - - const commits: GitCommit[] = []; - - // Get the first commit for each interval date - for (const date of intervalDates) { - try { - const nextDate = new Date(date); - nextDate.setDate(nextDate.getDate() + 1); - const nextDateStr = nextDate.toISOString().split('T')[0]!; - - const output = child_process.execSync( - `git log --since="${date}" --until="${nextDateStr}" --format="%H,%ct" --reverse -n 1`, - {encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe']} - ); - - const line = output.trim(); - if (line.length > 0) { - const [hash, timestamp] = line.split(','); - commits.push({ - hash: hash!, - date, - timestamp: parseInt(timestamp!, 10), - }); - } - } catch (error) { - // Skip dates with no commits - continue; - } - } - - return commits; -} - -// CSV helper functions -function escapeCsvField(field: string | number): string { - if (typeof field === 'number') return field.toString(); - if (field === null || field === undefined) return ''; - const fieldStr = field.toString(); - if (fieldStr.includes(',') || fieldStr.includes('"') || fieldStr.includes('\n')) { - return `"${fieldStr.replace(/"/g, '""')}"`; - } - return fieldStr; -} - -function arrayToCSV(data: Array>): string { - if (data.length === 0) return ''; - - const headers = Object.keys(data[0] ?? {}); - const csvHeaders = headers.map(escapeCsvField).join(','); - - const csvRows = data.map(row => - headers - .map(header => { - const value = row[header]; - return escapeCsvField(value !== null && value !== undefined ? value : ''); - }) - .join(',') - ); - - return [csvHeaders, ...csvRows].join('\n'); -} -interface Config { - components: Set | null; - debug: boolean; - interval: number; - outdir: string; - outputFormat: 'table' | 'csv'; - searchDir: string | null; - showLocations: boolean; - startDate: string | null; - targetFile: string | null; - topN: number; - useGlob: boolean; -} - -function parseArguments(args: string[]): Config { - const config: Config = { - debug: false, - topN: 10, - outputFormat: 'table', - searchDir: null, - targetFile: null, - showLocations: false, - useGlob: false, - components: null, - startDate: null, - interval: 7, - outdir: '/tmp/analyze-styled-output', - }; - - for (let i = 0; i < args.length; i++) { - if (args[i] === '--' && i + 1 < args.length) { - config.targetFile = args[i + 1] ?? null; - break; - } else if (args[i] === '-n' && i + 1 < args.length) { - const nextArgV = args[i + 1]; - if (!nextArgV) { - throw new Error('-n option requires a number argument'); - } - const nValue = parseInt(nextArgV, 10); - if (isNaN(nValue) || nValue <= 0) { - throw new Error('-n option must be a positive number'); - } - config.topN = nValue; - i++; - } else if (args[i] === '-c' && i + 1 < args.length) { - config.components = config.components || new Set(); - const componentsArg = args[i + 1]?.split(',') ?? []; - componentsArg.map(c => c.trim()).forEach(c => config.components!.add(c)); - i++; - } else if (args[i] === '-l') { - config.showLocations = true; - } else if (args[i] === '-g') { - config.useGlob = true; - } else if (args[i] === '-debug') { - config.debug = true; - } else if (args[i] === '--out') { - if (i + 1 >= args.length) { - throw new Error( - '--out option requires a format argument. Only "csv" is supported.' - ); - } - const format = args[i + 1]; - if (format === 'csv') { - config.outputFormat = 'csv'; - } else { - throw new Error(`Invalid output format: ${format}. Only 'csv' is supported.`); - } - i++; - } else if (args[i] === '--start-date' && i + 1 < args.length) { - config.startDate = args[i + 1] ?? null; - i++; - } else if (args[i] === '--interval' && i + 1 < args.length) { - const intervalValue = parseInt(args[i + 1] ?? '7', 10); - if (isNaN(intervalValue) || intervalValue <= 0) { - throw new Error('--interval option must be a positive number'); - } - config.interval = intervalValue; - i++; - } else if (args[i] === '--outdir' && i + 1 < args.length) { - config.outdir = args[i + 1] ?? './history-output'; - i++; - } else if (!config.searchDir) { - config.searchDir = args[i] ?? null; - } - } - - return config; -} - -function validateConfig(config: Config): void { - if (config.showLocations && !config.components) { - throw new Error( - '-l option requires -c to specify components filter, otherwise it will analyze all styled components and produce a lot of output.' - ); - } - - if (config.useGlob && !config.searchDir) { - throw new Error('-g option requires a glob pattern argument'); - } - - // Default to './static/app' if no directory specified - if (!config.searchDir && !config.targetFile) { - config.searchDir = './static/app'; - } -} - -function findTsxFiles(dir: string): string[] { - try { - const output = child_process.execSync( - `rg --files --type-add 'tsx:*.tsx' --type tsx "${dir}"`, - { - encoding: 'utf8', - stdio: ['pipe', 'pipe', 'pipe'], - } - ); - - return output - .trim() - .split('\n') - .filter(file => file.length > 0); - } catch (error) { - logger.debug( - `❌ Error running ripgrep: ${(error as Error).message}, falling back to fs` - ); - // Fallback to original implementation if ripgrep fails or is missing - const files = fs.readdirSync(dir); - const results: string[] = []; - - for (const file of files) { - const filePath = path.join(dir, file); - const stat = fs.statSync(filePath); - - if (stat.isDirectory() && !file.startsWith('.') && file !== 'node_modules') { - results.push(...findTsxFiles(filePath)); - } else if (file.endsWith('.tsx')) { - results.push(filePath); - } - } - - return results; - } -} - -function collectTsxFilesForCurrentState(c: Config): string[] { - if (c.targetFile) { - if (!fs.existsSync(c.targetFile)) { - logger.debug(`❌ Target file not found: ${c.targetFile}`); - return []; - } - if (!c.targetFile.endsWith('.tsx')) { - logger.debug(`❌ Target file is not a .tsx file: ${c.targetFile}`); - return []; - } - return [c.targetFile]; - } - const searchDir = c.searchDir || './static/app'; - - if (!fs.existsSync(searchDir)) { - logger.debug(`❌ Search directory not found: ${searchDir}`); - return []; - } - - if (c.useGlob) { - try { - return fs.globSync(searchDir); - } catch (error) { - logger.debug(`❌ Error with glob pattern: ${(error as Error).message}`); - return []; - } - } else { - return findTsxFiles(searchDir); - } -} - -const config = parseArguments(process.argv.slice(2)); -validateConfig(config); - -// Update global logger with debug setting -logger.setDebugEnabled(config.debug); - -// For non-historical analysis, collect files at startup -// For historical analysis, files will be collected per commit -let tsxFiles: string[] = []; - -if (!config.startDate) { - // Only collect files at startup for non-historical analysis - tsxFiles = collectTsxFilesForCurrentState(config); - - // For non-historical analysis, exit if no files found - if (tsxFiles.length === 0) { - if (config.targetFile) { - logger.fatal(`❌ File not found: ${config.targetFile}`); - } else { - logger.fatal('❌ No .tsx files found in search directory'); - } - process.exit(1); - } -} - -// Detector configuration interfaces -interface DetectorContext { - config: Config; - fileName: string; - sourceFile: ts.SourceFile; -} - -abstract class BaseDetector { - abstract name: string; - abstract execute(node: ts.Node, context: DetectorContext): void; - abstract results(): void; - abstract reset(): void; -} - -interface DetectorConfiguration { - detectors: BaseDetector[]; -} - -// Detector classes - -class CoreComponentImportsDetector extends BaseDetector { - name = 'CoreComponents'; - private components = new Set(); - private usage = new Map}>(); - - execute(node: ts.Node, context: DetectorContext): void { - // Handle import declarations - if (ts.isImportDeclaration(node)) { - const importDecl = node; - const moduleSpecifier = importDecl.moduleSpecifier; - - if (moduleSpecifier.kind === ts.SyntaxKind.StringLiteral) { - const moduleName = (moduleSpecifier as ts.StringLiteral).text; - if ( - moduleName.includes('sentry/components/core') && - importDecl.importClause?.namedBindings - ) { - const namedBindings = importDecl.importClause.namedBindings; - if (namedBindings.kind === ts.SyntaxKind.NamedImports) { - const namedImports = namedBindings; - namedImports.elements.forEach(element => { - this.components.add(element.name.text); - }); - } - } - } - } - - // Handle JSX element usage - if (ts.isJsxElement(node)) { - const jsxElement = node; - if (jsxElement.openingElement.tagName.kind === ts.SyntaxKind.Identifier) { - const tagName = jsxElement.openingElement.tagName.text; - - if (this.components.has(tagName)) { - const usage = this.usage.get(tagName); - if (usage) { - usage.files.add(context.fileName); - usage.count++; - } else { - this.usage.set(tagName, { - files: new Set([context.fileName]), - count: 1, - }); - } - } - } - } - } - - reset(): void { - this.components.clear(); - this.usage.clear(); - } - - results(): void { - if (this.usage.size === 0) { - // Early return for empty usage - no computation needed - if (config.outputFormat === 'csv') { - logger.log('=== CORE COMPONENT USAGE (LAYOUT) ==='); - logger.log(arrayToCSV([{Type: 'Layout', Component: '', Instances: 0}])); - logger.log(); - - logger.log('=== CORE COMPONENT USAGE (TEXT) ==='); - logger.log(arrayToCSV([{Type: 'Text', Component: '', Instances: 0}])); - logger.log(); - - logger.log('=== CORE COMPONENT USAGE (ALL COMPONENTS) ==='); - logger.log(arrayToCSV([{Component: '', Instances: 0}])); - logger.log(); - - logger.log('=== CORE COMPONENT USAGE (SUMMARY) ==='); - logger.log( - arrayToCSV([ - {Component: 'Total Files', Instances: 0}, - {Component: 'Core Components', Instances: 0}, - ]) - ); - logger.log(); - } else { - logger.log('\n🧩 Core Component Usage (from sentry/components/core):'); - logger.log('No core component usage found.'); - } - return; - } - - // Parse and categorize components - const layout = Array.from(this.usage.entries()) - .filter( - ([component]) => - component.includes('Flex') || - component.includes('Grid') || - component.includes('Stack') || - component.includes('Container') - ) - .sort((a, b) => b[1].count - a[1].count); - - const text = Array.from(this.usage.entries()) - .filter( - ([component]) => component.includes('Text') || component.includes('Heading') - ) - .sort((a, b) => b[1].count - a[1].count); - - // Prepare data structures for both output formats - const layoutData = layout.map(([component, {count, files: _files}]) => ({ - Type: 'Layout', - Component: component, - Instances: count, - })); - - const textData = text.map(([component, {count, files: _files}]) => ({ - Type: 'Text', - Component: component, - Instances: count, - })); - - const totalCount = Array.from(this.usage.values()).reduce( - (sum, {count}) => sum + count, - 0 - ); - - // Prepare all components data sorted by usage - const allComponentsData = Array.from(this.usage.entries()) - .map(([component, {count, files: _files}]) => ({ - Component: component, - Instances: count, - })) - .sort((a, b) => b.Instances - a.Instances); - - if (config.outputFormat === 'csv') { - // Always output layout section (with empty data if no results) - logger.log('=== CORE COMPONENT USAGE (LAYOUT) ==='); - logger.log( - arrayToCSV( - layoutData.length > 0 - ? layoutData - : [{Type: 'Layout', Component: '', Instances: 0}] - ) - ); - logger.log(); - - // Always output text section (with empty data if no results) - logger.log('=== CORE COMPONENT USAGE (TEXT) ==='); - logger.log( - arrayToCSV( - textData.length > 0 ? textData : [{Type: 'Text', Component: '', Instances: 0}] - ) - ); - logger.log(); - - // Always output all components listing - logger.log('=== CORE COMPONENT USAGE (ALL COMPONENTS) ==='); - logger.log( - arrayToCSV( - allComponentsData.length > 0 - ? allComponentsData - : [{Component: '', Instances: 0}] - ) - ); - logger.log(); - - // Always output summary - logger.log('=== CORE COMPONENT USAGE (SUMMARY) ==='); - logger.log( - arrayToCSV([ - {Component: 'Total Files', Instances: tsxFiles.length || 0}, - {Component: 'Core Components', Instances: totalCount || 0}, - ]) - ); - logger.log(); - } else { - logger.log('\n🧩 Core Component Usage (from sentry/components/core):\n'); - logger.table(layoutData); - logger.table(textData); - logger.log('\n📋 All Core Components:\n'); - logger.table(allComponentsData); - } - } -} - -interface StyledComponent { - component: string; - componentType: 'intrinsic' | 'component' | 'unknown'; - cssRules: string; - expressionCount: number; - file: string; - hasExpressions: boolean; - location: { - column: number; - line: number; - }; -} -class StyledComponentsDetector extends BaseDetector { - name = 'StyledComponents'; - private styledComponents: StyledComponent[] = []; - private componentCounts = new Map(); - private cssRuleCounts = new Map(); - - execute(node: ts.Node, context: DetectorContext): void { - if (!ts.isTaggedTemplateExpression(node)) return; - - const taggedExpr = node; - - if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { - const callExpr = taggedExpr.tag as ts.CallExpression; - - if (callExpr.expression.getText() === 'styled') { - const component = callExpr.arguments[0]; - if (!component) return; - - let componentName = ''; - let componentType: 'intrinsic' | 'component' | 'unknown' = 'unknown'; - - // Extract component name and type - if ( - component.kind === ts.SyntaxKind.StringLiteral || - component.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral - ) { - componentName = (component as ts.StringLiteral).text; - componentType = 'intrinsic'; - } else if (component.kind === ts.SyntaxKind.Identifier) { - componentName = (component as ts.Identifier).text; - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.PropertyAccessExpression) { - componentName = component.getText(); - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.ArrowFunction) { - componentName = 'InlineArrowFunction'; - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.AsExpression) { - const asExpr = component as ts.AsExpression; - componentName = asExpr.expression.getText(); - componentType = 'component'; - } else if ( - component.kind === ts.SyntaxKind.FunctionExpression || - component.kind === ts.SyntaxKind.CallExpression - ) { - componentName = 'InlineFunction'; - componentType = 'component'; - } else { - logger.debug( - 'Debug: ', - component.kind, - 'Text:', - component.getText(), - 'File:', - context.fileName - ); - } - - const cssRules = taggedExpr.template.getText(); - const {line, character} = context.sourceFile.getLineAndCharacterOfPosition( - node.getStart() - ); - - const expressions = - taggedExpr.template.kind === ts.SyntaxKind.TemplateExpression - ? taggedExpr.template.templateSpans - : []; - const hasExpressions = expressions.length > 0; - - const shouldInclude = - context.config.components === null || - context.config.components.has(componentName); - - if (shouldInclude) { - const styledComponent: StyledComponent = { - file: context.fileName, - component: componentName, - componentType, - cssRules: cssRules.trim(), - location: { - line: line + 1, - column: character + 1, - }, - hasExpressions, - expressionCount: expressions.length, - }; - - this.styledComponents.push(styledComponent); - - // Track component usage count - const currentComponentCount = this.componentCounts.get(componentName) || 0; - this.componentCounts.set(componentName, currentComponentCount + 1); - - // Track CSS rules usage count - this.trackCssRules(cssRules.trim()); - } - } - } - } - - reset(): void { - this.styledComponents = []; - this.componentCounts.clear(); - this.cssRuleCounts.clear(); - } - - private trackCssRules(cssRules: string): void { - for (const line of cssRules.split('\n')) { - if ( - line.match(/^\s*$/) || - line.trim() === '`' || - line.match(/^\s*}/) || - line.match(/^\s*\/\*.*\*\/\s*$/) - ) { - continue; - } - - // Skip selectors and media queries for rule counting - if ( - line.match(/^\s*[>&]/) || - line.match(/^\s*@media/) || - line.match(/^\s*@container/) - ) { - continue; - } - - // Handle special expressions - if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { - const currentCount = this.cssRuleCounts.get('overflowEllipsis') || 0; - this.cssRuleCounts.set('overflowEllipsis', currentCount + 1); - continue; - } - - // Parse CSS property:value pairs - const [property] = line.split(':'); - const trimmedProperty = property?.trim(); - - if (trimmedProperty && trimmedProperty.length > 0) { - const currentCount = this.cssRuleCounts.get(trimmedProperty) || 0; - this.cssRuleCounts.set(trimmedProperty, currentCount + 1); - } - } - } - - results(): void { - if (this.styledComponents.length === 0) { - // Early return for empty results - no computation needed - if (config.outputFormat === 'csv') { - logger.log(`=== TOP ${config.topN} MOST COMMONLY STYLED COMPONENTS ===`); - logger.log(arrayToCSV([{Component: '', Instances: 0}])); - logger.log(); - - logger.log(`=== TOP ${config.topN} MOST COMMONLY USED CSS RULES ===`); - logger.log(arrayToCSV([{CSSRule: '', Instances: 0}])); - logger.log(); - } else { - logger.log('\n📊 Found 0 styled components.'); - } - return; - } - - // ==== COMPUTATION PHASE ==== - // Calculate statistics from analysis results - let totalIntrinsic = 0; - let totalReactComponents = 0; - let unknownComponents = 0; - let dynamicExpressions = 0; - let staticExpressions = 0; - - this.styledComponents.forEach(sc => { - if (sc.componentType === 'component') totalReactComponents++; - else if (sc.componentType === 'intrinsic') totalIntrinsic++; - else unknownComponents++; - - // Count expressions - if (sc.hasExpressions) { - dynamicExpressions += sc.expressionCount; - } else { - staticExpressions++; - } - }); - - const totalStyledComponents = this.styledComponents.length; - - // Calculate expression statistics - const avgDynamic = - totalStyledComponents > 0 - ? (dynamicExpressions / totalStyledComponents).toFixed(2) - : '0.00'; - const avgStatic = - totalStyledComponents > 0 - ? (staticExpressions / totalStyledComponents).toFixed(2) - : '0.00'; - const totalExpressions = dynamicExpressions + staticExpressions; - const dynamicPerc = - totalExpressions > 0 - ? ((dynamicExpressions / totalExpressions) * 100).toFixed(1) - : '0.0'; - const staticPerc = - totalExpressions > 0 - ? ((staticExpressions / totalExpressions) * 100).toFixed(1) - : '0.0'; - - // Prepare top components and CSS rules - const topComponents = Array.from(this.componentCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, config.topN); - - const topCssRules = Array.from(this.cssRuleCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, config.topN); - - // Process styled info and parse errors - const styledInfo: Record< - string, - Array<{ - count: number; - expression: StyledComponent; - location: string; - ruleInfo: Record; - }> - > = {}; - - type RuleInfo = { - dynamic: number; - value: string; - children?: RuleInfo[]; - }; - - const parseErrors: string[] = []; - this.styledComponents.forEach(sc => { - const count = (styledInfo[sc.component]?.length ?? 0) + 1; - styledInfo[sc.component] = styledInfo[sc.component] || []; - - const ruleInfo: Record = {}; - - for (const line of sc.cssRules.split('\n')) { - if ( - line.match(/^\s*$/) || - line.trim() === '`' || - line.match(/^\s*}/) || - line.match(/^\s*\/\*.*\*\/\s*$/) - ) { - continue; - } - - if (line.match(/^\s*[>&]/)) { - const subSelector = line.match(/^\s*([>&])/)?.[1] || 'unknown sub selector'; - ruleInfo[subSelector] = ruleInfo[subSelector] || []; - ruleInfo[subSelector].push({ - value: line, - dynamic: 0, - }); - continue; - } - - if (line.match(/^\s*@media/) || line.match(/^\s*@container/)) { - const mediaQuery = '@media'; - ruleInfo[mediaQuery] = ruleInfo[mediaQuery] || []; - ruleInfo[mediaQuery].push({ - value: line, - dynamic: 0, - }); - continue; - } - - if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { - ruleInfo['@expression: p.theme.overflowEllipsis'] = - ruleInfo['@expression: p.theme.overflowEllipsis'] || []; - ruleInfo['@expression: p.theme.overflowEllipsis'].push({ - value: '${p => p.theme.overflowEllipsis}', - dynamic: 1, - }); - continue; - } - - let [property, value] = line.split(':'); - property = property?.trim(); - value = value?.trim().replace(/;$/, ''); - - if (property && value) { - ruleInfo[property] = ruleInfo[property] || []; - ruleInfo[property]!.push({ - value, - dynamic: (value.match(/\$\{[^}]+\}/g) || []).length, - }); - } else { - ruleInfo[line] = ruleInfo[line] || []; - ruleInfo[line].push({ - value: line, - dynamic: 0, - }); - logger.debug('❌ Error parsing line:\n', JSON.stringify(line, null, 2)); - parseErrors.push(`/////////////////\n${line}\n/////////////////\n`); - } - } - - styledInfo[sc.component]!.push({ - count, - location: `${sc.file}:${sc.location.line}:${sc.location.column}`, - expression: sc, - ruleInfo, - }); - }); - - const errorCounts = new Map(); - - for (const error of parseErrors) { - const existing = errorCounts.get(error); - if (existing) { - existing.count++; - } else { - errorCounts.set(error, { - count: 1, - }); - } - } - - // const deduplicatedErrors = Array.from(errorCounts.entries()) - // .sort((a, b) => b[1].count - a[1].count) - // .map(([error, info]) => { - // return `// Error occurred ${info.count} time(s)\n${error}`; - // }) - // .join('\n\n'); - - // ==== LOGGING PHASE ==== - if (config.outputFormat === 'csv') { - // Always output top components section - logger.log(`=== TOP ${config.topN} MOST COMMONLY STYLED COMPONENTS ===`); - const componentsData = - topComponents.length > 0 - ? topComponents.map(([component, count]) => ({ - Component: component, - Instances: count, - })) - : [{Component: '', Instances: 0}]; - logger.log(arrayToCSV(componentsData)); - logger.log(); - - // Always output top CSS rules section - logger.log(`=== TOP ${config.topN} MOST COMMONLY USED CSS RULES ===`); - const cssRulesData = - topCssRules.length > 0 - ? topCssRules.map(([rule, count]) => ({ - CSSRule: rule, - Instances: count, - })) - : [{CSSRule: '', Instances: 0}]; - logger.log(arrayToCSV(cssRulesData)); - logger.log(); - } else { - // Output summary - logger.log(` \n📊 Found ${totalStyledComponents} styled components:\n`); - logger.log(` • Intrinsic elements (div, span, etc.): ${totalIntrinsic}`); - logger.log(` • React components: ${totalReactComponents}`); - logger.log(` • Unknown components: ${unknownComponents}`); - logger.log(` • Total files analyzed: ${tsxFiles.length}`); - - // Output expression statistics - logger.log(' \n📄 Types of expressions:\n '); - logger.log(` • Dynamic expressions per styled component: ~${avgDynamic}`); - logger.log(` • Static expressions per styled component: ~${avgStatic}`); - logger.log(` • Total expressions: ${totalExpressions}`); - logger.log(` • Dynamic/Static ratio: ${dynamicPerc}% / ${staticPerc}%`); - - if (topComponents.length > 0) { - logger.log(`\n🏆 Top ${config.topN} most commonly styled components:\n`); - logger.table( - topComponents.map(([component, count], index) => ({ - Rank: index + 1, - Component: component, - Instances: count, - })) - ); - } - - if (topCssRules.length > 0) { - logger.log(`\n📏 Top ${config.topN} most commonly used CSS rules:\n`); - logger.table( - topCssRules.map(([rule, count], index) => ({ - Rank: index + 1, - 'CSS Rule': rule, - Instances: count, - })) - ); - } - } - } -} - -class StyledUsagePerFileDetector extends BaseDetector { - name = 'StyledUsagePerFile'; - private fileStyledCounts = new Map(); - private filesWithStyled = new Set(); - private totalTsxFiles = new Set(); - - execute(node: ts.Node, context: DetectorContext): void { - // Track all TSX files we analyze - this.totalTsxFiles.add(context.fileName); - - if (!ts.isTaggedTemplateExpression(node)) return; - - const taggedExpr = node; - - if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { - const callExpr = taggedExpr.tag as ts.CallExpression; - - if (callExpr.expression.getText() === 'styled') { - const currentCount = this.fileStyledCounts.get(context.fileName) || 0; - this.fileStyledCounts.set(context.fileName, currentCount + 1); - this.filesWithStyled.add(context.fileName); - } - } - } - - reset(): void { - this.fileStyledCounts.clear(); - this.filesWithStyled.clear(); - this.totalTsxFiles.clear(); - } - - results(): void { - const totalFiles = this.totalTsxFiles.size; - const filesWithStyledCount = this.filesWithStyled.size; - const percentageWithStyled = - totalFiles > 0 ? ((filesWithStyledCount / totalFiles) * 100).toFixed(1) : '0.0'; - - // Get top N files by styled call count - const sortedFiles = Array.from(this.fileStyledCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, config.topN); - - if (config.outputFormat === 'csv') { - // Files with most styled calls - logger.log(`=== TOP ${config.topN} FILES BY STYLED CALL COUNT ===`); - const filesData = - sortedFiles.length > 0 - ? sortedFiles.map(([file, count]) => ({ - File: file, - StyledCalls: count, - })) - : [{File: '', StyledCalls: 0}]; - logger.log(arrayToCSV(filesData)); - logger.log(); - - // Summary statistics - logger.log('=== STYLED USAGE STATISTICS ==='); - logger.log( - arrayToCSV([ - {Metric: 'Total TSX Files', Value: totalFiles}, - {Metric: 'Files with Styled Calls', Value: filesWithStyledCount}, - {Metric: 'Percentage with Styled Calls', Value: `${percentageWithStyled}%`}, - ]) - ); - logger.log(); - } else { - logger.log('\n📄 Styled Usage Per File:\n'); - - if (sortedFiles.length > 0) { - logger.log(`🏆 Top ${config.topN} files by styled call count:\n`); - logger.table( - sortedFiles.map(([file, count], index) => ({ - Rank: index + 1, - File: file.replace(process.cwd() + '/', ''), - 'Styled Calls': count, - })) - ); - } - - logger.log('\n📊 Usage Statistics:\n'); - logger.log(` • Total TSX files analyzed: ${totalFiles}`); - logger.log(` • Files containing styled calls: ${filesWithStyledCount}`); - logger.log( - ` • Percentage of TSX files with styled calls: ${percentageWithStyled}%` - ); - } - } -} - -class FlexOnlyDivsDetector extends BaseDetector { - name = 'FlexOnlyDivs'; - private styledComponents: StyledComponent[] = []; - - execute(node: ts.Node, context: DetectorContext): void { - if (!ts.isTaggedTemplateExpression(node)) return; - - const taggedExpr = node; - - if (taggedExpr.tag.kind === ts.SyntaxKind.CallExpression) { - const callExpr = taggedExpr.tag as ts.CallExpression; - - if (callExpr.expression.getText() === 'styled') { - const component = callExpr.arguments[0]; - if (!component) return; - - let componentName = ''; - let componentType: 'intrinsic' | 'component' | 'unknown' = 'unknown'; - - // Extract component name and type - if ( - component.kind === ts.SyntaxKind.StringLiteral || - component.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral - ) { - componentName = (component as ts.StringLiteral).text; - componentType = 'intrinsic'; - } else if (component.kind === ts.SyntaxKind.Identifier) { - componentName = (component as ts.Identifier).text; - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.PropertyAccessExpression) { - componentName = component.getText(); - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.ArrowFunction) { - componentName = 'InlineArrowFunction'; - componentType = 'component'; - } else if (component.kind === ts.SyntaxKind.AsExpression) { - const asExpr = component as ts.AsExpression; - componentName = asExpr.expression.getText(); - componentType = 'component'; - } else if ( - component.kind === ts.SyntaxKind.FunctionExpression || - component.kind === ts.SyntaxKind.CallExpression - ) { - componentName = 'InlineFunction'; - componentType = 'component'; - } - - // Only process divs and respect component filter - const shouldInclude = - componentName === 'div' && - (context.config.components === null || - context.config.components.has(componentName)); - - if (shouldInclude) { - const cssRules = taggedExpr.template.getText(); - const {line, character} = context.sourceFile.getLineAndCharacterOfPosition( - node.getStart() - ); - - const expressions = - taggedExpr.template.kind === ts.SyntaxKind.TemplateExpression - ? taggedExpr.template.templateSpans - : []; - - const styledComponent: StyledComponent = { - file: context.fileName, - component: componentName, - componentType, - cssRules: cssRules.trim(), - location: { - line: line + 1, - column: character + 1, - }, - hasExpressions: expressions.length > 0, - expressionCount: expressions.length, - }; - - this.styledComponents.push(styledComponent); - } - } - } - } - - reset(): void { - this.styledComponents = []; - } - - results(): void { - // ==== COMPUTATION PHASE ==== - const flexRules = new Set([ - 'flex', - 'flex-direction', - 'flex-wrap', - 'justify-content', - 'align-items', - 'align-content', - 'gap', - ]); - - const flexOnlyComponents: Array<{component: string; location: string}> = []; - - for (const sc of this.styledComponents) { - let hasOnlyFlexRules = true; - let hasDisplayFlexRule = false; - - // Parse CSS rules to check if it only contains flex rules - for (const line of sc.cssRules.split('\n')) { - if ( - line.match(/^\s*$/) || - line.trim() === '`' || - line.match(/^\s*}/) || - line.match(/^\s*\/\*.*\*\/\s*$/) || - line.match(/^\s*[>&]/) || - line.match(/^\s*@media/) || - line.match(/^\s*@container/) - ) { - continue; - } - - // Handle special expressions - if (line.match(/^\s*\$\{p => p.theme.overflowEllipsis\};?/)) { - // This is not a flex rule, so exclude this component - hasOnlyFlexRules = false; - break; - } - - // Parse CSS property:value pairs - const [property] = line.split(':'); - const trimmedProperty = property?.trim(); - - if (trimmedProperty && trimmedProperty.length > 0) { - if (trimmedProperty === 'display') { - // Check if the value contains 'flex' - const [, value] = line.split(':'); - const trimmedValue = value?.trim().replace(/;$/, ''); - if (trimmedValue === 'flex') { - hasDisplayFlexRule = true; - } - } else if (!flexRules.has(trimmedProperty)) { - hasOnlyFlexRules = false; - break; - } - } - } - - if (hasDisplayFlexRule && hasOnlyFlexRules) { - flexOnlyComponents.push({ - location: `${sc.file}:${sc.location.line}:${sc.location.column}`, - component: sc.component, - }); - } - } - - // Limit results to topN - const results = flexOnlyComponents.slice(0, config.topN); - - // ==== LOGGING PHASE ==== - if (results.length === 0) { - if (config.outputFormat === 'csv') { - logger.log('=== STYLED DIVS WITH ONLY FLEX RULES ==='); - logger.log(arrayToCSV([{Rank: 0, Component: '', Location: ''}])); - logger.log(); - } else { - logger.log('\n🎯 Styled Divs with Only Flex Rules:'); - logger.log('No styled divs found that only use flexbox rules.'); - } - return; - } - - if (config.outputFormat === 'csv') { - const csvData = results.map((result, index) => ({ - Rank: index + 1, - Component: result.component, - Location: result.location, - })); - logger.log('=== STYLED DIVS WITH ONLY FLEX RULES ==='); - logger.log(arrayToCSV(csvData)); - logger.log(); - } else { - logger.log(`\n🎯 Top ${config.topN} Styled Divs with Only Flex Rules:\n`); - logger.table( - results.map((result, index) => ({ - '#': index + 1, - Component: result.component, - Location: result.location, - })) - ); - } - } -} - -function analyze( - sourceFile: ts.SourceFile, - fileName: string, - detectorConfig: DetectorConfiguration -): void { - const context: DetectorContext = { - sourceFile, - fileName, - config, - }; - - function visit(node: ts.Node): void { - // Run all detectors for every node - for (const detector of detectorConfig.detectors) { - detector.execute(node, context); - } - - ts.forEachChild(node, visit); - } - - visit(sourceFile); -} - -// History analysis functions -let interruptRequested = false; - -interface CommitAnalysisResult { - commit: GitCommit; - filename: string; - output: string; -} - -function processCommit( - commit: GitCommit, - originalBranch: string -): CommitAnalysisResult | null { - if (interruptRequested) { - logger.log('🛑 Interrupt detected, skipping commit processing'); - return null; - } - - // Clean any uncommitted changes before checkout - // Use -f to force removal and -x to include ignored files - child_process.execSync('git clean -d -f -x', { - stdio: ['pipe', 'pipe', 'pipe'], - }); - - // Reset any staged changes - child_process.execSync('git reset --hard', { - stdio: ['pipe', 'pipe', 'pipe'], - }); - - // Checkout the commit (force checkout to handle conflicts) - child_process.execSync(`git checkout -f ${commit.hash}`, { - stdio: ['pipe', 'pipe', 'pipe'], - }); - - // Check if analyze-styled.ts exists in this commit - if (!fs.existsSync('scripts/analyze-styled.ts')) { - logger.debug('Script not found in commit, copying from current version...'); - // Get the current script content and write it temporarily - child_process.execSync( - `git show ${originalBranch}:scripts/analyze-styled.ts > scripts/analyze-styled.ts` - ); - } - - // Collect files for this specific commit - const commitTsxFiles = collectTsxFilesForCurrentState(config); - - if (commitTsxFiles.length === 0) { - logger.debug( - `No .tsx files found in commit ${commit.hash.substring(0, 7)}, skipping analysis.` - ); - return null; - } - - logger.debug( - `Found ${commitTsxFiles.length} .tsx files in commit ${commit.hash.substring(0, 7)}` - ); - - // Reset all detectors before analyzing this commit - for (const detector of detectorConfig.detectors) { - detector.reset(); - } - - // Capture the current stdout to save to file - const originalLog = logger.log; - const originalTable = logger.table; - let output = ''; - - logger.log = (...args: any[]) => { - output += args.join(' ') + '\n'; - }; - logger.table = (...args: any[]) => { - output += JSON.stringify(args[0], null, 2) + '\n'; - }; - - // Run the analysis for this commit with commit-specific files - runCurrentAnalysis(commitTsxFiles); - - // Restore original logging - logger.log = originalLog; - logger.table = originalTable; - - // Return analysis result instead of writing immediately - const filename = `${commit.date}-${commit.hash.substring(0, 7)}.txt`; - - logger.log(`✅ Completed analysis for ${commit.hash.substring(0, 7)}`); - - return { - commit, - output, - filename, - }; -} - -function runHistoryAnalysis(c: Config) { - // Reset interrupt flag - interruptRequested = false; - - if (!c.startDate) { - throw new Error('--start-date is required for historical analysis'); - } - - // Safety check: prevent running on dirty git tree - const gitInfo = getGitInfo(); - if (gitInfo.status !== 'clean') { - throw new Error( - 'Git tree is dirty. Please commit or stash your changes before running historical analysis.\n' + - 'This safety check prevents potential data loss during git operations.' - ); - } - - // Ensure output directory exists - if (!fs.existsSync(c.outdir)) { - fs.mkdirSync(c.outdir, {recursive: true}); - } - - // Get current git state for restoration - const originalBranch = child_process - .execSync('git rev-parse --abbrev-ref HEAD', {encoding: 'utf8'}) - .trim(); - const hasUnstagedChanges = - child_process.execSync('git status --porcelain', {encoding: 'utf8'}).trim().length > - 0; - - // Set up cleanup function for process interrupts - - // Set up interrupt handlers that set flag instead of immediately exiting - const handleInterrupt = (_signal: string) => { - // Set flag first, before any other operations - interruptRequested = true; - - logger.log('\n⚠️ Process interrupted, cleaning up...'); - try { - // Clean any uncommitted changes (including ignored files) - child_process.execSync('git clean -d -f -x', {stdio: ['pipe', 'pipe', 'pipe']}); - // Reset any staged changes - child_process.execSync('git reset --hard', {stdio: ['pipe', 'pipe', 'pipe']}); - // Force checkout to restore original branch - child_process.execSync(`git checkout -f ${originalBranch}`, { - stdio: ['pipe', 'pipe', 'pipe'], - }); - // Restore stashed changes if any existed - if (hasUnstagedChanges) { - child_process.execSync('git stash pop', {stdio: ['pipe', 'pipe', 'pipe']}); - } - logger.log('✅ Cleanup completed, restored to original state'); - } catch (error) { - logger.fatal('❌ Error during cleanup:', (error as Error).message); - } - process.exit(0); - }; - - // Register interrupt handlers with signal names - process.on('SIGINT', () => handleInterrupt('SIGINT')); - process.on('SIGTERM', () => handleInterrupt('SIGTERM')); - process.on('SIGUSR1', () => handleInterrupt('SIGUSR1')); - process.on('SIGUSR2', () => handleInterrupt('SIGUSR2')); - - // Collect all analysis results in memory - const analysisResults: CommitAnalysisResult[] = []; - - try { - // Stash changes if any exist - if (hasUnstagedChanges) { - child_process.execSync('git stash push -m "analyze-styled temporary stash"', { - stdio: 'inherit', - }); - } - const commits = getCommitsInDateRange(c.startDate, c.interval); - - logger.log(`Analyzing ${commits.length} commits at ${c.interval}-day intervals`); - - for (let i = 0; i < commits.length; i++) { - // Check for interrupt before processing each commit - if (interruptRequested) { - logger.log(`\n⚠️ Interrupt received, stopping after ${i} commits processed.`); - break; - } - - const commit = commits[i]!; - logger.log( - `\n[${i + 1}/${commits.length}] Processing commit ${commit.hash.substring(0, 7)} (${commit.date})` - ); - - try { - const result = processCommit(commit, originalBranch); - if (result) { - analysisResults.push(result); - } - } catch (error) { - logger.error( - `❌ Error processing commit ${commit.hash.substring(0, 7)}: ${(error as Error).message}` - ); - continue; - } - } - } finally { - // Remove interrupt handlers - process.removeAllListeners('SIGINT'); - process.removeAllListeners('SIGTERM'); - process.removeAllListeners('SIGUSR1'); - process.removeAllListeners('SIGUSR2'); - - // Restore original state - try { - // Clean any uncommitted changes before final checkout (including ignored files) - child_process.execSync('git clean -d -f -x', { - stdio: ['pipe', 'pipe', 'pipe'], - }); - // Reset any staged changes - child_process.execSync('git reset --hard', { - stdio: ['pipe', 'pipe', 'pipe'], - }); - // Force checkout to restore original branch - child_process.execSync(`git checkout -f ${originalBranch}`, { - stdio: ['pipe', 'pipe', 'pipe'], - }); - - // Restore stashed changes if any existed - if (hasUnstagedChanges) { - child_process.execSync('git stash pop', {stdio: ['pipe', 'pipe', 'pipe']}); - } - } catch (error) { - logger.fatal('❌ Error restoring git state:', (error as Error).message); - } - - // Write all analysis results to files after git state is restored - if (analysisResults.length > 0) { - logger.log(`\n📝 Writing ${analysisResults.length} analysis files...`); - - // Ensure output directory exists - if (!fs.existsSync(c.outdir)) { - fs.mkdirSync(c.outdir, {recursive: true}); - } - - for (const result of analysisResults) { - try { - const filepath = path.join(c.outdir, result.filename); - fs.writeFileSync(filepath, result.output); - logger.log(`✅ Saved analysis to ${result.filename}`); - } catch (error) { - logger.error( - `❌ Error writing ${result.filename}: ${(error as Error).message}` - ); - } - } - } - } - - logger.log(`\n🎉 Historical analysis complete! Results saved in ${config.outdir}`); -} - -function runCurrentAnalysis(filesToAnalyze?: string[]) { - // Use provided files or fall back to global tsxFiles - const files = filesToAnalyze || tsxFiles; - - let successfullyAnalyzedFiles = 0; - // Process all files with detectors - for (const file of files) { - try { - const sourceCode = fs.readFileSync(file, 'utf8'); - const sourceFile = ts.createSourceFile( - file, - sourceCode, - ts.ScriptTarget?.Latest, - true - ); - - analyze(sourceFile, file, detectorConfig); - successfullyAnalyzedFiles++; - } catch (error) { - logger.fatal(`❌ Error parsing ${file}:`, (error as Error).message); - } - } - - // Collect and output metadata first (for CSV format) - const searchDirectory = config.searchDir || config.targetFile || './static/app'; - const metadata = collectMetadata( - searchDirectory, - files.length, - successfullyAnalyzedFiles - ); - outputMetadata(metadata); - - // Execute detector results - for (const detector of detectorConfig.detectors) { - detector.results(); - } -} - -// Create detector configuration -const detectorConfig: DetectorConfiguration = { - detectors: [ - new StyledComponentsDetector(), - new CoreComponentImportsDetector(), - new FlexOnlyDivsDetector(), - new StyledUsagePerFileDetector(), - ], -}; - -// Setup graceful exit handler for main analysis -function setupGracefulExit(): void { - const cleanup = () => { - logger.log('\n⚠️ Process interrupted, exiting gracefully...'); - process.exit(0); - }; - - // Register interrupt handlers - process.on('SIGINT', cleanup); - process.on('SIGTERM', cleanup); - process.on('SIGUSR1', cleanup); - process.on('SIGUSR2', cleanup); -} - -// Main execution: check if historical analysis is requested -if (config.startDate) { - runHistoryAnalysis(config); -} else { - setupGracefulExit(); - runCurrentAnalysis(); -} diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py deleted file mode 100644 index b03b50a5c..000000000 --- a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py +++ /dev/null @@ -1,101 +0,0 @@ -from django.utils.decorators import method_decorator -from django.views.decorators.cache import never_cache -from drf_spectacular.utils import extend_schema -from rest_framework import status -from rest_framework.request import Request -from rest_framework.response import Response - -from sentry import features -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import cell_silo_endpoint -from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission -from sentry.api.paginator import OffsetPaginator -from sentry.api.serializers import serialize -from sentry.apidocs.constants import RESPONSE_BAD_REQUEST, RESPONSE_FORBIDDEN -from sentry.apidocs.examples.integration_examples import IntegrationExamples -from sentry.apidocs.parameters import GlobalParams -from sentry.apidocs.utils import inline_sentry_response_serializer -from sentry.integrations.api.serializers.models.data_forwarder import DataForwarderResponse -from sentry.integrations.api.serializers.rest_framework.data_forwarder import ( - DataForwarderSerializer, -) -from sentry.integrations.models.data_forwarder import DataForwarder -from sentry.web.decorators import set_referrer_policy - - -class OrganizationDataForwardingDetailsPermission(OrganizationPermission): - scope_map = { - "GET": ["org:write"], - "POST": ["org:write"], - } - - -@cell_silo_endpoint -@extend_schema(tags=["Integrations"]) -class DataForwardingIndexEndpoint(OrganizationEndpoint): - owner = ApiOwner.INTEGRATIONS - publish_status = { - "GET": ApiPublishStatus.PUBLIC, - "POST": ApiPublishStatus.PUBLIC, - } - permission_classes = (OrganizationDataForwardingDetailsPermission,) - - @extend_schema( - operation_id="Retrieve Data Forwarders for an Organization", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - responses={ - 200: inline_sentry_response_serializer( - "ListDataForwarderResponse", list[DataForwarderResponse] - ) - }, - examples=IntegrationExamples.LIST_DATA_FORWARDERS, - ) - @set_referrer_policy("strict-origin-when-cross-origin") - @method_decorator(never_cache) - def get(self, request: Request, organization) -> Response: - """ - Returns a list of data forwarders for an organization. - """ - queryset = DataForwarder.objects.filter(organization_id=organization.id) - - return self.paginate( - request=request, - queryset=queryset, - on_results=lambda x: serialize(x, request.user), - paginator_cls=OffsetPaginator, - ) - - @extend_schema( - operation_id="Create a Data Forwarder for an Organization", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - request=DataForwarderSerializer, - responses={ - 201: inline_sentry_response_serializer("DataForwarderResponse", DataForwarderResponse), - 400: RESPONSE_BAD_REQUEST, - 403: RESPONSE_FORBIDDEN, - }, - examples=IntegrationExamples.SINGLE_DATA_FORWARDER, - ) - @set_referrer_policy("strict-origin-when-cross-origin") - @method_decorator(never_cache) - def post(self, request: Request, organization) -> Response: - """ - Creates a new data forwarder for an organization. - Only one data forwarder can be created per provider for a given organization. - - Project-specific overrides can only be created after creating the data forwarder. - """ - if not features.has("organizations:data-forwarding", organization): - return self.respond(status=status.HTTP_403_FORBIDDEN) - - data = request.data - data["organization_id"] = organization.id - - serializer = DataForwarderSerializer(data=data, context={"organization": organization}) - if not serializer.is_valid(): - return self.respond(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - return self.respond( - serialize(serializer.save(), request.user), status=status.HTTP_201_CREATED - ) diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py deleted file mode 100644 index a95001515..000000000 --- a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py +++ /dev/null @@ -1,119 +0,0 @@ -from collections import defaultdict -from collections.abc import Mapping, Sequence -from datetime import datetime -from typing import Any, TypedDict - -from django.contrib.auth.models import AnonymousUser - -from sentry.api.serializers import Serializer, register, serialize -from sentry.integrations.models.data_forwarder import DataForwarder -from sentry.integrations.models.data_forwarder_project import DataForwarderProject -from sentry.users.models.user import User -from sentry.users.services.user import RpcUser - - -class ProjectResponse(TypedDict): - id: str - slug: str - platform: str | None - - -class DataForwarderProjectResponse(TypedDict): - id: str - isEnabled: bool - dataForwarderId: str - project: ProjectResponse - overrides: dict[str, str] - effectiveConfig: dict[str, str] - dateAdded: datetime - dateUpdated: datetime - - -class DataForwarderResponse(TypedDict): - id: str - organizationId: str - isEnabled: bool - enrollNewProjects: bool - enrolledProjects: list[ProjectResponse] - provider: str - config: dict[str, str] - projectConfigs: list[DataForwarderProjectResponse] - dateAdded: datetime - dateUpdated: datetime - - -@register(DataForwarder) -class DataForwarderSerializer(Serializer): - def get_attrs( - self, - item_list: Sequence[DataForwarder], - user: User | RpcUser | AnonymousUser, - **kwargs: Any, - ) -> dict[DataForwarder, dict[str, Any]]: - project_configs = DataForwarderProject.objects.filter( - data_forwarder__in=item_list - ).select_related("project", "data_forwarder") - project_config_attrs = defaultdict(set) - for project_config in project_configs: - project_config_attrs[project_config.data_forwarder].add(project_config) - - return { - data_forwarder: {"project_configs": project_config_attrs.get(data_forwarder, set())} - for data_forwarder in item_list - } - - def serialize( - self, - obj: DataForwarder, - attrs: Mapping[str, Any], - user: User | RpcUser | AnonymousUser, - **kwargs: Any, - ) -> DataForwarderResponse: - project_configs = attrs.get("project_configs", set()) - - return { - "id": str(obj.id), - "organizationId": str(obj.organization_id), - "isEnabled": obj.is_enabled, - "enrollNewProjects": obj.enroll_new_projects, - "enrolledProjects": [ - { - "id": str(project_config.project_id), - "slug": project_config.project.slug, - "platform": project_config.project.platform, - } - for project_config in project_configs - ], - "provider": obj.provider, - "config": obj.config, - "projectConfigs": [ - serialize(project_config, user=user) for project_config in project_configs - ], - "dateAdded": obj.date_added, - "dateUpdated": obj.date_updated, - } - - -@register(DataForwarderProject) -class DataForwarderProjectSerializer(Serializer): - def serialize( - self, - obj: DataForwarderProject, - attrs: Mapping[str, Any], - user: User | RpcUser | AnonymousUser, - **kwargs: Any, - ) -> DataForwarderProjectResponse: - return { - "id": str(obj.id), - "isEnabled": obj.is_enabled, - "dataForwarderId": str(obj.data_forwarder_id), - "project": { - "id": str(obj.project_id), - "slug": obj.project.slug, - "platform": obj.project.platform, - }, - "overrides": obj.overrides, - "effectiveConfig": obj.get_config(), - "dateAdded": obj.date_added, - "dateUpdated": obj.date_updated, - } diff --git a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py b/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py deleted file mode 100644 index d748fb6f2..000000000 --- a/packages/evals/fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py +++ /dev/null @@ -1,38 +0,0 @@ -from django.db import models - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import DefaultFieldsModel, FlexibleForeignKey, cell_silo_model -from sentry.db.models.fields.encryption.encrypted_json_field import EncryptedJSONField - - -@cell_silo_model -class DataForwarder(DefaultFieldsModel): - """ - Configuration for data forwarding to external services (Segment, SQS, or Splunk). - This is a general config that can apply to specific projects or all projects in an organization. - """ - - __relocation_scope__ = RelocationScope.Organization - - organization = FlexibleForeignKey("sentry.Organization", on_delete=models.CASCADE) - is_enabled = models.BooleanField(default=True) - - enroll_new_projects = models.BooleanField(default=False) - enrolled_projects = models.ManyToManyField( - "sentry.Project", through="sentry.DataForwarderProject", related_name="data_forwarders" - ) - - provider = models.CharField( - max_length=64, - choices=[ - ("segment", "Segment"), - ("sqs", "Amazon SQS"), - ("splunk", "Splunk"), - ], - ) - config = EncryptedJSONField(default=dict) - - class Meta: - app_label = "sentry" - db_table = "sentry_dataforwarder" - unique_together = (("organization", "provider"),) diff --git a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py b/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py deleted file mode 100644 index b454582fc..000000000 --- a/packages/evals/fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py +++ /dev/null @@ -1,116 +0,0 @@ -from collections.abc import Mapping -from typing import Any - -from google.protobuf.timestamp_pb2 import Timestamp -from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE -from sentry_protos.snuba.v1.trace_item_pb2 import ( - AnyValue, - ArrayValue, - KeyValue, - KeyValueList, - TraceItem, -) - -from sentry.models.project import Project -from sentry.services.eventstore.models import Event, GroupEvent -from sentry.utils import json -from sentry.utils.eap import hex_to_item_id - -# Max depth for recursive encoding to protobuf AnyValue. -_ENCODE_MAX_DEPTH = 6 - - -def serialize_event_data_as_item( - event: Event | GroupEvent, event_data: Mapping[str, Any], project: Project -) -> TraceItem: - return TraceItem( - item_id=hex_to_item_id(event_data["event_id"]), - item_type=TRACE_ITEM_TYPE_OCCURRENCE, - trace_id=event_data["contexts"]["trace"]["trace_id"], - timestamp=Timestamp(seconds=int(event_data["timestamp"])), - organization_id=project.organization_id, - project_id=project.id, - received=( - Timestamp(seconds=int(event_data["received"])) if "received" in event_data else None - ), - retention_days=event_data.get("retention_days", 90), - attributes=encode_attributes( - event, event_data, ignore_fields={"event_id", "timestamp", "tags"} - ), - ) - - -def _encode_value(value: Any, _depth: int = 0) -> AnyValue: - if _depth > _ENCODE_MAX_DEPTH: - # Beyond max depth, stringify to prevent protobuf nesting limit errors. - return AnyValue(string_value=json.dumps(value)) - - if isinstance(value, str): - return AnyValue(string_value=value) - elif isinstance(value, bool): - # Note: bool check must come before int check since bool is a subclass of int - return AnyValue(bool_value=value) - elif isinstance(value, int): - # int_value is a signed int64, so it has a range of valid values. - # if value doesn't fit into an int64, cast it to string. - if abs(value) >= (2**63): - return AnyValue(string_value=str(value)) - return AnyValue(int_value=value) - elif isinstance(value, float): - return AnyValue(double_value=value) - elif isinstance(value, list) or isinstance(value, tuple): - # Not yet processed on EAP side - return AnyValue( - array_value=ArrayValue( - values=[_encode_value(v, _depth + 1) for v in value if v is not None] - ) - ) - elif isinstance(value, dict): - # Not yet processed on EAP side - return AnyValue( - kvlist_value=KeyValueList( - values=[ - KeyValue(key=str(kv[0]), value=_encode_value(kv[1], _depth + 1)) - for kv in value.items() - if kv[1] is not None - ] - ) - ) - else: - raise NotImplementedError(f"encode not supported for {type(value)}") - - -def encode_attributes( - event: Event | GroupEvent, event_data: Mapping[str, Any], ignore_fields: set[str] | None = None -) -> Mapping[str, AnyValue]: - attributes = {} - ignore_fields = ignore_fields or set() - - for key, value in event_data.items(): - if key in ignore_fields: - continue - if value is None: - continue - attributes[key] = _encode_value(value) - - if event.group_id: - attributes["group_id"] = AnyValue(int_value=event.group_id) - - format_tag_key = lambda key: f"tags[{key}]" - - tag_keys = set() - tags = event_data.get("tags") - if tags is not None: - for tag in tags: - if tag is None: - continue - key, value = tag - if value is None: - continue - formatted_key = format_tag_key(key) - attributes[formatted_key] = _encode_value(value) - tag_keys.add(formatted_key) - - attributes["tag_keys"] = _encode_value(sorted(tag_keys)) - - return attributes diff --git a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py b/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py deleted file mode 100644 index bbff7d671..000000000 --- a/packages/evals/fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py +++ /dev/null @@ -1,106 +0,0 @@ -from collections.abc import Mapping -from typing import Any - -from google.protobuf.timestamp_pb2 import Timestamp -from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE -from sentry_protos.snuba.v1.trace_item_pb2 import ( - AnyValue, - ArrayValue, - KeyValue, - KeyValueList, - TraceItem, -) - -from sentry.models.project import Project -from sentry.services.eventstore.models import Event, GroupEvent -from sentry.utils.eap import hex_to_item_id - - -def serialize_event_data_as_item( - event: Event | GroupEvent, event_data: Mapping[str, Any], project: Project -) -> TraceItem: - return TraceItem( - item_id=hex_to_item_id(event_data["event_id"]), - item_type=TRACE_ITEM_TYPE_OCCURRENCE, - trace_id=event_data["contexts"]["trace"]["trace_id"], - timestamp=Timestamp(seconds=int(event_data["timestamp"])), - organization_id=project.organization_id, - project_id=project.id, - received=( - Timestamp(seconds=int(event_data["received"])) if "received" in event_data else None - ), - retention_days=event_data.get("retention_days", 90), - attributes=encode_attributes( - event, event_data, ignore_fields={"event_id", "timestamp", "tags"} - ), - ) - - -def _encode_value(value: Any) -> AnyValue: - if isinstance(value, str): - return AnyValue(string_value=value) - elif isinstance(value, bool): - # Note: bool check must come before int check since bool is a subclass of int - return AnyValue(bool_value=value) - elif isinstance(value, int): - # int_value is a signed int64, so it has a range of valid values. - # if value doesn't fit into an int64, cast it to string. - if abs(value) >= (2**63): - return AnyValue(string_value=str(value)) - return AnyValue(int_value=value) - elif isinstance(value, float): - return AnyValue(double_value=value) - elif isinstance(value, list) or isinstance(value, tuple): - # Not yet processed on EAP side - return AnyValue( - array_value=ArrayValue(values=[_encode_value(v) for v in value if v is not None]) - ) - elif isinstance(value, dict): - # Not yet processed on EAP side - return AnyValue( - kvlist_value=KeyValueList( - values=[ - KeyValue(key=str(kv[0]), value=_encode_value(kv[1])) - for kv in value.items() - if kv[1] is not None - ] - ) - ) - else: - raise NotImplementedError(f"encode not supported for {type(value)}") - - -def encode_attributes( - event: Event | GroupEvent, event_data: Mapping[str, Any], ignore_fields: set[str] | None = None -) -> Mapping[str, AnyValue]: - attributes = {} - ignore_fields = ignore_fields or set() - - for key, value in event_data.items(): - if key in ignore_fields: - continue - if value is None: - continue - attributes[key] = _encode_value(value) - - if event.group_id: - attributes["group_id"] = AnyValue(int_value=event.group_id) - - format_tag_key = lambda key: f"tags[{key}]" - - tag_keys = set() - tags = event_data.get("tags") - if tags is not None: - for tag in tags: - if tag is None: - continue - key, value = tag - if value is None: - continue - formatted_key = format_tag_key(key) - attributes[formatted_key] = _encode_value(value) - tag_keys.add(formatted_key) - - attributes["tag_keys"] = _encode_value(sorted(tag_keys)) - - return attributes diff --git a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py b/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py deleted file mode 100644 index 72bcdfb63..000000000 --- a/packages/evals/fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py +++ /dev/null @@ -1,1668 +0,0 @@ -import logging -from collections.abc import Callable, Sequence -from dataclasses import dataclass, field -from datetime import datetime -from re import Match -from typing import Any, Literal, cast - -from parsimonious.exceptions import ParseError -from sentry_protos.snuba.v1.attribute_conditional_aggregation_pb2 import ( - AttributeConditionalAggregation, -) -from sentry_protos.snuba.v1.endpoint_trace_item_table_pb2 import ( - AggregationAndFilter, - AggregationComparisonFilter, - AggregationFilter, - AggregationOrFilter, - Column, -) -from sentry_protos.snuba.v1.formula_pb2 import Literal as LiteralValue -from sentry_protos.snuba.v1.request_common_pb2 import RequestMeta, TraceItemType -from sentry_protos.snuba.v1.trace_item_attribute_pb2 import ( - AttributeAggregation, - AttributeKey, - AttributeValue, - DoubleArray, - IntArray, - StrArray, - VirtualColumnContext, -) -from sentry_protos.snuba.v1.trace_item_filter_pb2 import ( - AndFilter, - ComparisonFilter, - ExistsFilter, - NotFilter, - OrFilter, - TraceItemFilter, -) -from sentry_sdk import traces - -from sentry.api import event_search -from sentry.discover import arithmetic -from sentry.exceptions import InvalidSearchQuery -from sentry.explore.models import ExploreSavedFormula -from sentry.models.group import Group, parse_short_id -from sentry.models.project import Project -from sentry.search.eap import constants -from sentry.search.eap.columns import ( - AggregateDefinition, - AttributeArgumentDefinition, - ColumnDefinitions, - FormulaDefinition, - ResolvedArgument, - ResolvedArguments, - ResolvedAttribute, - ResolvedColumn, - ResolvedEquation, - ResolvedFunction, - ResolvedLiteral, - ValueArgumentDefinition, - VirtualColumnDefinition, -) -from sentry.search.eap.rpc_utils import and_trace_item_filters -from sentry.search.eap.sampling import validate_sampling -from sentry.search.eap.spans.attributes import SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS -from sentry.search.eap.types import EAPResponse, SearchResolverConfig, SupportedTraceItemType -from sentry.search.events import constants as qb_constants -from sentry.search.events import fields -from sentry.search.events import filter as event_filter -from sentry.search.events.filter import to_list -from sentry.search.events.types import SAMPLING_MODES, SnubaParams -from sentry.search.exceptions import InvalidIssueSearchQuery - -logger = logging.getLogger(__name__) - - -def collect_issue_short_ids_from_parsed_terms(terms: Sequence[object]) -> set[str]: - """Collect non-empty issue filter values from a parsed search tree (including OR/AND).""" - out: set[str] = set() - for term in terms: - if isinstance(term, event_search.SearchFilter): - if term.key.name == "issue": - for v in to_list(term.value.value): - if v: - out.add(str(v)) - elif isinstance(term, event_search.ParenExpression): - out |= collect_issue_short_ids_from_parsed_terms(term.children) - return out - - -class HiddenApiAttribute(InvalidSearchQuery): - pass - - -@dataclass(frozen=True) -class SearchResolver: - """The only attributes are things we want to cache and params - - Please do not store any state on the SearchResolver - """ - - params: SnubaParams - config: SearchResolverConfig - definitions: ColumnDefinitions - granularity_secs: int | None = None - _query_result_cache: dict[str, EAPResponse] = field(default_factory=dict) - _resolved_attribute_cache: dict[ - str, tuple[ResolvedAttribute, list[VirtualColumnDefinition | None]] - ] = field(default_factory=dict) - _resolved_function_cache: dict[ - str, - tuple[ - ResolvedFunction, - list[VirtualColumnDefinition | None], - ], - ] = field(default_factory=dict) - qualified_short_id_to_group_id_cache: dict[int, dict[str, int]] = field(default_factory=dict) - _internal_name_to_column: dict[str, ResolvedAttribute] = field(default_factory=dict, repr=False) - - def _find_column_by_internal_name(self, internal_name: str) -> ResolvedAttribute | None: - """Look up a column definition by its internal name (e.g. 'sentry.item_id' -> 'id' column). - - Uses a lazily-built reverse mapping from internal_name -> column definition, - cached for the lifetime of this resolver instance. - """ - if not self._internal_name_to_column: - self._internal_name_to_column.update( - { - col.internal_name: col - for col in self.definitions.columns.values() - if not col.secondary_alias - } - ) - return self._internal_name_to_column.get(internal_name) - - def get_function_definition( - self, function_name: str - ) -> FormulaDefinition | AggregateDefinition: - if function_name in self.definitions.aggregates: - return self.definitions.aggregates[function_name] - elif function_name in self.definitions.formulas: - return self.definitions.formulas[function_name] - else: - raise InvalidSearchQuery(f"Unknown function {function_name}") - - @traces.trace - def resolve_meta( - self, - referrer: str, - sampling_mode: SAMPLING_MODES | None = None, - filter_project: Callable[[Project], bool] | None = None, - ) -> RequestMeta: - if self.params.organization_id is None: - raise Exception("An organization is required to resolve queries") - span = traces.get_current_span() - if span: - span.set_attribute("SearchResolver.params", repr(self.params)) - - projects = self.params.projects - - # If a filter is specified, use it to narrow down the list - # of projects to query on. - if filter_project: - projects = [project for project in projects if filter_project(project)] - - # if filtering removed all projects, we reset to all - # selected project again to prevent potential snuba errors - if not projects: - projects = self.params.projects - - return RequestMeta( - organization_id=self.params.organization_id, - referrer=referrer, - project_ids=[project.id for project in projects], - start_timestamp=self.params.rpc_start_date, - end_timestamp=self.params.rpc_end_date, - trace_item_type=self.definitions.trace_item_type, - downsampled_storage_config=validate_sampling(sampling_mode), - ) - - @traces.trace - def resolve_query( - self, querystring: str | None - ) -> tuple[ - TraceItemFilter | None, - AggregationFilter | None, - list[VirtualColumnDefinition | None], - ]: - """Given a query string in the public search syntax eg. `span.description:foo` construct the TraceItemFilter - - This is the public interface to resolver the query, for the logic see __resolve_query, this is because we - also append the environment before returning the final TraceItemFilter""" - environment_query = self.__resolve_environment_query() - where, having, contexts = self.__resolve_query(querystring) - span = traces.get_current_span() - if span and querystring is not None: - span.set_attribute("SearchResolver.query_string", querystring) - if span and where is not None: - span.set_attribute("SearchResolver.resolved_query", repr(where)) - if span and environment_query is not None: - span.set_attribute("SearchResolver.environment_query", repr(environment_query)) - - where = and_trace_item_filters( - where, - # The RPC request meta does not contain the environment. - # So we have to inject it as a query condition. - environment_query, - ) - - return where, having, contexts - - @traces.trace - def resolve_query_with_columns( - self, - querystring: str | None, - selected_columns: list[str] | None, - equations: list[str] | None, - ) -> tuple[ - TraceItemFilter | None, - AggregationFilter | None, - list[VirtualColumnDefinition | None], - ]: - where, having, contexts = self.resolve_query(querystring) - - # Some datasets like trace metrics require we inject additional - # conditions in the top level. - dataset_conditions = self.resolve_dataset_conditions(selected_columns, equations) - where = and_trace_item_filters(where, dataset_conditions) - - return where, having, contexts - - def __resolve_environment_query(self) -> TraceItemFilter | None: - resolved_column, _ = self.resolve_column("environment") - if not isinstance(resolved_column.proto_definition, AttributeKey): - return None - - envs = [env.name for env in self.params.environments if env is not None] - - if not envs: - return None - - return TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=ComparisonFilter.OP_IN, - value=AttributeValue(val_str_array=StrArray(values=envs)), - ) - ) - - def _init_issue_short_id_cache( - self, - parsed_terms: Sequence[object], - ) -> None: - """One bulk Group lookup for issue short ids in the query; store maps on this resolver.""" - if "issue" not in self.definitions.filter_aliases or self.params.organization_id is None: - return - - collected = collect_issue_short_ids_from_parsed_terms(parsed_terms) - if not collected: - return - try: - groups = list( - Group.objects.by_qualified_short_id_bulk( - organization_id=self.params.organization_id, - short_ids_raw=list(collected), - # org-wide: the Snuba query is already scoped to the requested projects. - project_ids=None, - ) - ) - except Group.DoesNotExist: - raise InvalidIssueSearchQuery(sorted(collected)) - - idx = {(g.project.slug.lower(), g.short_id): g for g in groups} - - for raw in collected: - parsed = parse_short_id(raw) - if parsed is None: - raise InvalidIssueSearchQuery(sorted(collected)) - g = idx.get((parsed.project_slug, parsed.short_id)) - if g is None: - raise InvalidIssueSearchQuery(sorted(collected)) - if g.project.id not in self.qualified_short_id_to_group_id_cache: - self.qualified_short_id_to_group_id_cache[g.project.id] = {} - self.qualified_short_id_to_group_id_cache[g.project.id][raw] = g.id - - def parse_search_query(self, querystring: str) -> Sequence[event_search.QueryToken]: - """Helper function so this can be called by validate separately""" - return event_search.parse_search_query( - querystring, - config=event_search.SearchConfig.create_from( - event_search.default_config, - wildcard_free_text=True, - allow_regex=self._allows_regex(), - ), - params=self.params.filter_params, - get_field_type=self.get_field_type, - get_function_result_type=self.get_field_type, - ) - - def _allows_regex(self) -> bool: - return self.definitions.trace_item_type == TraceItemType.TRACE_ITEM_TYPE_LOG - - def collect_terms(self, parsed_terms: Sequence[event_search.QueryToken]) -> list[str]: - """Helper function to collect all the search terms from a parsed query ignoring the actual query tree""" - terms = [] - for term in parsed_terms: - if event_search.SearchBoolean.is_operator(term): - continue - elif isinstance(term, event_search.ParenExpression): - for collected_term in self.collect_terms(term.children): - if collected_term not in terms: - terms.append(collected_term) - else: - if isinstance(term, event_search.SearchFilter): - for converted_term in self.convert_term(term): - column = converted_term.key.name - if column not in terms: - terms.append(column) - else: - column = term.key.name - if column not in terms: - terms.append(column) - return terms - - def __resolve_query( - self, querystring: str | None - ) -> tuple[ - TraceItemFilter | None, - AggregationFilter | None, - list[VirtualColumnDefinition | None], - ]: - if querystring is None: - return None, None, [] - try: - parsed_terms = self.parse_search_query(querystring) - except ParseError as e: - if e.expr is not None: - raise InvalidSearchQuery(f"Parse error: {e.expr.name} (column {e.column():d})") - else: - raise InvalidSearchQuery(f"Parse error for: {querystring}") - - # If occurrences dataset, cache group_id to issues mapping. - self._init_issue_short_id_cache(parsed_terms) - - has_boolean = False - for term in parsed_terms: - if (isinstance(term, event_search.ParenExpression)) or ( - event_search.SearchBoolean.is_operator(term) and len(parsed_terms) > 1 - ): - has_boolean = True - break - if has_boolean: - return self._resolve_boolean_conditions(parsed_terms) - else: - return self._resolve_terms(parsed_terms) - - def _resolve_boolean_conditions( - self, terms: event_filter.ParsedTerms - ) -> tuple[ - TraceItemFilter | None, - AggregationFilter | None, - list[VirtualColumnDefinition | None], - ]: - if len(terms) == 0: - return None, None, [] - elif len(terms) == 1: - if isinstance(terms[0], event_search.ParenExpression): - return self._resolve_boolean_conditions(terms[0].children) - elif isinstance(terms[0], event_search.SearchFilter): - return self._resolve_terms([terms[0]]) - elif isinstance(terms[0], event_search.AggregateFilter): - return self._resolve_terms([terms[0]]) - else: - return self._resolve_terms(terms) - - # Filter out any ANDs since we can assume anything without an OR is an AND. Also do some - # basic sanitization of the query: can't have two operators next to each other, and can't - # start or end a query with an operator. - previous_term: event_filter.ParsedTerm | None = None - new_terms = [] - term: event_filter.ParsedTerm | None = None - for term in terms: - if previous_term: - if event_search.SearchBoolean.is_operator( - previous_term - ) and event_search.SearchBoolean.is_operator(term): - raise InvalidSearchQuery( - f"Missing condition in between two condition operators: '{previous_term} {term}'" - ) - else: - if event_search.SearchBoolean.is_operator(term): - raise InvalidSearchQuery( - f"Condition is missing on the left side of '{term}' operator" - ) - - if term != event_search.SearchBoolean.BOOLEAN_AND: - new_terms.append(term) - - previous_term = term - - if term is not None and event_search.SearchBoolean.is_operator(term): - raise InvalidSearchQuery(f"Condition is missing on the right side of '{term}' operator") - terms = new_terms - - # We put precedence on AND, which sort of counter-intuitively means we have to split the query - # on ORs first, so the ANDs are grouped together. Search through the query for ORs and split the - # query on each OR. - # We want to maintain a binary tree, so split the terms on the first OR we can find and recurse on - # the two sides. If there is no OR, split the first element out to AND - index = None - lhs, rhs = None, None - try: - index = terms.index(event_search.SearchBoolean.BOOLEAN_OR) - lhs, rhs = terms[:index], terms[index + 1 :] - operator: Literal["and" | "or"] = "or" - except Exception: - lhs, rhs = terms[:1], terms[1:] - operator = "and" - - where_lhs, having_lhs, contexts_lhs = self._resolve_boolean_conditions(lhs) - where_rhs, having_rhs, contexts_rhs = self._resolve_boolean_conditions(rhs) - contexts = contexts_lhs + contexts_rhs - - where = None - having = None - - if where_lhs is not None and where_rhs is not None: - if operator == "and": - where = TraceItemFilter(and_filter=AndFilter(filters=[where_lhs, where_rhs])) - else: - where = TraceItemFilter(or_filter=OrFilter(filters=[where_lhs, where_rhs])) - elif where_lhs is None and where_rhs is not None: - where = where_rhs - elif where_lhs is not None and where_rhs is None: - where = where_lhs - - if having_lhs is not None and having_rhs is not None: - if operator == "and": - having = AggregationFilter( - and_filter=AggregationAndFilter(filters=[having_lhs, having_rhs]) - ) - else: - having = AggregationFilter( - or_filter=AggregationOrFilter(filters=[having_lhs, having_rhs]) - ) - elif having_lhs is None and having_rhs is not None: - having = having_rhs - elif having_lhs is not None and having_rhs is None: - having = having_lhs - - return where, having, contexts - - def _resolve_terms( - self, terms: event_filter.ParsedTerms - ) -> tuple[ - TraceItemFilter | None, - AggregationFilter | None, - list[VirtualColumnDefinition | None], - ]: - where, where_contexts = self._resolve_where(terms) - having, having_contexts = self._resolve_having(terms) - return where, having, where_contexts + having_contexts - - def _resolve_where( - self, terms: event_filter.ParsedTerms - ) -> tuple[TraceItemFilter | None, list[VirtualColumnDefinition | None]]: - parsed_terms = [] - resolved_contexts = [] - for item in terms: - if isinstance(item, event_search.SearchFilter): - resolved_term, resolved_context = self.resolve_term(item) - parsed_terms.extend(resolved_term) - resolved_contexts.extend(resolved_context) - - if len(parsed_terms) > 1: - return TraceItemFilter(and_filter=AndFilter(filters=parsed_terms)), resolved_contexts - elif len(parsed_terms) == 1: - return parsed_terms[0], resolved_contexts - return None, [] - - def _resolve_having( - self, terms: event_filter.ParsedTerms - ) -> tuple[AggregationFilter | None, list[VirtualColumnDefinition | None]]: - if not self.config.use_aggregate_conditions: - return None, [] - - parsed_terms = [] - resolved_contexts = [] - for item in terms: - if isinstance(item, event_search.AggregateFilter): - resolved_term, resolved_context = self.resolve_aggregate_term(item) - parsed_terms.append(resolved_term) - resolved_contexts.extend(resolved_context) - - if len(parsed_terms) > 1: - return ( - AggregationFilter(and_filter=AggregationAndFilter(filters=parsed_terms)), - resolved_contexts, - ) - elif len(parsed_terms) == 1: - return parsed_terms[0], resolved_contexts - return None, [] - - def resolve_virtual_context_term( - self, - term: str, - raw_value: str | list[str], - resolved_column: ResolvedAttribute, - context: VirtualColumnDefinition, - ) -> list[str] | str: - # Convert the term to the expected values - final_raw_value: str | list[str] = [] - resolved_context = context.constructor(self.params, self) - reversed_context = {v: k for k, v in resolved_context.value_map.items()} - if isinstance(raw_value, list): - new_value = [] - for raw_iterable in raw_value: - if context.default_value and context.default_value == raw_iterable: - # Avoiding this for now, while this could work with the Unknown:"" mapping - # But that won't work once we use the VirtualColumnContext.default_value - raise InvalidSearchQuery( - f"Using {raw_iterable} in an IN filter is not currently supported" - ) - elif raw_iterable not in reversed_context: - valid_values = list(reversed_context.keys())[:5] - if len(valid_values) > 5: - valid_values.append("...") - raise InvalidSearchQuery( - constants.REVERSE_CONTEXT_ERROR.format( - raw_value, term, ", ".join(valid_values) - ) - ) - else: - new_value.append(reversed_context[raw_iterable]) - final_raw_value = new_value - elif raw_value in reversed_context: - final_raw_value = reversed_context[raw_value] - elif context.default_value and context.default_value == raw_value: - # Avoiding this for now, while this could work with the Unknown:"" mapping - # But that won't work once we use the VirtualColumnContext.default_value - raise InvalidSearchQuery(f"Using {raw_value} is not currently supported") - else: - valid_values = list(reversed_context.keys())[:5] - if len(valid_values) > 5: - valid_values.append("...") - raise InvalidSearchQuery( - constants.REVERSE_CONTEXT_ERROR.format( - raw_value, term, ", ".join(list(reversed_context.keys())[:5]) - ) - ) - return final_raw_value - - def convert_term(self, term: event_search.SearchFilter) -> list[event_search.SearchFilter]: - name = term.key.name - - converter = self.definitions.filter_aliases.get(name) - if converter is not None: - if term.value.is_regex: - # The converters resolve values against Sentry models, so they would treat the - # pattern as a literal rather than matching against it - raise InvalidSearchQuery(f"Cannot use regular expressions with {name}") - return converter(self.params, term, self) - - return [term] - - def resolve_term( - self, term: event_search.SearchFilter - ) -> tuple[list[TraceItemFilter], list[VirtualColumnDefinition | None]]: - terms = self.convert_term(term) - - resolved_terms = [] - resolved_contexts = [] - - for t in terms: - resolved_term, resolved_context = self._resolve_term(t) - resolved_terms.append(resolved_term) - resolved_contexts.append(resolved_context) - - return resolved_terms, resolved_contexts - - def _resolve_term( - self, term: event_search.SearchFilter - ) -> tuple[TraceItemFilter, VirtualColumnDefinition | None]: - resolved_column, context_definitions = self.resolve_column(term.key.name) - non_none_definitions = [defn for defn in context_definitions if defn is not None] - if len(non_none_definitions) > 1: - raise ValueError(f"{term.key.name} resolved with multiple contexts") - elif len(non_none_definitions) == 1: - context_definition = non_none_definitions[0] - else: - context_definition = None - - self._raise_if_hidden_api_attribute(term.key.name, resolved_column) - - if term.value.is_regex: - return self._resolve_regex_term(term, resolved_column, context_definition), None - - if context_definition is not None and term.value.is_wildcard(): - raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}") - - value = term.value.value - if self.params.is_timeseries_request and context_definition is not None: - resolved_column, value = self.map_search_term_context_to_original_column( - term, context_definition - ) - self._raise_if_hidden_api_attribute(term.key.name, resolved_column) - context_definition = None - - if not isinstance(resolved_column.proto_definition, AttributeKey): - raise ValueError(f"{term.key.name} is not valid search term") - - if term.value.is_wildcard(): - is_list = False - if term.operator == "=": - operator = ComparisonFilter.OP_LIKE - elif term.operator == "!=": - operator = ComparisonFilter.OP_NOT_LIKE - elif term.operator == "IN": - operator = ComparisonFilter.OP_LIKE - is_list = True - elif term.operator == "NOT IN": - operator = ComparisonFilter.OP_NOT_LIKE - is_list = True - else: - raise InvalidSearchQuery(f"Cannot use operator: {term.operator} with wildcards") - - if is_list: - raw_value = cast(list[str], term.value.raw_value) - filters = [ - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=self._resolve_search_value( - resolved_column, - ( - "=" if operator == ComparisonFilter.OP_LIKE else "!=" - ), # tell this function the single operator since its being ORed - event_search.translate_wildcard_as_clickhouse_pattern(str(value)), - ), - ) - ) - for value in raw_value - ] - return ( - ( - TraceItemFilter(or_filter=OrFilter(filters=filters)) - if term.operator == "IN" - else TraceItemFilter(and_filter=AndFilter(filters=filters)) - ), - context_definition, - ) - else: - value = str(term.value.raw_value) - value = event_search.translate_wildcard_as_clickhouse_pattern(value) - elif term.operator in constants.OPERATOR_MAP: - operator = constants.OPERATOR_MAP[term.operator] - else: - raise InvalidSearchQuery(f"Unknown operator: {term.operator}") - - if value is None: - exists_filter = TraceItemFilter( - exists_filter=ExistsFilter( - key=resolved_column.proto_definition, - ) - ) - if term.operator == "=": - not_exists_filter = TraceItemFilter(not_filter=NotFilter(filters=[exists_filter])) - return not_exists_filter, context_definition - elif term.operator == "!=": - return exists_filter, context_definition - else: - raise InvalidSearchQuery(f"Unsupported operator for None {term.operator}") - - # Handle lists containing None for IN/NOT IN operators - if isinstance(value, list) and None in value and term.operator in constants.IN_OPERATORS: - non_none_values = [v for v in value if v is not None] - - exists_filter = TraceItemFilter( - exists_filter=ExistsFilter( - key=resolved_column.proto_definition, - ) - ) - - if term.operator == "IN": - filters = [] - filters.append(TraceItemFilter(not_filter=NotFilter(filters=[exists_filter]))) - - if non_none_values: - filters.append( - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=self._resolve_search_value( - resolved_column, term.operator, non_none_values - ), - ignore_case=self.params.case_insensitive - and resolved_column.search_type == "string", - ) - ) - ) - - return ( - TraceItemFilter(or_filter=OrFilter(filters=filters)), - context_definition, - ) - else: # NOT IN - filters = [exists_filter] - - if non_none_values: - filters.append( - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=self._resolve_search_value( - resolved_column, term.operator, non_none_values - ), - ignore_case=self.params.case_insensitive - and resolved_column.search_type == "string", - ) - ) - ) - - return ( - TraceItemFilter(and_filter=AndFilter(filters=filters)), - context_definition, - ) - - if value == "" and context_definition is None: - exists_filter = TraceItemFilter( - exists_filter=ExistsFilter( - key=resolved_column.proto_definition, - ) - ) - # Attributes that reject the empty string can never hold it, and comparing against it - # fails in snuba for the ones backed by a typed column (eg. trace id parsed as a UUID). - compare_to_empty_string = ( - resolved_column.proto_definition.type == constants.STRING - and resolved_column.allows_value(value) - ) - if term.operator == "!=": - filters = [exists_filter] - if compare_to_empty_string: - filters.append( - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=AttributeValue(val_str=value), - ) - ) - ) - return ( - TraceItemFilter(and_filter=AndFilter(filters=filters)), - context_definition, - ) - elif term.operator == "=": - filters = [TraceItemFilter(not_filter=NotFilter(filters=[exists_filter]))] - if compare_to_empty_string: - filters.append( - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=AttributeValue(val_str=value), - ) - ) - ) - return ( - TraceItemFilter(or_filter=OrFilter(filters=filters)), - context_definition, - ) - else: - raise InvalidSearchQuery(f"Unsupported operator for empty strings {term.operator}") - - if not self.params.is_timeseries_request and context_definition: - value = self.remap_value_using_context_definition(context_definition, value) - - return ( - TraceItemFilter( - comparison_filter=ComparisonFilter( - key=resolved_column.proto_definition, - op=operator, - value=self._resolve_search_value(resolved_column, term.operator, value), - ignore_case=self.params.case_insensitive - and resolved_column.search_type == "string", - ) - ), - context_definition, - ) - - def map_context_to_original_column( - self, - context_definition: VirtualColumnDefinition, - ) -> ResolvedAttribute: - """ - Time series request do not support virtual column contexts, so we have to remap the value back to the original column. - (see https://github.com/getsentry/eap-planning/issues/236) - """ - context = context_definition.constructor(self.params, self) - - is_number_column = ( - context.from_column_name in SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"] - ) - - public_alias = ( - SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"].get(context.from_column_name) - if is_number_column - else context.from_column_name - ) - - if public_alias is None: - raise InvalidSearchQuery(f"Cannot map {context.from_column_name} to a public alias") - - resolved_column, _ = self.resolve_column(public_alias) - - if not isinstance(resolved_column.proto_definition, AttributeKey): - raise ValueError(f"{resolved_column.public_alias} is not valid search term") - - self._raise_if_hidden_api_attribute(context.to_column_name, resolved_column) - - return resolved_column - - def map_search_term_context_to_original_column( - self, - term: event_search.SearchFilter, - context_definition: VirtualColumnDefinition, - ) -> tuple[ResolvedAttribute, str | int | list[str]]: - """ - Time series request do not support virtual column contexts, so we have to remap the value back to the original column. - (see https://github.com/getsentry/eap-planning/issues/236) - """ - context = context_definition.constructor(self.params, self) - is_number_column = ( - context.from_column_name in SPANS_INTERNAL_TO_PUBLIC_ALIAS_MAPPINGS["number"] - ) - - resolved_column = self.map_context_to_original_column(context_definition) - - value = term.value.value - - inverse_value_map: dict[str, list[str]] = {} - for key, val in context.value_map.items(): - inverse_value_map[val] = inverse_value_map.get(val, []) + [key] - - def remap_value(old_value: str) -> list[str]: - if old_value in inverse_value_map: - return inverse_value_map[old_value] - elif old_value in context.value_map: - return [old_value] - elif context.default_value: - return [context.default_value] - else: - raise InvalidSearchQuery(f"Unknown value {old_value}") - - final_value: list[str] = [] - if isinstance(value, list): - value_set: set[str] = set() - for v in value: - for mapped_values in remap_value(v): - value_set.add(mapped_values) - - final_value = list(value_set) - - else: - final_value = remap_value(value) - - if len(final_value) == 1 and not isinstance(value, list): - if is_number_column: - return resolved_column, int(final_value[0]) - return resolved_column, final_value[0] - - return resolved_column, final_value - - def resolve_aggregate_term( - self, term: event_search.AggregateFilter - ) -> tuple[AggregationFilter, list[VirtualColumnDefinition | None]]: - resolved_column, context = self.resolve_column(term.key.name) - self._raise_if_hidden_api_attribute(term.key.name, resolved_column) - proto_definition = resolved_column.proto_definition - - if not isinstance( - proto_definition, - (AttributeAggregation, AttributeConditionalAggregation, Column.BinaryFormula), - ): - raise ValueError(f"{term.key.name} is not valid search term") - - # TODO: Handle different units properly - value = term.value.value - - if term.operator in constants.OPERATOR_MAP: - operator = constants.AGGREGATION_OPERATOR_MAP[term.operator] - else: - raise InvalidSearchQuery(f"Unknown operator: {term.operator}") - - kwargs = {"op": operator, "val": value} - if isinstance(proto_definition, AttributeAggregation): - aggregation_key = "aggregation" - elif isinstance(proto_definition, AttributeConditionalAggregation): - aggregation_key = "conditional_aggregation" - elif isinstance(proto_definition, Column.BinaryFormula): - aggregation_key = "formula" - else: - raise InvalidSearchQuery(f"{term.key.name} is not a valid search") - - kwargs[aggregation_key] = proto_definition - return ( - AggregationFilter( - comparison_filter=AggregationComparisonFilter(**kwargs), - ), - context, - ) - - def _resolve_regex_term( - self, - term: event_search.SearchFilter, - resolved_column: ResolvedAttribute | ResolvedFunction, - context_definition: VirtualColumnDefinition | None, - ) -> TraceItemFilter: - if context_definition is not None: - raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") - - key = resolved_column.proto_definition - if not isinstance(key, AttributeKey) or key.type not in constants.REGEXP_ATTRIBUTE_TYPES: - raise InvalidSearchQuery( - f"Cannot use regular expressions with {term.key.name}, it is not a string attribute" - ) - - # Snuba's `ignore_case` lowercases the pattern along with the value, rewriting `[A-Z]` - # and inverting escapes like `\D`. RE2's inline flag leaves the pattern intact. - prefix = "(?i)" if self.params.case_insensitive else "" - match = TraceItemFilter( - comparison_filter=ComparisonFilter( - key=key, - op=ComparisonFilter.OP_REGEXP, - value=AttributeValue(val_str=f"{prefix}{term.value.raw_value}"), - ) - ) - - if term.operator == "!=": - # There is no OP_NOT_REGEXP, so negation is expressed by wrapping the match - return TraceItemFilter(not_filter=NotFilter(filters=[match])) - return match - - def _resolve_search_value( - self, - column: ResolvedAttribute, - operator: str, - value: str | float | datetime | Sequence[float] | Sequence[str], - ) -> AttributeValue: - column.validate(value) - value = column.normalize(value) - if isinstance(column.proto_definition, AttributeKey): - column_type = column.proto_definition.type - if column_type == constants.STRING: - if operator in constants.IN_OPERATORS: - if isinstance(value, list) and all(isinstance(item, str) for item in value): - return AttributeValue(val_str_array=StrArray(values=value)) - else: - raise InvalidSearchQuery( - f"{value} is not a valid value for doing an IN filter" - ) - else: - return AttributeValue(val_str=str(value)) - elif column_type == constants.INT: - # The search parser will always convert a value to a float, so we need to cast back to an int - if operator in constants.IN_OPERATORS: - if isinstance(value, list): - return AttributeValue( - val_int_array=IntArray(values=[int(val) for val in value]) - ) - else: - raise InvalidSearchQuery( - f"{value} is not a valid value for doing an IN filter" - ) - elif isinstance(value, (float, int)): - return AttributeValue(val_int=int(value)) - elif column_type == constants.DOUBLE: - if operator in constants.IN_OPERATORS: - if isinstance(value, list): - return AttributeValue( - val_double_array=DoubleArray( - values=[ - val.timestamp() if isinstance(val, datetime) else val - for val in value - ] - ) - ) - else: - raise InvalidSearchQuery( - f"{value} is not a valid value for doing an IN filter" - ) - elif isinstance(value, datetime): - return AttributeValue(val_double=value.timestamp()) - elif isinstance(value, float): - return AttributeValue(val_double=value) - elif column_type == constants.BOOLEAN: - if operator in constants.IN_OPERATORS: - raise InvalidSearchQuery( - f"{column.public_alias} cannot be used with an IN filter" - ) - elif isinstance(value, str): - lowered_value = value.lower() - if lowered_value not in constants.BOOLEAN_VALUES: - raise InvalidSearchQuery( - f"{value} is not a valid boolean value, expecting true or false" - ) - bool_value = lowered_value in constants.TRUTHY_VALUES - return AttributeValue(val_bool=bool_value) - elif isinstance(value, bool): - return AttributeValue(val_bool=value) - elif column_type == constants.ARRAY: - # Only scalar value membership in an array is allowed. - # Allowed operators: =,!=, LIKE, NOT_LIKE. - # TODO: Add support for scalar: >, < for numbers - if operator in constants.IN_OPERATORS: - raise InvalidSearchQuery( - f"{column.public_alias} (array) cannot be used with an IN filter; " - f"use {column.public_alias}[*]:value for membership" - ) - # All primitive types are converted to strings on EAP before comparison. - return AttributeValue(val_str=str(value)) - raise InvalidSearchQuery( - f"{value} is not a valid filter value for {column.public_alias}, expecting {constants.TYPE_TO_STRING_MAP[column_type]}, but got a {type(value)}" - ) - else: - raise NotImplementedError("Aggregate Queries not implemented yet") - - def resolve_contexts( - self, context_definitions: list[VirtualColumnDefinition | None] - ) -> list[VirtualColumnContext]: - """Given a list of contexts that may have None in them, remove the Nones and remove the dupes""" - final_contexts = [] - existing_target_columns = set() - for context_definition in context_definitions: - if context_definition is None: - continue - context = context_definition.constructor(self.params, self) - if context is None or context.to_column_name in existing_target_columns: - continue - else: - existing_target_columns.add(context.to_column_name) - final_contexts.append(context) - return final_contexts - - @traces.trace - def resolve_columns( - self, selected_columns: list[str], has_aggregates: bool = False - ) -> tuple[ - list[ResolvedAttribute | ResolvedFunction], - list[VirtualColumnDefinition | None], - ]: - """Given a list of columns resolve them and get their context if applicable - - This function will also dedupe the virtual column contexts if necessary - """ - span = traces.get_current_span() - resolved_columns = [] - resolved_contexts = [] - stripped_columns = [column.strip() for column in selected_columns] - if span: - span.set_attribute("SearchResolver.selected_columns", stripped_columns) - for column in stripped_columns: - match = fields.is_function(column) - has_aggregates = has_aggregates or match is not None - try: - resolved_column, context = self.resolve_column(column, match) - except HiddenApiAttribute: - continue - if isinstance(resolved_column, ResolvedAttribute) and self._should_hide_api_attribute( - column, resolved_column - ): - continue - if ( - self.config.disable_array_attributes - and isinstance(resolved_column, ResolvedAttribute) - and resolved_column.internal_type == constants.ARRAY - ): - continue - resolved_columns.append(resolved_column) - resolved_contexts.extend(context) - - if self.config.auto_fields: - # Ensure fields we require to build a functioning interface are present. - if not has_aggregates and "id" not in stripped_columns: - id_column, id_context = self.resolve_column("id") - resolved_columns.append(id_column) - resolved_contexts.extend(id_context) - stripped_columns.append("id") - if "id" in stripped_columns and "project.id" not in stripped_columns: - project_column, project_context = self.resolve_column("project.name") - resolved_columns.append(project_column) - resolved_contexts.extend(project_context) - - return resolved_columns, resolved_contexts - - def resolve_column( - self, - column: str, - match: Match[str] | None = None, - public_alias_override: str | None = None, - default_value: float | None = None, - ) -> tuple[ - ResolvedAttribute | ResolvedFunction, - list[VirtualColumnDefinition | None], - ]: - """Column is either an attribute or an aggregate, this function will determine which it is and call the relevant - resolve function""" - match = fields.is_function(column) - if match: - return self.resolve_function( - column, match, public_alias_override, default_value=default_value - ) - else: - return self.resolve_attribute(column, public_alias_override) - - def get_field_type(self, column: str) -> str: - resolved_column, _ = self.resolve_column(column) - return resolved_column.search_type - - @traces.trace - def resolve_attributes( - self, columns: list[str] - ) -> tuple[list[ResolvedAttribute], list[VirtualColumnDefinition | None]]: - """Helper function to resolve a list of attributes instead of 1 attribute at a time""" - resolved_columns = [] - resolved_contexts = [] - for column in columns: - col, context = self.resolve_attribute(column) - self._raise_if_hidden_api_attribute(column, col) - if self.config.disable_array_attributes and col.internal_type == constants.ARRAY: - continue - resolved_columns.append(col) - resolved_contexts.extend(context) - return resolved_columns, resolved_contexts - - def should_hide_api_column( - self, column: str, resolved_column: ResolvedAttribute | ResolvedFunction - ) -> bool: - if not isinstance(resolved_column, ResolvedAttribute): - return False - return self._should_hide_api_attribute(column, resolved_column) - - def _should_hide_api_attribute( - self, column: str, resolved_attribute: ResolvedAttribute - ) -> bool: - if self.config.api_attribute_visibility_item_type is None: - return False - - from sentry.search.eap.utils import can_expose_attribute_to_api - - item_type = SupportedTraceItemType(self.config.api_attribute_visibility_item_type) - if column in self.definitions.contexts and resolved_attribute.internal_name != column: - visibility_attribute = resolved_attribute.internal_name - elif column in self.definitions.contexts or column in self.definitions.columns: - visibility_attribute = column - else: - visibility_attribute = resolved_attribute.internal_name - return not can_expose_attribute_to_api( - visibility_attribute, - item_type, - include_internal=self.config.api_attribute_visibility_include_internal, - include_internal_convention_attributes=self.config.api_attribute_visibility_include_internal_convention_attributes, - ) - - def _raise_if_hidden_api_attribute( - self, column: str, resolved_column: ResolvedAttribute | ResolvedFunction - ) -> None: - if isinstance(resolved_column, ResolvedAttribute) and self._should_hide_api_attribute( - column, resolved_column - ): - raise HiddenApiAttribute(f"Could not parse {column}") - - def resolve_attribute( - self, column: str, public_alias_override: str | None = None - ) -> tuple[ResolvedAttribute, list[VirtualColumnDefinition | None]]: - """Attributes are columns that aren't 'functions' or 'aggregates', usually this means string or numeric - attributes (aka. tags), but can also refer to fields like span.description""" - # If a virtual context is defined the column definition is always the same - if column in self._resolved_attribute_cache: - return self._resolved_attribute_cache[column] - - alias = column - if public_alias_override is not None: - alias = public_alias_override - - if column in self.definitions.contexts: - column_context = self.definitions.contexts[column] - column_definition = ResolvedAttribute( - public_alias=alias, - internal_name=column, - search_type="string", - processor=column_context.processor, - ) - elif column in self.definitions.columns: - column_context = None - column_definition = self.definitions.columns[column] - if column_definition.private and column not in self.config.fields_acl.attributes: - raise InvalidSearchQuery(f"The field {column} is not allowed for this query") - # Need to override the ResolvedAttribute entirely - if public_alias_override: - column_definition = ResolvedAttribute( - public_alias=public_alias_override, - internal_name=column_definition.internal_name, - search_type=column_definition.search_type, - ) - elif (internal_match := self._find_column_by_internal_name(column)) is not None: - column_context = None - column_definition = ResolvedAttribute( - public_alias=alias, - internal_name=internal_match.internal_name, - search_type=internal_match.search_type, - internal_type=internal_match.internal_type, - validator=internal_match.validator, - normalizer=internal_match.normalizer, - processor=internal_match.processor, - ) - else: - if len(column) > qb_constants.MAX_TAG_KEY_LENGTH: - raise InvalidSearchQuery( - f"{column} is too long, can be a maximum of 200 characters" - ) - - tag_match = qb_constants.TYPED_TAG_KEY_RE.search(column) - if tag_match is None: - tag_match = qb_constants.TAG_KEY_RE.search(column) - field_type = "string" - else: - field_type = None - # make sure to remove surrounding quotes if it's a tag - field = tag_match.group("tag") if tag_match else column - if field is None: - raise InvalidSearchQuery(f"Could not parse {column}") - # Assume string if a type isn't passed. eg. tags[foo] - if field_type is None: - field_type = tag_match.group("type") if tag_match else None - - if field_type not in constants.TYPE_MAP: - raise InvalidSearchQuery(f"Unsupported type {field_type} in {column}") - - if column.startswith("sentry_tags"): - field = f"sentry.{field}" - - if self.definitions.alias_to_column is not None: - mapped_column = self.definitions.alias_to_column(field) - if mapped_column is not None: - field = mapped_column - - column_definition = ResolvedAttribute( - public_alias=alias, internal_name=field, search_type=field_type - ) - column_context = None - - if column_definition: - self._resolved_attribute_cache[column] = (column_definition, [column_context]) - return self._resolved_attribute_cache[column] - else: - raise InvalidSearchQuery(f"Could not parse {column}") - - @traces.trace - def resolve_functions( - self, columns: list[str] - ) -> tuple[ - list[ResolvedFunction], - list[VirtualColumnDefinition | None], - ]: - """Helper function to resolve a list of functions instead of 1 attribute at a time""" - resolved_functions, resolved_contexts = [], [] - for column in columns: - try: - function, contexts = self.resolve_function(column) - except HiddenApiAttribute: - continue - resolved_functions.append(function) - resolved_contexts.extend(contexts) - return resolved_functions, resolved_contexts - - def resolve_function( - self, - column: str, - match: Match[str] | None = None, - public_alias_override: str | None = None, - default_value: float | None = None, - ) -> tuple[ResolvedFunction, list[VirtualColumnDefinition | None]]: - if match is None: - match = fields.is_function(column) - if match is None: - raise InvalidSearchQuery(f"{column} is not a function") - - function_name = match.group("function") - columns = match.group("columns") - # Alias defaults to the name of the function - alias = match.group("alias") or column - if public_alias_override is not None: - alias = public_alias_override - - # Don't use cache if default_value is passed - if alias in self._resolved_function_cache and default_value is None: - return self._resolved_function_cache[alias] - # Check if the column looks like a function (matches a pattern), parse the function name and args out - - if self.config.saved_formulas is not None: - if function_name in self.config.saved_formulas: - resolved_column, contexts = self.resolve_formula( - self.config.saved_formulas[function_name], columns, alias - ) - if not isinstance(resolved_column, ResolvedFunction): - raise InvalidSearchQuery( - f"The formula {function_name} must resolve to a function or equation" - ) - return resolved_column, contexts - - function_definition = self.get_function_definition(function_name) - if ( - self.definitions.aggregate_deprecations - and function_name in self.definitions.aggregate_deprecations - ): - deprecated_definition = self.definitions.aggregate_deprecations[function_name] - else: - deprecated_definition = None - - # Temporary while we get rid of the old _if combinators - if deprecated_definition: - try: - return self._resolve_function( - function_definition, function_name, alias, columns, default_value - ) - except Exception: - logger.warning( - "resolver.deprecated_function_used", - extra={ - "organization_id": self.params.organization_id, - "function_name": function_name, - "column": column, - "query_string": self.params.query_string, - }, - ) - if self.config.disable_deprecated_functions: - raise - return self._resolve_function( - deprecated_definition, function_name, alias, columns, default_value - ) - else: - return self._resolve_function( - function_definition, function_name, alias, columns, default_value - ) - - def _resolve_function( - self, - function_definition: FormulaDefinition | AggregateDefinition, - function_name: str, - alias: str, - columns: str, - default_value: float | None = None, - ) -> tuple[ResolvedFunction, list[VirtualColumnDefinition | None]]: - if function_definition.private and function_name not in self.config.fields_acl.functions: - raise InvalidSearchQuery(f"The function {function_name} is not allowed for this query") - - parsed_args: list[ResolvedAttribute | str | int | float | TraceItemFilter] = [] - - # Parse the arguments - arguments = fields.parse_arguments(function_name, columns) - if len(arguments) < len(function_definition.required_arguments): - raise InvalidSearchQuery( - f"Invalid number of arguments for {function_name}, was expecting {len(function_definition.required_arguments)} arguments" - ) - - missing_args = len(function_definition.arguments) - len(arguments) - argument_index = 0 - - for argument_definition in function_definition.arguments: - if argument_definition.ignored: - continue - - # If there are missing arguments, and the argument definition has a default arg, use the default arg - # this assumes the missing args are at the beginning or end of the arguments list - if missing_args > 0 and argument_definition.default_arg is not None: - if isinstance(argument_definition, ValueArgumentDefinition): - parsed_args.append(argument_definition.default_arg) - else: - parsed_argument, _ = self.resolve_attribute(argument_definition.default_arg) - self._raise_if_hidden_api_attribute( - argument_definition.default_arg, parsed_argument - ) - parsed_args.append(parsed_argument) - missing_args -= 1 - continue - - if argument_index < len(arguments): - argument = arguments[argument_index] - argument_index += 1 - if argument_definition.validator is not None: - if not argument_definition.validator(argument): - raise InvalidSearchQuery( - f"{argument} is not a valid argument for {function_name}" - ) - if isinstance(argument_definition, AttributeArgumentDefinition): - parsed_argument, _ = self.resolve_attribute(argument) - self._raise_if_hidden_api_attribute(argument, parsed_argument) - parsed_args.append(parsed_argument) - else: - if argument_definition.argument_types is None: - parsed_args.append(argument) # assume it's a string - continue - for arg_type in argument_definition.argument_types: - if arg_type == "integer": - parsed_args.append(int(argument)) - elif arg_type == "number": - parsed_args.append(float(argument)) - elif arg_type == "query": - # Only TraceItemFilter currently supported - trace_item_filters = self.resolve_query(argument[1:-1])[0] - if trace_item_filters is None: - raise InvalidSearchQuery( - "The if combinator requires non-aggregate filters" - ) - parsed_args.append(trace_item_filters) - else: - parsed_args.append(argument) - continue - else: - raise InvalidSearchQuery( - f"Invalid number of arguments for {function_name}, was expecting {len(function_definition.required_arguments)} arguments" - ) - - if isinstance(argument_definition, AttributeArgumentDefinition) and ( - argument_definition.attribute_types is not None - and parsed_argument.search_type not in argument_definition.attribute_types - ): - if argument_definition.field_allowlist is not None: - if parsed_argument.public_alias not in argument_definition.field_allowlist: - raise InvalidSearchQuery( - f"{parsed_argument.public_alias} is invalid for parameter {argument_index} in {function_name}." - ) - else: - raise InvalidSearchQuery( - f"{parsed_argument.public_alias} is invalid for parameter {argument_index} in {function_name}. Its a {parsed_argument.search_type} type field, but it must be one of these types: {argument_definition.attribute_types}" - ) - - resolved_arguments: ResolvedArguments = [] - for parsed_arg in parsed_args: - resolved_argument: ResolvedArgument - if not isinstance(parsed_arg, ResolvedAttribute): - resolved_argument = parsed_arg - search_type = function_definition.default_search_type - else: - resolved_argument = parsed_arg.proto_definition - resolved_arguments.append(resolved_argument) - - if len(parsed_args) == 0 or not isinstance(parsed_args[0], ResolvedAttribute): - search_type = function_definition.default_search_type - else: - # unless infer_search_type_from_arguments is passed we assume the first argument is the search_type - search_type = ( - parsed_args[0].search_type - if function_definition.infer_search_type_from_arguments - else function_definition.default_search_type - ) - - resolved_function = function_definition.resolve( - alias=alias, - search_type=search_type, - resolved_arguments=resolved_arguments, - snuba_params=self.params, - query_result_cache=self._query_result_cache, - search_config=self.config, - default_value=default_value, - ) - - resolved_context: list[VirtualColumnDefinition | None] = [None] - if default_value is None: - self._resolved_function_cache[alias] = (resolved_function, resolved_context) - return self._resolved_function_cache[alias] - else: - return resolved_function, resolved_context - - def resolve_equations( - self, equations: list[str] - ) -> tuple[ - list[ResolvedColumn], - list[VirtualColumnDefinition | None], - ]: - formulas = [] - contexts = [] - for equation in equations: - try: - formula, context = self.resolve_equation(equation) - except HiddenApiAttribute: - continue - formulas.append(formula) - contexts.extend(context) - return formulas, contexts - - def resolve_equation( - self, equation: str, alias: str | None = None - ) -> tuple[ - ResolvedColumn, - list[VirtualColumnDefinition | None], - ]: - """Resolve an equation creating a ResolvedEquation object, we don't just return a Column.BinaryFormula since - it'll help callers with extra information, like the existence of aggregates and the search type - """ - operation, fields, functions = arithmetic.parse_arithmetic( - equation, definitions=self.definitions - ) - # Handle the case where the equation is just a single term - if isinstance(operation, str): - # Resolve the column, and turn it into a RPC Column so it can be used in a BinaryFormula - col, context = self.resolve_column( - operation, public_alias_override=f"equation|{equation}" if alias is None else alias - ) - if isinstance(col, ResolvedAttribute): - self._raise_if_hidden_api_attribute(operation, col) - return col, context - elif isinstance(operation, float): - return ( - ResolvedLiteral( - public_alias=f"equation|{equation}" if alias is None else alias, - search_type="number", - value=operation, - ), - [], - ) - # Fold equations that are only literals so Snuba does not evaluate them. - # Field equations such as `span.duration / 1000` still go through the RPC. - elif len(fields) == 0 and len(functions) == 0: - try: - value = arithmetic.resolve_arithmetic(operation) - except ZeroDivisionError: - raise arithmetic.ArithmeticValidationError("division by 0 is not allowed") - return ( - ResolvedLiteral( - public_alias=f"equation|{equation}" if alias is None else alias, - search_type="number", - value=value, - ), - [], - ) - - lhs, lhs_contexts = ( - self._resolve_operation(operation.lhs) if operation.lhs is not None else (None, []) - ) - rhs, rhs_contexts = ( - self._resolve_operation(operation.rhs) if operation.rhs is not None else (None, []) - ) - has_aggregates = False - for function in functions: - resolved_function, _ = self.resolve_function(function) - if resolved_function.is_aggregate: - has_aggregates = True - break - return ( - ResolvedEquation( - public_alias=f"equation|{equation}" if alias is None else alias, - # Type of equations can become complex very quickly. We could try to make sure all the columns have the - # same type and return that, but then ratios would have types eg. (p75/p50), as well managing multiple - # column types is strange too (p75+count). Keeping this as just `number` for now - search_type="number", - operator=constants.ARITHMETIC_OPERATOR_MAP[operation.operator], - lhs=lhs, - rhs=rhs, - is_aggregate=has_aggregates, - ), - lhs_contexts + rhs_contexts, - ) - - def _resolve_operation( - self, operation: arithmetic.OperandType - ) -> tuple[ - Column, - list[VirtualColumnDefinition | None], - ]: - """This function is to recursively step into the branches of the arithmetic to resolve branches to Columns for - the RPC, but we can't only use this since we want the resolver to return a ResolvedEquation so the resolver API - matches between resolved arithmetic and columns - """ - if isinstance(operation, arithmetic.Operation): - lhs, lhs_contexts = ( - self._resolve_operation(operation.lhs) if operation.lhs is not None else (None, []) - ) - rhs, rhs_contexts = ( - self._resolve_operation(operation.rhs) if operation.rhs is not None else (None, []) - ) - vcc = [] - if lhs_contexts: - vcc += lhs_contexts - if rhs_contexts: - vcc += rhs_contexts - return ( - Column( - formula=Column.BinaryFormula( - op=constants.ARITHMETIC_OPERATOR_MAP[operation.operator], - left=lhs, - right=rhs, - ) - ), - vcc, - ) - elif isinstance(operation, float): - return Column(literal=LiteralValue(val_double=operation)), [] - - # Resolve the column, and turn it into a RPC Column so it can be used in a BinaryFormula - # Columns in equations must pass default_value=0 otherwise they may become a null and ruin the entire formula - col, context = self.resolve_column(operation, default_value=0) - if isinstance(col, ResolvedAttribute): - self._raise_if_hidden_api_attribute(operation, col) - proto_definition = col.proto_definition - - if isinstance(proto_definition, AttributeKey): - return Column(key=proto_definition), context - - if isinstance(proto_definition, AttributeAggregation): - return Column(aggregation=proto_definition), context - - if isinstance(proto_definition, AttributeConditionalAggregation): - return Column(conditional_aggregation=proto_definition), context - - if isinstance(proto_definition, Column.BinaryFormula): - return Column(formula=proto_definition), context - - raise TypeError(f"Unsupported proto definition type: {type(proto_definition)}") - - def resolve_dataset_conditions( - self, - selected_columns: list[str] | None, - equations: list[str] | None, - ) -> TraceItemFilter | None: - extra_conditions = self.config.extra_conditions(self, selected_columns, equations) - - return and_trace_item_filters(extra_conditions) - - def remap_value_using_context_definition( - self, context_definition: VirtualColumnDefinition, value: str | int | list[str] | Any - ) -> str | int | list[str] | Any: - context = context_definition.constructor(self.params, self) - - # if the value passed is one of the potential values, then it's expected - # and we should pass it through as is - for val in context.value_map.values(): - if val == value: - return value - - # if the value passed is one of the potential keys, then it should before - # remapped to the value - if isinstance(value, str) and value in context.value_map: - value = context.value_map[value] - - # now that we've checked all potentially allowed values, we should fall back - # to using the default - if context.default_value: - return context.default_value - - return value - - def resolve_formula( - self, formula: ExploreSavedFormula, columns: str, alias: str - ) -> tuple[ResolvedColumn, list[VirtualColumnDefinition | None]]: - """Given a formula object, parse the arguments, then resolve it as an equation - - Like equations these aren't cached - """ - from sentry.search.eap.utils import resolve_and_parse_formula - - arguments = fields.parse_arguments(formula.name, columns) - - equation = resolve_and_parse_formula(formula, arguments, self) - return self.resolve_equation(equation, alias) diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml deleted file mode 100644 index 8c5d61662..000000000 --- a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Dispatch a request to getsentry to run getsentry test suites -name: getsentry dispatcher - -on: - # XXX: We are using `pull_request_target` instead of `pull_request` because we want - # this to run on forks. It allows forks to access secrets safely by - # only running workflows from the main branch. Prefer to use `pull_request` when possible. - # - # See https://github.com/getsentry/sentry/pull/21600 for more details - pull_request_target: - types: [labeled, opened, reopened, synchronize] - -# disable all other special privileges -permissions: - # needed for `actions/checkout` to clone the code - contents: read - # needed to remove the pull-request label - pull-requests: write - -jobs: - dispatch: - if: "github.event.action != 'labeled' || github.event.label.name == 'Trigger: getsentry tests'" - name: getsentry dispatch - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - persist-credentials: false - - - name: permissions - run: | - python3 -uS .github/workflows/scripts/getsentry-dispatch-setup \ - --repo-id ${{ github.event.repository.id }} \ - --pr ${{ github.event.number }} \ - --event ${{ github.event.action }} \ - --username "$ARG_USERNAME" \ - --label-names "$ARG_LABEL_NAMES" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # these can contain special characters - ARG_USERNAME: ${{ github.event.pull_request.user.login }} - ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} - - - name: Check for file changes - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Get changed files for selective testing - id: changed-files - env: - GH_TOKEN: ${{ github.token }} - run: | - gh api repos/${{ github.repository }}/pulls/${{ github.event.number }}/files \ - --paginate | python3 .github/workflows/scripts/parse-pr-files.py >> "$GITHUB_OUTPUT" - - - name: getsentry token - uses: getsentry/action-github-app-token@5c1e90706fe007857338ac1bfbd7a4177db2f789 # v4.0.0 - id: getsentry - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - name: Wait for PR merge commit - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - id: mergecommit - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - const { waitForMergeCommit } = await import(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit.js`); - await waitForMergeCommit({ - github, - context, - core, - }); - - - name: Dispatch getsentry tests - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - SENTRY_CHANGED_FILES: ${{ steps.changed-files.outputs.files }} - SENTRY_PREVIOUS_FILENAMES: ${{ steps.changed-files.outputs.previous-filenames }} - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - const { dispatch } = await import(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/getsentry-dispatch.js`); - await dispatch({ - github, - context, - core, - mergeCommitSha: '${{ steps.mergecommit.outputs.mergeCommitSha }}', - fileChanges: ${{ toJson(steps.changes.outputs) }}, - sentryChangedFiles: process.env.SENTRY_CHANGED_FILES, - sentryPreviousFilenames: process.env.SENTRY_PREVIOUS_FILENAMES, - }); diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py deleted file mode 100644 index a027c0c03..000000000 --- a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -"""Parse the GitHub PR files API response into changed files and previous filenames. - -Reads JSON from stdin (the output of `gh api .../pulls/N/files --paginate`). -Note: --paginate can emit multiple JSON arrays (one per page), so we handle -concatenated arrays by decoding incrementally. - -Outputs two lines (suitable for appending to $GITHUB_OUTPUT): - files= - previous-filenames= - -Usage: - gh api repos/OWNER/REPO/pulls/N/files --paginate \ - | python3 parse-pr-files.py >> "$GITHUB_OUTPUT" -""" - -from __future__ import annotations - -import json -import sys - - -def main() -> None: - raw = sys.stdin.read() - decoder = json.JSONDecoder() - files = [] - idx = 0 - while idx < len(raw): - while idx < len(raw) and raw[idx].isspace(): - idx += 1 - if idx >= len(raw): - break - obj, end = decoder.raw_decode(raw, idx) - if isinstance(obj, list): - files.extend(obj) - idx = end - - changed = [f["filename"] for f in files] - previous = [ - f["previous_filename"] - for f in files - if f.get("status") == "renamed" and f.get("previous_filename") - ] - - print(f"files={' '.join(changed)}") - print(f"previous-filenames={' '.join(previous)}") - - -if __name__ == "__main__": - main() diff --git a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py b/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py deleted file mode 100644 index 92a29227b..000000000 --- a/packages/evals/fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py +++ /dev/null @@ -1,695 +0,0 @@ -import dataclasses -import logging -import re -from collections import Counter, OrderedDict, defaultdict -from collections.abc import Sequence -from ipaddress import ip_address, ip_interface, ip_network -from typing import Any, Callable - -from sentry.utils import metrics - -logger = logging.getLogger("sentry.events.grouping") - - -# Counter for logging a set amount of example data. Not meant to be used directly. (Use the -# `_log_example_data` helper instead.) -LOGGING_COUNTER: Counter[str] = Counter() - -# Function parameterization regexes can specify to provide a customized replacement string. Can also -# be used to do conditional replacement, by returning the original value in cases where replacement -# shouldn't happen. -ParameterizationReplacementFunction = Callable[[str], str] - - -# Log examples, up to the given limit. -def _log_example_data( - key: str, # Key used for tracking log count and in logger `event` string - extra: dict[str, Any], # Extra data to add to the log (should include example data) - limit: int = 100, # Number of logs to be gathered per deployment -) -> None: - # Note: In a multi-threaded environment, it's possible to run into a race condition where - # multiple threads are simultaneously logging what should theoretically be the last example. As - # result, we may end up logging a few more examples than the given limit. To fix this, we'd need - # to wrap everything here in a lock, but given that a few extra logs don't hurt anything, it's - # not worth blocking ingest by doing so. - if LOGGING_COUNTER[key] < limit: - logger.info(f"grouping.parameterization.{key}", extra=extra) - LOGGING_COUNTER[key] += 1 - - -@dataclasses.dataclass -class ParameterizationRegex: - name: str # name of the pattern (also used as group name in combined regex) - raw_pattern: str # regex pattern w/o matching group name - raw_pattern_experimental: str | None = None - # Function which takes the matched value and returns the replacement value. - replacement_callback: ParameterizationReplacementFunction | None = None - - # These need to be used with `(?x)`, to tell the regex compiler to ignore comments - # and unescaped whitespace, so we can use newlines and indentation for better legibility. - - @property - def pattern(self) -> str: - return self._get_pattern(self.raw_pattern) - - @property - def experimental_pattern(self) -> str | None: - if not self.raw_pattern_experimental: - return None - return self._get_pattern(self.raw_pattern_experimental) - - def _get_pattern(self, raw_pattern: str) -> str: - """ - Returns the regex pattern inside of a named matching group. - """ - return rf"(?P<{self.name}>{raw_pattern})" - - -def is_valid_ip(maybe_ip_str: str) -> bool: - # Validate the string by attempting to pass it to the three built-in factory functions for - # creating different types of ip address objects. If any of them succeeds, it's a valid IP. If - # all three raise an error, it's not. - for fn, kwargs in ( - (ip_address, {}), - (ip_interface, {}), - (ip_network, {"strict": False}), # `strict: False` allows host bits - ): - try: - fn(maybe_ip_str, **kwargs) - except ValueError: - pass - else: - return True - - return False - - -# fmt: off -TLDS = {"com", "net", "org", "jp", "de", "uk", "fr", "br", "it", "ru", "es", "me", "gov", "pl", "ca", "au", "cn", "co", "in", "nl", "edu", "info", "eu", "ch", "id", "at", "kr", "cz", "mx", "be", "tv", "se", "tr", "tw", "al", "ua", "ir", "vn", "cl", "sk", "ly", "cc", "to", "no", "fi", "us", "pt", "dk", "ar", "hu", "tk", "gr", "il", "news", "ro", "my", "biz", "ie", "za", "nz", "sg", "ee", "th", "io", "xyz", "pe", "bg", "hk", "rs", "lt", "link", "ph", "club", "si", "site", "mobi", "by", "cat", "wiki", "la", "ga", "xxx", "cf", "hr", "ng", "jobs", "online", "kz", "ug", "gq", "ae", "is", "lv", "pro", "fm", "tips", "ms", "sa", "app"} -# fmt: on - - -def is_valid_hostname(maybe_hostname_str: str) -> bool: - # By spec, hostnames have a max length of 255 - if len(maybe_hostname_str) > 255: - return False - - # Almost anything is an allowable top-level domain since ICANN opened it up, but that means that - # paths like `some.path.to.a.module` and attribute access like `someobj.someproperty` and - # filenames like `somefile.txt` all would match as hostnames, which isn't what we want. So, at - # the risk of occasional false negatives, we check against the top 99 TLDs and if it doesn't - # match one of those, we don't count it as a hostname. (Fortunately there's not a ton of overlap - # between popular TLDs and common file extensions, and most of them aren't actual words, so - # they're also less likely to be used as module or property names.) - maybe_tld = maybe_hostname_str.split(".")[-1] - if maybe_tld.lower() not in TLDS: - return False - - return True - - -DEFAULT_PARAMETERIZATION_REGEXES = [ - ParameterizationRegex( - name="email", - raw_pattern=r"""[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]{1,254}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*""", - ), - ParameterizationRegex( - name="url", - raw_pattern=r""" - # Scheme - by spec, must start with a letter, but after that can contain anything - # alphanumeric in addition to plus, minus (has to be escaped so it's clear it's not part - # of a range), and dot - [a-zA-Z][a-zA-Z0-9+\-.]* - # The normal separator - :// - # First character of the domain (or path, if the domain is empty) - must be a valid URL - # character (so no quotes, backticks, backslashes, angle brackets, sqiggly brackets, - # pipes, carets, or whitespace) and must be allowable in the first spot (no starting the - # querystring right away, for example) - [^'"`\\<>{}|\^\s$.?#] - # The rest of the URL is technically optional - ( - [^'"`\\<>{}|\^\s]* # Any number of copies of anything not globally invalid - [^'"`\\<>{}|\^\s.,;] # Final character - must be both valid in general and allowable - # in the last spot (so no trailing punctuation) - )? - """, - ), - ParameterizationRegex( - name="hostname", - # This regex is intentionally loose, in that it doesn't encode the hostname spec's overall - # length restriction, and in that it catches things which are technically valid hostnames - # but which we don't want to parameterize that way (module paths, filenames with extensions, - # etc.). Max length is complicated to bake into a regex, and depending on how you do it, - # using the regex to narrow matches down to the most popular top-level domains is either - # slow or too aggressive. So here we use the regex to find hostname-like strings, and then - # validate them using a callback. - raw_pattern=r""" - # The overall pattern here expresses "2 to 128 dot-separated segments, each segment - # consisting of up to 63 letters/numbers/dashes, as long as no segment starts or ends - # with a dash." Individual parts labeled below. - \b - # All segments but the final one, each followed by a dot - ( - # Lookahead guaranteeing at least one letter - (?= [a-zA-Z0-9\-]* [a-zA-Z]) - # Segment body - either all letters/numbers (no dashes), or with interior-only dashes - ( - [a-zA-Z0-9]{1,63} - | - [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] - ) - \. - ){1,127} - # Final segment has no dot - ( - [a-zA-Z0-9]{1,63} - | - [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] - ) - \b - """, - # Validate that the matched string actually is a valid hostname before replacing it. If not, - # leave it alone. - replacement_callback=lambda orig_value: "" - if is_valid_hostname(orig_value) - else orig_value, - ), - ParameterizationRegex( - name="traceparent", - raw_pattern=r""" - # https://www.w3.org/TR/trace-context/#traceparent-header - (\b00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]\b) | - - # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html#request-tracing-syntax - (\b1-[0-9a-f]{8}-[0-9a-f]{24}\b) - """, - ), - ParameterizationRegex( - name="uuid", - raw_pattern=r"""\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b""", - ), - ParameterizationRegex(name="sha1", raw_pattern=r"""\b[0-9a-fA-F]{40}\b"""), - ParameterizationRegex(name="md5", raw_pattern=r"""\b[0-9a-fA-F]{32}\b"""), - ParameterizationRegex( - name="date", - raw_pattern=r""" - # No word boundaries required around dates. Should there be? - # RFC822, RFC1123, RFC1123Z - ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s\d{1,2}\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2,4}\s\d{1,2}:\d{1,2}(:\d{1,2})?\s([-\+][\d]{2}[0-5][\d]|(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z]))) - | - # Similar to RFC822, but "Mon Jan 02, 1999", "Jan 02, 1999" - (((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-3]\d,\s\d{2,4}) - | - # RFC850 - ((?:Sun|Sunday|Mon|Monday|Tue|Tuesday|Wed|Wednesday|Thu|Thursday|Fri|Friday|Sat|Saturday),\s\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}\s\d{2}:\d{2}:\d{2}\s(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z])) - | - # RFC3339, RFC3339Nano - (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?([+-]?\d{2}:\d{2})?) - | - # JavaScript - ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2}\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT[+-]\d{4}(?:\s\([^)]+\))?) - | - # Datetime with timezone offset (Z=UTC): - # (Note: These come before plain datetimes so the offset is seen as part of the time and - # not a separate int value.) - ( - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+([+-][0-2]\d:?[0-5]\d|Z))| # decimal seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z))| # seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z)) # no seconds - ) - | - # Datetime - ( - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+)| # decimal seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d)| # seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d) # no seconds - ) - | - # Kitchen, 12-hr - ( - (?) - (?.5) - (?!\.\d) - \b - ) - | - # IPv6-like strings - # - # Note: We can't use word boundaries here as we did with IPv4s because IPv6s contain - # non-word characters. Instead, we use a negative lookbehind and a negative lookahead, - # respectively, to specify the beginning and end of the pattern. These protect against - # two things: - # - Cases where the initial or end characters are all valid, but there are too many of - # them. (IOW, we don't want to match `2345:...` inside of an otherwise-invalid IP - # like `12345:...`, and the same applies to `...:1234` inside of `...:12345`. - # Similar logic applies to initial and final colons - we don't want `:::1` to turn - # into `:` because it's matching on just the second and third colons.) - # - Cases where `::` (which is a valid IPv6 address) appears inside of expressions - # like `SomeClass::someMethod()`, especially when the characters bordering the `::` - # are valid hex, like `Space::explore()`. - # This doesn't fix edge cases like `Fee::add()`, where it's all hex and also fewer than - # 5 characters on either side, but those are presumably pretty rare. - (?" if is_valid_ip(orig_value) else orig_value, - ), - ParameterizationRegex( - name="swift_txn_id", - raw_pattern=r""" - # OpenStack Swift transaction IDs: "tx" + 21 hex chars + "-" + 10 hex chars, - # optionally followed by a suffix (e.g. cluster ID or client-provided extra) - # https://docs.openstack.org/api-ref/object-store/ - (\btx[0-9a-f]{21}-[0-9a-f]{10}\S*) - """, - ), - ParameterizationRegex( - name="hex", - raw_pattern=r""" - # Hex value with `0x/0X` prefix (any length, with any mix of numbers and/or letters - - # the prefix pretty much guarantees it's hex). - (\b0[xX][0-9a-fA-F]+\b) | - - # Hex value without `0x/0X` prefix (at least 4 characters long, containing both a letter - # and number if shorter than 8 characters, and either all uppercase or all lowercase - - # we're more conservative here in order to reduce false positives). - ( - # For positive values, a negative lookbehind to create a word boundary but with - # underscores allowed (to permit things like `some_file_31d12.py`) - (? 96% of 7-digit - # hex values are mixed, so that's a tradeoff we're okay with.) Also, it only includes - # lowercase letters, since git shas are always expressed that way. - (\b(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7}\b) - """, - ), - # This pattern must come after the hex-based patterns so it doesn't catch strings which happen - # to contain only letters between A and F. - ParameterizationRegex( - name="random_id", - raw_pattern=r""" - \b - # Random nonsense alphanumeric id strings. To avoid false positives, we require the - # following: - # - A mix of uppercase letters, lowercase letters, and numbers - # - A minimum of 4 characters, with at least a certain level of "mixed-up-ness". For - # our purposes, that means the string must switch back and forth between letters and - # numbers at least 3 times. This rules out human-readable strings like `dogNumber1` - # (1 switch) and `bball4lyfe` (2 switches), while catching strings like `aKj8XLr2`. - # - # Lookahead guaranteeing at least one uppercase letter - (?= [a-z0-9]* [A-Z]) - # Lookahead guaranteeing at least one lowercase letter - (?= [A-Z0-9]* [a-z]) - # Lookahead enforcing letter/number switches. Two versions depending on whether the - # string starts with a letter or number. This also takes care of the "contains a number" - # requirement. - (?= - \d+ [a-zA-Z]+ \d+ [a-zA-Z]+ - | - [a-zA-Z]+ \d+ [a-zA-Z]+ \d+ - ) - # The pattern itself - [a-zA-Z0-9]{4,128} - \b - """, - ), - ParameterizationRegex( - name="float", - raw_pattern=r""" - ( - # For positive values, in addition to the wordbreak, a negative lookbehind to ensure - # this isn't part of a longer set of dot-delimited ints (IOW, to prevent `1.2.3` - # from matching as `1.`) - ( - \b - (?.3) - (?!\.\d) - \b - """, - ), - ParameterizationRegex( - name="int", - raw_pattern=r""" - ( - # For positive values, a negative lookbehind to create a word boundary but with - # underscores allowed (to permit things like `some_file_1121.py`) - (?=`-type key-value - # pairs, rather than all quoted strings - (?<=[=]) - ( - '([^']+)' | - "([^"]+)" - ) - """, - ), - ParameterizationRegex( - name="bool", - raw_pattern=r""" - # Lookbehind to ensure we'll only match the value half of `=`-type key-value - # pairs, rather than all instances of the words 'true' and 'false' - (?<=[=]) - ( - True | true | False | false - ) - """, - ), -] - - -class Parameterizer: - # Inputs longer than this are not parameterized, as a defense against ReDoS - MAX_INPUT_LENGTH = 8192 - - def __init__( - self, - # List of `ParameterizationRegex` objects defining the regexes to use. If nothing is passed, - # the default set will be used. - regexes: Sequence[ParameterizationRegex] = DEFAULT_PARAMETERIZATION_REGEXES, - *, - # List of `ParameterizationRegex.name` values, used to selectively enable pattern types. To - # use all available parameterization, omit this argument. - regex_keys: Sequence[str] | None = None, - # Whether to use experimental patterns, if available. (Pattern types without an experimental - # pattern will fall back to the standard pattern.) - use_experimental_regexes: bool = False, - ): - # Filter regexes by the specified keys, if given - if regex_keys: - regexes = [r for r in regexes if r.name in regex_keys] - - self.is_experimental = ( - use_experimental_regexes - # Only mark the parameterizer as experimental if there are actually any experiments - # running. If there aren't, then both parameterizers use the default regex patterns, so - # the "experimental" parameterizer isn't actually experimental. - and any(r.experimental_pattern is not None for r in regexes) - ) - - if self.is_experimental: - # This uses an `OrderedDict` to guarantee we apply the patterns in the given order. - patterns_by_name = OrderedDict( - ( - r.name, - r.experimental_pattern if r.experimental_pattern else r.pattern, - ) - for r in regexes - ) - else: - patterns_by_name = OrderedDict((r.name, r.pattern) for r in regexes) - - # Combine the individual patterns into one giant regex to check against. (This is faster - # than checking each pattern individually because it entails less overhead.) - self.combined_regex = re.compile( - # The `(?x)` tells the regex compiler to ignore comments and unescaped whitespace, so we - # can use newlines and indentation for better legibility when defining regexes - rf"(?x){'|'.join(patterns_by_name.values())}" - ) - - # Collect replacement callbacks, if any - self.replacement_functions = { - r.name: r.replacement_callback for r in regexes if r.replacement_callback - } - - # Store the individual regexes in case we need to handle replacement function false - # positives by manually looping through the patterns. This uses an `OrderedDict` to - # guarantee we apply them in the same order as in the main/combined regex. - self.compiled_regexes_by_name = OrderedDict( - (name, re.compile(rf"(?x){pattern}")) for name, pattern in patterns_by_name.items() - ) - - def parameterize(self, input_str: str) -> str: - """ - Replace all regex matches in the input string with placeholder strings, using the regexes - with which the parameterizer was initialized. - - For example, turn "Error with order #1231" into "Error with order #". - """ - - if len(input_str) > self.MAX_INPUT_LENGTH: - metrics.incr("grouping.parameterization_skipped_long_input") - return input_str - - replacement_counts: defaultdict[str, int] = defaultdict(int) - # Track whether any regex matches don't lead to a replacement - found_false_positive = False - # Flag allowing us to only count false positives during the main parameterization, not the - # fallback run - emit_false_positive_metric = True - - def _handle_regex_match(match: re.Match[str]) -> str: - # Ensure we're dealing with the flag from the outer scope, rather than shadowing it - nonlocal found_false_positive - - # This handler gets called on two types of regexes: - # - The main/combination regex, which consists of a bunch of named capturing groups - # (with no nested named groups) separated by `|`. - # - Individual regexes for each pattern type, each consisting of one of the - # alternatives in the main regex. - # In the former case, only one alternative can have been matched, so its group name will - # be the only (and therefore last) matched group name. In the latter case, there is only - # one named group to match, so its name will automatically be the only/last matched - # group name. Thus `lastgroup` should give us the group name in either case. - matched_key = match.lastgroup - orig_value = match.groupdict().get( - matched_key or "" # Empty string for mypy appeasment - ) - - if not matched_key or not orig_value: # Insurance - shouldn't happen IRL - return "" - - replacement_callback = self.replacement_functions.get(matched_key) - replacement_string = ( - replacement_callback(orig_value) if replacement_callback else f"<{matched_key}>" - ) - - # The replacement callback might return the original value, if it determines it's not - # something which should be replaced, and we don't want to count that - if replacement_string != orig_value: - replacement_counts[matched_key] += 1 - else: - found_false_positive = True - - # This is only true during the main combo-regex parameterization, not during - # fallback, so that we don't double-count these occurrences - if emit_false_positive_metric: - # Track the number of false positive matches, and what pattern produced them. We - # can compare this to the same key's `grouping.value_parameterized` metric below - # to see how often our maybe-matches pan out to be actual matches. - metrics.incr( - "grouping.parameterization_false_positive", tags={"key": matched_key} - ) - # TODO: Remove this once we have enough sample data - _log_example_data( - "ip_false_positive", extra={"input_str": input_str, "value": orig_value} - ) - - return replacement_string - - with metrics.timer( - "grouping.parameterize", tags={"experimental": self.is_experimental} - ) as metric_tags: - parameterized = self.combined_regex.sub(_handle_regex_match, input_str) - - # Our big combo regex will short-circuit when it finds a match, which is great for - # performance but problematic in cases where a replacement callback declines to - # parameterize a value, because then the value never gets checked against later - # patterns. To protect against that, in those cases we cycle through all patterns - # individually, which is slower but ensures we check them all. - if found_false_positive: - metric_tags["false_positive"] = True - - # Reset values before applying the patterns again - replacement_counts = defaultdict(int) - parameterized = input_str - - # Prevent double-counting of false positives - emit_false_positive_metric = False - - # Apply patterns one by one, with no short-circuiting - for regex_key, regex in self.compiled_regexes_by_name.items(): - parameterized = regex.sub(_handle_regex_match, parameterized) - - metric_tags["changed"] = parameterized != input_str - - for regex_key, count in replacement_counts.items(): - # Track the kinds of replacements being made - metrics.incr("grouping.value_parameterized", amount=count, tags={"key": regex_key}) - - return parameterized - - -parameterizer = Parameterizer(use_experimental_regexes=False) -experimental_parameterizer = Parameterizer(use_experimental_regexes=True) diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py b/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py deleted file mode 100644 index 48139acaf..000000000 --- a/packages/evals/fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py +++ /dev/null @@ -1,696 +0,0 @@ -import dataclasses -import logging -import re -from collections import Counter, OrderedDict, defaultdict -from collections.abc import Sequence -from ipaddress import ip_address, ip_interface, ip_network -from typing import Any, Callable - -from sentry.utils import metrics - -logger = logging.getLogger("sentry.events.grouping") - - -# Counter for logging a set amount of example data. Not meant to be used directly. (Use the -# `_log_example_data` helper instead.) -LOGGING_COUNTER: Counter[str] = Counter() - -# Function parameterization regexes can specify to provide a customized replacement string. Can also -# be used to do conditional replacement, by returning the original value in cases where replacement -# shouldn't happen. -ParameterizationReplacementFunction = Callable[[str], str] - - -# Log examples, up to the given limit. -def _log_example_data( - key: str, # Key used for tracking log count and in logger `event` string - extra: dict[str, Any], # Extra data to add to the log (should include example data) - limit: int = 100, # Number of logs to be gathered per deployment -) -> None: - # Note: In a multi-threaded environment, it's possible to run into a race condition where - # multiple threads are simultaneously logging what should theoretically be the last example. As - # result, we may end up logging a few more examples than the given limit. To fix this, we'd need - # to wrap everything here in a lock, but given that a few extra logs don't hurt anything, it's - # not worth blocking ingest by doing so. - if LOGGING_COUNTER[key] < limit: - logger.info(f"grouping.parameterization.{key}", extra=extra) - LOGGING_COUNTER[key] += 1 - - -@dataclasses.dataclass -class ParameterizationRegex: - name: str # name of the pattern (also used as group name in combined regex) - raw_pattern: str # regex pattern w/o matching group name - raw_pattern_experimental: str | None = None - # Function which takes the matched value and returns the replacement value. - replacement_callback: ParameterizationReplacementFunction | None = None - - # These need to be used with `(?x)`, to tell the regex compiler to ignore comments - # and unescaped whitespace, so we can use newlines and indentation for better legibility. - - @property - def pattern(self) -> str: - return self._get_pattern(self.raw_pattern) - - @property - def experimental_pattern(self) -> str | None: - if not self.raw_pattern_experimental: - return None - return self._get_pattern(self.raw_pattern_experimental) - - def _get_pattern(self, raw_pattern: str) -> str: - """ - Returns the regex pattern inside of a named matching group. - """ - return rf"(?P<{self.name}>{raw_pattern})" - - -def is_valid_ip(maybe_ip_str: str) -> bool: - # Validate the string by attempting to pass it to the three built-in factory functions for - # creating different types of ip address objects. If any of them succeeds, it's a valid IP. If - # all three raise an error, it's not. - for fn, kwargs in ( - (ip_address, {}), - (ip_interface, {}), - (ip_network, {"strict": False}), # `strict: False` allows host bits - ): - try: - fn(maybe_ip_str, **kwargs) - except ValueError: - pass - else: - return True - - return False - - -# fmt: off -TLDS = {"com", "net", "org", "jp", "de", "uk", "fr", "br", "it", "ru", "es", "me", "gov", "pl", "ca", "au", "cn", "co", "in", "nl", "edu", "info", "eu", "ch", "id", "at", "kr", "cz", "mx", "be", "tv", "se", "tr", "tw", "al", "ua", "ir", "vn", "cl", "sk", "ly", "cc", "to", "no", "fi", "us", "pt", "dk", "ar", "hu", "tk", "gr", "il", "news", "ro", "my", "biz", "ie", "za", "nz", "sg", "ee", "th", "io", "xyz", "pe", "bg", "hk", "rs", "lt", "link", "ph", "club", "si", "site", "mobi", "by", "cat", "wiki", "la", "ga", "xxx", "cf", "hr", "ng", "jobs", "online", "kz", "ug", "gq", "ae", "is", "lv", "pro", "fm", "tips", "ms", "sa", "app"} -# fmt: on - - -def is_valid_hostname(maybe_hostname_str: str) -> bool: - # By spec, hostnames have a max length of 255 - if len(maybe_hostname_str) > 255: - return False - - # Almost anything is an allowable top-level domain since ICANN opened it up, but that means that - # paths like `some.path.to.a.module` and attribute access like `someobj.someproperty` and - # filenames like `somefile.txt` all would match as hostnames, which isn't what we want. So, at - # the risk of occasional false negatives, we check against the top 99 TLDs and if it doesn't - # match one of those, we don't count it as a hostname. (Fortunately there's not a ton of overlap - # between popular TLDs and common file extensions, and most of them aren't actual words, so - # they're also less likely to be used as module or property names.) - maybe_tld = maybe_hostname_str.split(".")[-1] - if maybe_tld.lower() not in TLDS: - return False - - return True - - -DEFAULT_PARAMETERIZATION_REGEXES = [ - ParameterizationRegex( - name="email", - raw_pattern=r"""[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]{1,254}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*""", - ), - ParameterizationRegex( - name="url", - raw_pattern=r""" - # Scheme - by spec, must start with a letter, but after that can contain anything - # alphanumeric in addition to plus, minus (has to be escaped so it's clear it's not part - # of a range), and dot. The length is bounded (real schemes are short) to prevent - # quadratic backtracking on long runs of scheme-valid characters that lack a `://`. - [a-zA-Z][a-zA-Z0-9+\-.]{0,32} - # The normal separator - :// - # First character of the domain (or path, if the domain is empty) - must be a valid URL - # character (so no quotes, backticks, backslashes, angle brackets, sqiggly brackets, - # pipes, carets, or whitespace) and must be allowable in the first spot (no starting the - # querystring right away, for example) - [^'"`\\<>{}|\^\s$.?#] - # The rest of the URL is technically optional - ( - [^'"`\\<>{}|\^\s]* # Any number of copies of anything not globally invalid - [^'"`\\<>{}|\^\s.,;] # Final character - must be both valid in general and allowable - # in the last spot (so no trailing punctuation) - )? - """, - ), - ParameterizationRegex( - name="hostname", - # This regex is intentionally loose, in that it doesn't encode the hostname spec's overall - # length restriction, and in that it catches things which are technically valid hostnames - # but which we don't want to parameterize that way (module paths, filenames with extensions, - # etc.). Max length is complicated to bake into a regex, and depending on how you do it, - # using the regex to narrow matches down to the most popular top-level domains is either - # slow or too aggressive. So here we use the regex to find hostname-like strings, and then - # validate them using a callback. - raw_pattern=r""" - # The overall pattern here expresses "2 to 128 dot-separated segments, each segment - # consisting of up to 63 letters/numbers/dashes, as long as no segment starts or ends - # with a dash." Individual parts labeled below. - \b - # All segments but the final one, each followed by a dot - ( - # Lookahead guaranteeing at least one letter - (?= [a-zA-Z0-9\-]* [a-zA-Z]) - # Segment body - either all letters/numbers (no dashes), or with interior-only dashes - ( - [a-zA-Z0-9]{1,63} - | - [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] - ) - \. - ){1,127} - # Final segment has no dot - ( - [a-zA-Z0-9]{1,63} - | - [a-zA-Z0-9] [a-zA-Z0-9\-]{1,61} [a-zA-Z0-9] - ) - \b - """, - # Validate that the matched string actually is a valid hostname before replacing it. If not, - # leave it alone. - replacement_callback=lambda orig_value: "" - if is_valid_hostname(orig_value) - else orig_value, - ), - ParameterizationRegex( - name="traceparent", - raw_pattern=r""" - # https://www.w3.org/TR/trace-context/#traceparent-header - (\b00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]\b) | - - # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html#request-tracing-syntax - (\b1-[0-9a-f]{8}-[0-9a-f]{24}\b) - """, - ), - ParameterizationRegex( - name="uuid", - raw_pattern=r"""\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b""", - ), - ParameterizationRegex(name="sha1", raw_pattern=r"""\b[0-9a-fA-F]{40}\b"""), - ParameterizationRegex(name="md5", raw_pattern=r"""\b[0-9a-fA-F]{32}\b"""), - ParameterizationRegex( - name="date", - raw_pattern=r""" - # No word boundaries required around dates. Should there be? - # RFC822, RFC1123, RFC1123Z - ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s\d{1,2}\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2,4}\s\d{1,2}:\d{1,2}(:\d{1,2})?\s([-\+][\d]{2}[0-5][\d]|(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z]))) - | - # Similar to RFC822, but "Mon Jan 02, 1999", "Jan 02, 1999" - (((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s)?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-3]\d,\s\d{2,4}) - | - # RFC850 - ((?:Sun|Sunday|Mon|Monday|Tue|Tuesday|Wed|Wednesday|Thu|Thursday|Fri|Friday|Sat|Saturday),\s\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}\s\d{2}:\d{2}:\d{2}\s(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z])) - | - # RFC3339, RFC3339Nano - (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?([+-]?\d{2}:\d{2})?) - | - # JavaScript - ((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)\s(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{2}\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT[+-]\d{4}(?:\s\([^)]+\))?) - | - # Datetime with timezone offset (Z=UTC): - # (Note: These come before plain datetimes so the offset is seen as part of the time and - # not a separate int value.) - ( - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+([+-][0-2]\d:?[0-5]\d|Z))| # decimal seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z))| # seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d([+-][0-2]\d:?[0-5]\d|Z)) # no seconds - ) - | - # Datetime - ( - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d\.\d+)| # decimal seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d:?[0-5]\d)| # seconds - (\d{4}-?[01]\d-?[0-3]\d[\sT][0-2]\d:?[0-5]\d) # no seconds - ) - | - # Kitchen, 12-hr - ( - (?) - (?.5) - (?!\.\d) - \b - ) - | - # IPv6-like strings - # - # Note: We can't use word boundaries here as we did with IPv4s because IPv6s contain - # non-word characters. Instead, we use a negative lookbehind and a negative lookahead, - # respectively, to specify the beginning and end of the pattern. These protect against - # two things: - # - Cases where the initial or end characters are all valid, but there are too many of - # them. (IOW, we don't want to match `2345:...` inside of an otherwise-invalid IP - # like `12345:...`, and the same applies to `...:1234` inside of `...:12345`. - # Similar logic applies to initial and final colons - we don't want `:::1` to turn - # into `:` because it's matching on just the second and third colons.) - # - Cases where `::` (which is a valid IPv6 address) appears inside of expressions - # like `SomeClass::someMethod()`, especially when the characters bordering the `::` - # are valid hex, like `Space::explore()`. - # This doesn't fix edge cases like `Fee::add()`, where it's all hex and also fewer than - # 5 characters on either side, but those are presumably pretty rare. - (?" if is_valid_ip(orig_value) else orig_value, - ), - ParameterizationRegex( - name="swift_txn_id", - raw_pattern=r""" - # OpenStack Swift transaction IDs: "tx" + 21 hex chars + "-" + 10 hex chars, - # optionally followed by a suffix (e.g. cluster ID or client-provided extra) - # https://docs.openstack.org/api-ref/object-store/ - (\btx[0-9a-f]{21}-[0-9a-f]{10}\S*) - """, - ), - ParameterizationRegex( - name="hex", - raw_pattern=r""" - # Hex value with `0x/0X` prefix (any length, with any mix of numbers and/or letters - - # the prefix pretty much guarantees it's hex). - (\b0[xX][0-9a-fA-F]+\b) | - - # Hex value without `0x/0X` prefix (at least 4 characters long, containing both a letter - # and number if shorter than 8 characters, and either all uppercase or all lowercase - - # we're more conservative here in order to reduce false positives). - ( - # For positive values, a negative lookbehind to create a word boundary but with - # underscores allowed (to permit things like `some_file_31d12.py`) - (? 96% of 7-digit - # hex values are mixed, so that's a tradeoff we're okay with.) Also, it only includes - # lowercase letters, since git shas are always expressed that way. - (\b(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7}\b) - """, - ), - # This pattern must come after the hex-based patterns so it doesn't catch strings which happen - # to contain only letters between A and F. - ParameterizationRegex( - name="random_id", - raw_pattern=r""" - \b - # Random nonsense alphanumeric id strings. To avoid false positives, we require the - # following: - # - A mix of uppercase letters, lowercase letters, and numbers - # - A minimum of 4 characters, with at least a certain level of "mixed-up-ness". For - # our purposes, that means the string must switch back and forth between letters and - # numbers at least 3 times. This rules out human-readable strings like `dogNumber1` - # (1 switch) and `bball4lyfe` (2 switches), while catching strings like `aKj8XLr2`. - # - # Lookahead guaranteeing at least one uppercase letter - (?= [a-z0-9]* [A-Z]) - # Lookahead guaranteeing at least one lowercase letter - (?= [A-Z0-9]* [a-z]) - # Lookahead enforcing letter/number switches. Two versions depending on whether the - # string starts with a letter or number. This also takes care of the "contains a number" - # requirement. - (?= - \d+ [a-zA-Z]+ \d+ [a-zA-Z]+ - | - [a-zA-Z]+ \d+ [a-zA-Z]+ \d+ - ) - # The pattern itself - [a-zA-Z0-9]{4,128} - \b - """, - ), - ParameterizationRegex( - name="float", - raw_pattern=r""" - ( - # For positive values, in addition to the wordbreak, a negative lookbehind to ensure - # this isn't part of a longer set of dot-delimited ints (IOW, to prevent `1.2.3` - # from matching as `1.`) - ( - \b - (?.3) - (?!\.\d) - \b - """, - ), - ParameterizationRegex( - name="int", - raw_pattern=r""" - ( - # For positive values, a negative lookbehind to create a word boundary but with - # underscores allowed (to permit things like `some_file_1121.py`) - (?=`-type key-value - # pairs, rather than all quoted strings - (?<=[=]) - ( - '([^']+)' | - "([^"]+)" - ) - """, - ), - ParameterizationRegex( - name="bool", - raw_pattern=r""" - # Lookbehind to ensure we'll only match the value half of `=`-type key-value - # pairs, rather than all instances of the words 'true' and 'false' - (?<=[=]) - ( - True | true | False | false - ) - """, - ), -] - - -class Parameterizer: - # Inputs longer than this are not parameterized, as a defense against ReDoS - MAX_INPUT_LENGTH = 8192 - - def __init__( - self, - # List of `ParameterizationRegex` objects defining the regexes to use. If nothing is passed, - # the default set will be used. - regexes: Sequence[ParameterizationRegex] = DEFAULT_PARAMETERIZATION_REGEXES, - *, - # List of `ParameterizationRegex.name` values, used to selectively enable pattern types. To - # use all available parameterization, omit this argument. - regex_keys: Sequence[str] | None = None, - # Whether to use experimental patterns, if available. (Pattern types without an experimental - # pattern will fall back to the standard pattern.) - use_experimental_regexes: bool = False, - ): - # Filter regexes by the specified keys, if given - if regex_keys: - regexes = [r for r in regexes if r.name in regex_keys] - - self.is_experimental = ( - use_experimental_regexes - # Only mark the parameterizer as experimental if there are actually any experiments - # running. If there aren't, then both parameterizers use the default regex patterns, so - # the "experimental" parameterizer isn't actually experimental. - and any(r.experimental_pattern is not None for r in regexes) - ) - - if self.is_experimental: - # This uses an `OrderedDict` to guarantee we apply the patterns in the given order. - patterns_by_name = OrderedDict( - ( - r.name, - r.experimental_pattern if r.experimental_pattern else r.pattern, - ) - for r in regexes - ) - else: - patterns_by_name = OrderedDict((r.name, r.pattern) for r in regexes) - - # Combine the individual patterns into one giant regex to check against. (This is faster - # than checking each pattern individually because it entails less overhead.) - self.combined_regex = re.compile( - # The `(?x)` tells the regex compiler to ignore comments and unescaped whitespace, so we - # can use newlines and indentation for better legibility when defining regexes - rf"(?x){'|'.join(patterns_by_name.values())}" - ) - - # Collect replacement callbacks, if any - self.replacement_functions = { - r.name: r.replacement_callback for r in regexes if r.replacement_callback - } - - # Store the individual regexes in case we need to handle replacement function false - # positives by manually looping through the patterns. This uses an `OrderedDict` to - # guarantee we apply them in the same order as in the main/combined regex. - self.compiled_regexes_by_name = OrderedDict( - (name, re.compile(rf"(?x){pattern}")) for name, pattern in patterns_by_name.items() - ) - - def parameterize(self, input_str: str) -> str: - """ - Replace all regex matches in the input string with placeholder strings, using the regexes - with which the parameterizer was initialized. - - For example, turn "Error with order #1231" into "Error with order #". - """ - - if len(input_str) > self.MAX_INPUT_LENGTH: - metrics.incr("grouping.parameterization_skipped_long_input") - return input_str - - replacement_counts: defaultdict[str, int] = defaultdict(int) - # Track whether any regex matches don't lead to a replacement - found_false_positive = False - # Flag allowing us to only count false positives during the main parameterization, not the - # fallback run - emit_false_positive_metric = True - - def _handle_regex_match(match: re.Match[str]) -> str: - # Ensure we're dealing with the flag from the outer scope, rather than shadowing it - nonlocal found_false_positive - - # This handler gets called on two types of regexes: - # - The main/combination regex, which consists of a bunch of named capturing groups - # (with no nested named groups) separated by `|`. - # - Individual regexes for each pattern type, each consisting of one of the - # alternatives in the main regex. - # In the former case, only one alternative can have been matched, so its group name will - # be the only (and therefore last) matched group name. In the latter case, there is only - # one named group to match, so its name will automatically be the only/last matched - # group name. Thus `lastgroup` should give us the group name in either case. - matched_key = match.lastgroup - orig_value = match.groupdict().get( - matched_key or "" # Empty string for mypy appeasment - ) - - if not matched_key or not orig_value: # Insurance - shouldn't happen IRL - return "" - - replacement_callback = self.replacement_functions.get(matched_key) - replacement_string = ( - replacement_callback(orig_value) if replacement_callback else f"<{matched_key}>" - ) - - # The replacement callback might return the original value, if it determines it's not - # something which should be replaced, and we don't want to count that - if replacement_string != orig_value: - replacement_counts[matched_key] += 1 - else: - found_false_positive = True - - # This is only true during the main combo-regex parameterization, not during - # fallback, so that we don't double-count these occurrences - if emit_false_positive_metric: - # Track the number of false positive matches, and what pattern produced them. We - # can compare this to the same key's `grouping.value_parameterized` metric below - # to see how often our maybe-matches pan out to be actual matches. - metrics.incr( - "grouping.parameterization_false_positive", tags={"key": matched_key} - ) - # TODO: Remove this once we have enough sample data - _log_example_data( - "ip_false_positive", extra={"input_str": input_str, "value": orig_value} - ) - - return replacement_string - - with metrics.timer( - "grouping.parameterize", tags={"experimental": self.is_experimental} - ) as metric_tags: - parameterized = self.combined_regex.sub(_handle_regex_match, input_str) - - # Our big combo regex will short-circuit when it finds a match, which is great for - # performance but problematic in cases where a replacement callback declines to - # parameterize a value, because then the value never gets checked against later - # patterns. To protect against that, in those cases we cycle through all patterns - # individually, which is slower but ensures we check them all. - if found_false_positive: - metric_tags["false_positive"] = True - - # Reset values before applying the patterns again - replacement_counts = defaultdict(int) - parameterized = input_str - - # Prevent double-counting of false positives - emit_false_positive_metric = False - - # Apply patterns one by one, with no short-circuiting - for regex_key, regex in self.compiled_regexes_by_name.items(): - parameterized = regex.sub(_handle_regex_match, parameterized) - - metric_tags["changed"] = parameterized != input_str - - for regex_key, count in replacement_counts.items(): - # Track the kinds of replacements being made - metrics.incr("grouping.value_parameterized", amount=count, tags={"key": regex_key}) - - return parameterized - - -parameterizer = Parameterizer(use_experimental_regexes=False) -experimental_parameterizer = Parameterizer(use_experimental_regexes=True) diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py b/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py deleted file mode 100644 index 76ab87d9c..000000000 --- a/packages/evals/fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py +++ /dev/null @@ -1,71 +0,0 @@ -from django.conf import settings -from rest_framework.request import Request -from rest_framework.response import Response - -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import Endpoint, all_silo_endpoint -from sentry.api.permissions import SuperuserPermission -from sentry.conf.server import SENTRY_EARLY_FEATURES -from sentry.runner.settings import configure, discover_configs - - -@all_silo_endpoint -class InternalFeatureFlagsEndpoint(Endpoint): - permission_classes = (SuperuserPermission,) - owner = ApiOwner.HYBRID_CLOUD - publish_status = { - "GET": ApiPublishStatus.PRIVATE, - "PUT": ApiPublishStatus.PRIVATE, - } - - def get(self, request: Request) -> Response: - if not settings.SENTRY_SELF_HOSTED: - return Response("You are not self-hosting Sentry.", status=403) - - result = {} - for key in SENTRY_EARLY_FEATURES: - result[key] = { - "value": settings.SENTRY_FEATURES.get(key, False), - "description": SENTRY_EARLY_FEATURES[key], - } - - return Response(result) - - def put(self, request: Request) -> Response: - if not settings.SENTRY_SELF_HOSTED: - return Response("You are not self-hosting Sentry.", status=403) - - data = request.data.keys() - valid_feature_flags = [flag for flag in data if SENTRY_EARLY_FEATURES.get(flag, False)] - _, py, yml = discover_configs() - # Open the file for reading and writing - with open(py, "r+") as file: - lines = file.readlines() - # print(lines) - for valid_flag in valid_feature_flags: - match_found = False - new_string = ( - f'\nSENTRY_FEATURES["{valid_flag}"]={request.data.get(valid_flag, False)}\n' - ) - # Search for the string match and update lines - for i, line in enumerate(lines): - if valid_flag in line: - match_found = True - lines[i] = new_string - - break - - # If no match found, append a new line - if not match_found: - lines.append(new_string) - - # Move the file pointer to the beginning and truncate the file - file.seek(0) - file.truncate() - - # Write modified lines back to the file - file.writelines(lines) - configure(None, py, yml) - - return Response(status=200) diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py b/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py deleted file mode 100644 index 47e09f7dd..000000000 --- a/packages/evals/fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py +++ /dev/null @@ -1,117 +0,0 @@ -from collections.abc import Mapping - -from django.conf import settings -from rest_framework.request import Request -from rest_framework.response import Response - -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import Endpoint, all_silo_endpoint -from sentry.api.permissions import SuperuserPermission -from sentry.conf.server import SENTRY_EARLY_FEATURES -from sentry.runner.settings import configure, discover_configs - - -def _coerce_early_feature_flag_value(value: object) -> bool | None: - """ - Map API input to a bool for SENTRY_FEATURES. Accepts JSON booleans, 0/1 - (common in scripts), and the strings "true"/"false" (case-insensitive). - Returns None if the value cannot be interpreted safely. - """ - if isinstance(value, bool): - return value - if isinstance(value, int): - if value == 1: - return True - if value == 0: - return False - return None - if isinstance(value, str): - lowered = value.lower() - if lowered == "true": - return True - if lowered == "false": - return False - return None - return None - - -@all_silo_endpoint -class InternalFeatureFlagsEndpoint(Endpoint): - permission_classes = (SuperuserPermission,) - owner = ApiOwner.HYBRID_CLOUD - publish_status = { - "GET": ApiPublishStatus.PRIVATE, - "PUT": ApiPublishStatus.PRIVATE, - } - - def get(self, request: Request) -> Response: - if not settings.SENTRY_SELF_HOSTED: - return Response("You are not self-hosting Sentry.", status=403) - - result = {} - for key in SENTRY_EARLY_FEATURES: - result[key] = { - "value": settings.SENTRY_FEATURES.get(key, False), - "description": SENTRY_EARLY_FEATURES[key], - } - - return Response(result) - - def put(self, request: Request) -> Response: - if not settings.SENTRY_SELF_HOSTED: - return Response("You are not self-hosting Sentry.", status=403) - - payload: object = request.data - if not isinstance(payload, Mapping): - return Response( - {"detail": "Feature flag updates must be a JSON object."}, - status=400, - ) - - valid_feature_flags = [flag for flag in payload if flag in SENTRY_EARLY_FEATURES] - coerced_values: dict[str, bool] = {} - for valid_flag in valid_feature_flags: - coerced = _coerce_early_feature_flag_value(payload.get(valid_flag)) - if coerced is None: - return Response( - { - "detail": ( - f'Feature flag "{valid_flag}" must be true or false ' - f"(boolean, 0 or 1, or the string true or false)." - ) - }, - status=400, - ) - coerced_values[valid_flag] = coerced - - _, py, yml = discover_configs() - # Open the file for reading and writing - with open(py, "r+") as file: - lines = file.readlines() - # print(lines) - for valid_flag in valid_feature_flags: - match_found = False - python_bool = "True" if coerced_values[valid_flag] else "False" - new_string = f'\nSENTRY_FEATURES["{valid_flag}"]={python_bool}\n' - # Search for the string match and update lines - for i, line in enumerate(lines): - if valid_flag in line: - match_found = True - lines[i] = new_string - - break - - # If no match found, append a new line - if not match_found: - lines.append(new_string) - - # Move the file pointer to the beginning and truncate the file - file.seek(0) - file.truncate() - - # Write modified lines back to the file - file.writelines(lines) - configure(None, py, yml) - - return Response(status=200) diff --git a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py b/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py deleted file mode 100644 index 396c2cece..000000000 --- a/packages/evals/fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py +++ /dev/null @@ -1,274 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterable, Sequence -from typing import NoReturn, NotRequired, TypedDict - -from django.http import HttpRequest -from drf_spectacular.utils import extend_schema -from rest_framework import serializers -from rest_framework.exceptions import APIException, ParseError -from rest_framework.negotiation import BaseContentNegotiation -from rest_framework.parsers import BaseParser -from rest_framework.renderers import BaseRenderer -from rest_framework.request import Request -from rest_framework.response import Response -from rest_framework.views import APIView -from sentry_sdk import traces - -from sentry.api.api_owners import ApiOwner -from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission -from sentry.auth.services.auth import auth_service -from sentry.models.organization import Organization -from sentry.organizations.services.organization import RpcOrganization, RpcUserOrganizationContext - -from .constants import SCIM_400_INVALID_FILTER, SCIM_API_ERROR, SCIM_API_LIST - -SCIM_CONTENT_TYPES = ["application/json", "application/json+scim"] -ACCEPTED_FILTERED_KEYS = ["userName", "value", "displayName"] - - -class SCIMApiError(APIException): - def __init__(self, detail: str, status_code: int = 400, scim_type: str | None = None) -> None: - span = traces.get_current_span() - if span is not None: - span.set_attribute("http.status_code", status_code) - error_body = { - "schemas": [SCIM_API_ERROR], - "status": str(status_code), - "detail": detail, - } - if scim_type is not None: - error_body["scimType"] = scim_type - super().__init__(error_body) - self.status_code = status_code - - -class SCIMFilterError(ValueError): - pass - - -class SCIMClientNegotiation(BaseContentNegotiation): - # SCIM uses the content type "application/json+scim" - # which is just json for our purposes. - def select_parser(self, request: Request, parsers: Iterable[BaseParser]) -> BaseParser | None: - """ - Select the first parser in the `.parser_classes` list. - """ - for parser in parsers: - if parser.media_type in SCIM_CONTENT_TYPES: - return parser - return None - - def select_renderer( - self, request: Request, renderers: Iterable[BaseRenderer], format_suffix: str | None = None - ) -> tuple[BaseRenderer, str] | None: - """ - Select the first renderer in the `.renderer_classes` list. - """ - for renderer in renderers: - if renderer.media_type in SCIM_CONTENT_TYPES: - return (renderer, renderer.media_type) - return None - - -class SCIMQueryParamSerializer(serializers.Serializer): - # SCIM API parameters are standardized camelCase by RFC7644. - # We convert them to snake_case using the source field - - startIndex = serializers.IntegerField( - min_value=1, - required=False, - default=1, - source="start_index", - help_text="SCIM 1-offset based index for pagination.", - ) - count = serializers.IntegerField( - min_value=0, - required=False, - default=100, - help_text="The maximum number of results the query should return, maximum of 100.", - ) - filter = serializers.CharField( - required=False, - default=None, - help_text="A SCIM filter expression. The only operator currently supported is `eq`.", - ) - excludedAttributes = serializers.ListField( - child=serializers.CharField(), - required=False, - default=list, - source="excluded_attributes", - help_text="Fields that should be left off of return values. Right now the only supported field for this query is members.", - ) - - def validate_filter(self, filter: str) -> str | int | None: - try: - return parse_filter_conditions(filter) - except SCIMFilterError: - raise serializers.ValidationError("invalid filter") - - -class SCIMQueryParams(TypedDict): - start_index: int - count: int - filter: NotRequired[str | None] - excluded_attributes: NotRequired[list[str]] - - -class OrganizationSCIMPermission(OrganizationPermission): - def has_object_permission( - self, - request: Request, - view: APIView, - organization: Organization | RpcOrganization | RpcUserOrganizationContext, - ) -> bool: - result = super().has_object_permission(request, view, organization) - # The scim endpoints should only be used in conjunction with a SAML2 integration - if not result: - return result - if isinstance(organization, RpcUserOrganizationContext): - organization = organization.organization - provider = auth_service.get_auth_provider(organization_id=organization.id) - return provider is not None and provider.flags.scim_enabled - - -class OrganizationSCIMMemberPermission(OrganizationSCIMPermission): - scope_map = { - "GET": ["member:read", "member:write", "member:admin"], - "POST": ["member:write", "member:admin"], - "PATCH": ["member:write", "member:admin"], - "PUT": ["member:write", "member:admin"], - "DELETE": ["member:admin"], - } - - -class OrganizationSCIMTeamPermission(OrganizationSCIMPermission): - scope_map = { - "GET": ["team:read", "team:write", "team:admin"], - "POST": ["team:write", "team:admin"], - "PATCH": ["team:write", "team:admin"], - "DELETE": ["team:admin"], - } - - -class SCIMListBaseResponse(TypedDict): - schemas: list[str] - totalResults: int - startIndex: int - itemsPerPage: int - - -class SCIMMetaResponse(TypedDict): - resourceType: str - location: str - - -@extend_schema(tags=["SCIM"]) -class SCIMEndpoint(OrganizationEndpoint): - owner = ApiOwner.FOUNDATIONS - requires_user = True - content_negotiation_class = SCIMClientNegotiation - cursor_name = "startIndex" - - def add_cursor_headers( - self, request: Request, response: Response, cursor_result: object - ) -> None: - pass - - def list_api_format( - self, results: Sequence[object], total_results: int, start_index: int - ) -> dict[str, object]: - return { - "schemas": [SCIM_API_LIST], - "totalResults": total_results, # TODO: audit perf of queryset.count() - "startIndex": start_index, - "itemsPerPage": len(results), # what's max? - "Resources": results, - } - - def get_query_parameters(self, request: Request) -> SCIMQueryParams: - serializer = SCIMQueryParamSerializer(data=request.GET) - if not serializer.is_valid(): - if "filter" in serializer.errors: - # invalid filter needs to return a specific formatted - # error response - raise ParseError(detail=SCIM_400_INVALID_FILTER) - raise ParseError(serializer.errors) - - return serializer.validated_data - - -class SCIMDiscoveryEndpoint(SCIMEndpoint): - """Base for the RFC 7644 §4 discovery endpoints (ServiceProviderConfig, - ResourceTypes, Schemas). Discovery resources are static and GET-only: - query parameters are ignored rather than validated, a supplied filter is - rejected, and other HTTP methods get SCIM-format 405 bodies. - """ - - permission_classes = (OrganizationSCIMMemberPermission,) - - def reject_filter_param(self, request: Request) -> None: - # RFC 7644 §4: a filter on a discovery endpoint SHOULD be rejected - # with 403 so clients can't assume its conditions were applied. - # Handlers must call this first, rather than us hooking initial(): - # initial() runs before convert_args resolves the org, and the org - # access + scim_enabled checks must fail before the filter check. - if "filter" in request.GET: - raise SCIMApiError( - detail="Filtering is not supported on this endpoint.", status_code=403 - ) - - def http_method_not_allowed( - self, request: HttpRequest, *args: object, **kwargs: object - ) -> NoReturn: - raise SCIMApiError(detail=f'Method "{request.method}" not allowed.', status_code=405) - - -def parse_filter_conditions(raw_filters: str | None) -> str | int | None: - """ - this function parses a scim filter, see: - https://datatracker.ietf.org/doc/html/rfc7644#section-3.4.2.2 - - right now the only subset of that filtering we support is the simple "eq" - operator. the input would look like so: - userName eq "test.user@okta.local" - - We may want to support further SCIM grammar for other IDPs and may use - a package to replace this functionality. - """ - # TODO: support "and" operator - # TODO: support email querying/filtering - # TODO: graceful error handling when unsupported operators are used - - if raw_filters is None: - return None - - try: - conditions = raw_filters.split(",") - except ValueError: - raise SCIMFilterError - - # we don't support multiple filters right now. - if len(conditions) > 1: - raise SCIMFilterError - - condition = conditions[0] - try: - [key, value] = condition.split(" eq ") - except ValueError: - raise SCIMFilterError - - if not key or not value: - raise SCIMFilterError - key = key.strip() - value = value.strip() - - # remove encasing quotes around the value - value = value[1:-1] - - if key not in ACCEPTED_FILTERED_KEYS: - raise SCIMFilterError - if key == "value": - return int(value) - - return value diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py deleted file mode 100644 index 4f6ae12a5..000000000 --- a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py +++ /dev/null @@ -1,627 +0,0 @@ -import logging -from collections import defaultdict -from collections.abc import Sequence -from dataclasses import dataclass -from datetime import datetime -from functools import partial -from typing import Any, NotRequired, TypedDict - -from django.db.models import OuterRef, Subquery -from drf_spectacular.utils import OpenApiParameter, extend_schema -from rest_framework import serializers -from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError -from rest_framework.request import Request -from rest_framework.response import Response -from sentry_conventions.attributes import ATTRIBUTE_NAMES - -from sentry import features -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import cell_silo_endpoint -from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission -from sentry.api.paginator import OffsetPaginator -from sentry.api.serializers.rest_framework.base import convert_dict_key_case, snake_to_camel_case -from sentry.api.utils import to_valid_int_id, to_valid_int_id_list -from sentry.apidocs.parameters import CursorQueryParam, GlobalParams -from sentry.apidocs.utils import inline_sentry_response_serializer -from sentry.constants import ALL_ACCESS_PROJECT_ID, ALL_ACCESS_PROJECTS -from sentry.models.organization import Organization -from sentry.ratelimits.config import RateLimitConfig -from sentry.seer.models.workflow import ( - SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, - SeerWorkflowConfig, - SeerWorkflowConfigExtras, - SeerWorkflowConfigProject, - SeerWorkflowFilters, - SeerWorkflowRun, - SeerWorkflowSchedule, - SeerWorkflowStrategy, -) -from sentry.seer.workflows.breached_metrics import ( - BreachedMetricWorkflow, - get_config_monitors_bulk, - get_monitor_project_ids, -) -from sentry.seer.workflows.registry import get_seer_workflow -from sentry.seer.workflows.workflow import ( - AllProjects, - SeerWorkflow, - SelectedProjects, - WorkflowUpdateError, -) -from sentry.types.ratelimit import RateLimit, RateLimitCategory - -logger = logging.getLogger(__name__) - - -class SeerWorkflowConfigResponse(TypedDict): - id: str - strategy: str - enabled: bool - schedule: str - version: int - projects: list[int] - monitorIds: list[int] - environments: list[str] - filters: dict[str, Any] - outputPolicy: dict[str, Any] - seededDefault: str | None - dateAdded: datetime - dateUpdated: datetime - dateLastRun: NotRequired[datetime] - lastRunStatus: NotRequired[str | None] - name: NotRequired[str] - - -class SeerWorkflowConfigsResponse(TypedDict): - configs: list[SeerWorkflowConfigResponse] - - -class WorkflowConfigListErrorResponse(TypedDict): - detail: dict[str, str] - - -class SeerWorkflowConfigPermission(OrganizationPermission): - # POST and PUT are further limited to the projects the caller can access. - scope_map = { - "GET": ["org:read", "org:write", "org:admin"], - "POST": ["project:read", "project:write", "project:admin"], - "PUT": ["project:read", "project:write", "project:admin"], - } - - -class WorkflowConfigCreateSerializer(serializers.Serializer): - strategy = serializers.ChoiceField( - choices=sorted(s.value for s in SEER_CONFIGURABLE_WORKFLOW_STRATEGIES) - ) - enabled = serializers.BooleanField(default=False) - schedule = serializers.ChoiceField(choices=SeerWorkflowSchedule.choices, required=False) - environments = serializers.ListField( - child=serializers.CharField(max_length=64), default=list, allow_empty=True - ) - # [-1] selects all projects; omitted means no projects. - projectIds = serializers.ListField( - child=serializers.IntegerField(min_value=ALL_ACCESS_PROJECT_ID), - required=False, - allow_empty=True, - ) - name = serializers.CharField(default=None, allow_null=True, max_length=64) - # Metric automations only; their projects come from the connected monitors. - monitorIds = serializers.ListField( - child=serializers.IntegerField(min_value=1), - required=False, - allow_empty=True, - ) - - def validate_projectIds(self, value: list[int]) -> list[int]: - if ALL_ACCESS_PROJECT_ID in value and value != [ALL_ACCESS_PROJECT_ID]: - raise serializers.ValidationError("[-1] cannot be combined with project IDs.") - return value - - def validate(self, data: dict[str, Any]) -> dict[str, Any]: - if data["strategy"] == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - if "projectIds" in data: - raise serializers.ValidationError( - { - "projectIds": "Metric automation projects are selected by connecting monitors." - } - ) - if "schedule" in data: - raise serializers.ValidationError( - { - "schedule": "Metric automations run when a monitor breaches, not on a schedule." - } - ) - elif "monitorIds" in data: - raise serializers.ValidationError( - {"monitorIds": "Only metric automations support monitor selection."} - ) - return data - - -class WorkflowConfigUpdateSerializer(serializers.Serializer): - projectIds = serializers.ListField( - child=serializers.IntegerField(min_value=ALL_ACCESS_PROJECT_ID), - required=False, - allow_empty=True, - ) - - monitorIds = serializers.ListField( - child=serializers.IntegerField(min_value=1), - required=False, - allow_empty=True, - ) - - def validate_projectIds(self, value: list[int]) -> list[int]: - if ALL_ACCESS_PROJECT_ID in value and value != [ALL_ACCESS_PROJECT_ID]: - raise serializers.ValidationError("[-1] cannot be combined with project IDs.") - return value - - def validate(self, data: dict[str, Any]) -> dict[str, Any]: - if ("projectIds" in data) == ("monitorIds" in data): - raise serializers.ValidationError("Send exactly one of projectIds or monitorIds.") - return data - - -@dataclass(frozen=True) -class WorkflowConfigLastRun: - config_id: int - last_run_at: datetime | None - last_run_status: str | None - - -def _check_project_access( - project_ids: Sequence[int], - *, - request: Request, - organization: Organization, - endpoint: OrganizationEndpoint, -) -> str | None: - """ - Check the caller can read every project in a list. Returns a user-facing denial - detail, or None when access is allowed. - - - Every explicit project in ``projectIds`` must be active in the organization, - and the caller must have read access to it. - - Requesting all projects with ``[-1]`` requires global access. - """ - if set(project_ids) == ALL_ACCESS_PROJECTS: - if not request.access.has_global_access: - return "Requesting all projects requires global access" - elif project_ids: - # Raises PermissionDenied unless every project is active and accessible. - requested = endpoint.get_projects(request, organization, project_ids=set(project_ids)) - for project in requested: - if not request.access.has_project_scope(project, "project:read"): - # TODO(aliu): unreachable unless the endpoint permission class is looser - return f"You do not have access to project {project.id}" - - return None - - -@cell_silo_endpoint -@extend_schema(tags=["Seer"]) -class OrganizationSeerWorkflowConfigsEndpoint(OrganizationEndpoint): - owner = ApiOwner.ML_AI - publish_status = {"GET": ApiPublishStatus.PRIVATE, "POST": ApiPublishStatus.PRIVATE} - permission_classes = (SeerWorkflowConfigPermission,) - enforce_rate_limit = True - rate_limits = RateLimitConfig( - limit_overrides={ - "POST": { - RateLimitCategory.USER: RateLimit(limit=5, window=60), - RateLimitCategory.ORGANIZATION: RateLimit(limit=20, window=60 * 60), - }, - } - ) - - @extend_schema( - operation_id="List Seer automation workflows", - parameters=[ - GlobalParams.ORG_ID_OR_SLUG, - CursorQueryParam, - OpenApiParameter( - name="id", - type=int, - location=OpenApiParameter.QUERY, - many=True, - explode=True, - description="Filter by config IDs. Repeat the parameter for multiple IDs.", - ), - ], - responses={ - 200: inline_sentry_response_serializer( - "SeerWorkflowConfigs", SeerWorkflowConfigsResponse - ), - 400: inline_sentry_response_serializer( - "SeerWorkflowConfigListError", WorkflowConfigListErrorResponse - ), - }, - ) - def get(self, request: Request, organization: Organization) -> Response: - """List saved Seer automation workflows. - - The ``configs`` list contains the current page; pagination links are in headers. - Filter by config IDs with ``?id=123&id=456``; omit ``id`` to list all configs. - Invalid IDs return 400; no visible matches return ``{"configs": []}``. - ``projects`` is ``[-1]`` for all projects, ``[]`` for no projects, or a list - of selected active project IDs accessible to the caller. An empty - ``environments`` list means all environments; otherwise it contains names. - ``filters`` contains strategy-specific filters. ``seededDefault`` records - the seeded strategy even after edits. - ``outputPolicy`` is the saved, opaque policy document. - ``name`` is the user-supplied name, omitted if unset. - ``dateLastRun`` is the latest visible run's creation time and ``lastRunStatus`` - is its status (null for older runs that predate status tracking). Both are - omitted if no runs are visible under the runs endpoint's project-scope rules. - """ - if not features.has("organizations:seer-automations", organization, actor=request.user): - raise NotFound - - try: - config_ids = to_valid_int_id_list("id", request.query_params.getlist("id")) - except ValidationError as error: - return Response({"detail": error.detail}, status=400) - - projects = self.get_projects(request, organization, include_all_accessible=True) - configs = SeerWorkflowConfig.objects.filter( - organization=organization, strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES - ) - if config_ids: - configs = configs.filter(id__in=config_ids) - return self.paginate( - request=request, - queryset=configs, - order_by="id", - paginator_cls=OffsetPaginator, - on_results=partial( - serialize_workflow_configs, - accessible_project_ids=[p.id for p in projects], - has_global_access=request.access.has_global_access, - ), - ) - - @extend_schema( - operation_id="Create a Seer automation workflow", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - request=WorkflowConfigCreateSerializer, - responses={ - 201: inline_sentry_response_serializer( - "SeerWorkflowConfig", SeerWorkflowConfigResponse - ), - }, - ) - def post(self, request: Request, organization: Organization) -> Response: - """Create a Seer automation workflow. - - Only ``strategy`` is required. ``enabled`` defaults to false, ``schedule`` to - daily, and ``environments`` to all. ``name`` is optional. Omitting - ``projectIds`` selects no projects. Use ``[-1]`` for all projects. - - Metric automations (``breached_metric_investigation``) take ``monitorIds`` - instead of ``projectIds``; their projects come from the connected monitors, and - the caller needs read access to each monitor's project. - - The number of automations per organization is capped, and creation is rate - limited. Failures past request validation, such as reaching the cap, return an - empty 400. Returns the new config in the same shape as an item in ``configs``. - """ - if not features.has("organizations:seer-automations", organization, actor=request.user): - raise NotFound - - serializer = WorkflowConfigCreateSerializer(data=request.data) - if not serializer.is_valid(): - return Response(serializer.errors, status=400) - data = serializer.validated_data - strategy = SeerWorkflowStrategy(data["strategy"]) - - project_ids: list[int] = data.get("projectIds") or [] - monitor_ids: list[int] = data.get("monitorIds") or [] - if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - denied = _check_project_access( - get_monitor_project_ids(monitor_ids, organization), - request=request, - organization=organization, - endpoint=self, - ) - else: - denied = _check_project_access( - project_ids, request=request, organization=organization, endpoint=self - ) - if denied is not None: - raise PermissionDenied(detail=denied) - - try: - workflow: SeerWorkflow - if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - workflow = BreachedMetricWorkflow.create_with_monitors( - organization.id, - enabled=data["enabled"], - environments=data["environments"], - monitor_ids=monitor_ids, - name=data.get("name"), - ) - else: - workflow = SeerWorkflow.create( - organization_id=organization.id, - strategy=strategy, - enabled=data["enabled"], - schedule=SeerWorkflowSchedule(data.get("schedule", SeerWorkflowSchedule.DAILY)), - environments=data["environments"], - project_ids=project_ids, - name=data.get("name"), - ) - except ValueError as e: - logger.exception("bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400}) - if isinstance(e, WorkflowUpdateError): - return Response({"detail": str(e)}, status=400) - return Response(status=400) - - projects = self.get_projects(request, organization, include_all_accessible=True) - result = serialize_workflow_configs( - [workflow.get_config()], - accessible_project_ids=[p.id for p in projects], - has_global_access=request.access.has_global_access, - ) - return Response(result["configs"][0], status=201) - - -@cell_silo_endpoint -@extend_schema(tags=["Seer"]) -class OrganizationSeerWorkflowConfigDetailsEndpoint(OrganizationEndpoint): - owner = ApiOwner.ML_AI - publish_status = {"GET": ApiPublishStatus.PRIVATE, "PUT": ApiPublishStatus.PRIVATE} - permission_classes = (SeerWorkflowConfigPermission,) - - def serialize( - self, request: Request, organization: Organization, config: SeerWorkflowConfig - ) -> SeerWorkflowConfigResponse: - projects = self.get_projects(request, organization, include_all_accessible=True) - return serialize_workflow_configs( - [config], - accessible_project_ids=[p.id for p in projects], - has_global_access=request.access.has_global_access, - )["configs"][0] - - @extend_schema( - operation_id="Retrieve a Seer automation workflow", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - responses={ - 200: inline_sentry_response_serializer( - "SeerWorkflowConfig", SeerWorkflowConfigResponse - ), - }, - ) - def get(self, request: Request, organization: Organization, config_id: str) -> Response: - """Retrieve a saved Seer automation workflow by ID. - - The response has the same shape as an item in the list endpoint's ``configs``. - """ - if not features.has("organizations:seer-automations", organization, actor=request.user): - raise NotFound - - try: - config = SeerWorkflowConfig.objects.get( - id=to_valid_int_id("config_id", config_id), - organization=organization, - strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, - ) - except (ValueError, SeerWorkflowConfig.DoesNotExist) as e: - raise NotFound from e - - return Response(self.serialize(request, organization, config)) - - @extend_schema( - operation_id="Update a Seer automation workflow", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - request=WorkflowConfigUpdateSerializer, - responses={ - 200: inline_sentry_response_serializer( - "SeerWorkflowConfig", SeerWorkflowConfigResponse - ), - }, - ) - def put(self, request: Request, organization: Organization, config_id: str) -> Response: - """Update a saved Seer automation workflow. - - Only the fields sent are changed. ``projectIds`` replaces the projects the - caller can access; selected projects the caller can't access are kept. - - - The same project-access rules apply here AND to the associated projects of ``monitorIds``. - - Changing the projects of a config that covers all projects requires global - access. - - ``monitorIds`` replaces the monitors of a metric automation and is rejected - for other strategies. - - Failures past request validation return an empty 400. Returns the updated config - in the same shape as an item in the list endpoint's ``configs``. - """ - if not features.has("organizations:seer-automations", organization, actor=request.user): - raise NotFound - - serializer = WorkflowConfigUpdateSerializer(data=request.data) - if not serializer.is_valid(): - return Response(serializer.errors, status=400) - - try: - config = SeerWorkflowConfig.objects.get( - id=to_valid_int_id("config_id", config_id), - organization=organization, - strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, - ) - except (ValueError, SeerWorkflowConfig.DoesNotExist) as e: - raise NotFound from e - workflow = get_seer_workflow(config) - - # Validate and update projects - project_ids: list[int] | None = serializer.validated_data.get("projectIds") - all_projects = project_ids == [ALL_ACCESS_PROJECT_ID] - if project_ids is not None: - denied = _check_project_access( - project_ids, - request=request, - organization=organization, - endpoint=self, - ) - if denied is not None: - raise PermissionDenied(detail=denied) - - try: - if all_projects: - workflow.set_project_selection(AllProjects()) - else: - workflow.set_project_selection(SelectedProjects(project_ids=tuple(project_ids))) - except ValueError as e: - logger.exception( - "bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400} - ) - if isinstance(e, WorkflowUpdateError): - return Response({"detail": str(e)}, status=400) - return Response(status=400) - config = workflow.get_config() - - # Validate and update monitors - monitor_ids: list[int] | None = serializer.validated_data.get("monitorIds") - if monitor_ids is not None: - if not isinstance(workflow, BreachedMetricWorkflow): - return Response( - {"detail": "Only metric automations support monitor selection."}, status=400 - ) - denied = _check_project_access( - get_monitor_project_ids(monitor_ids, organization), - request=request, - organization=organization, - endpoint=self, - ) - if denied is not None: - raise PermissionDenied(detail=denied) - try: - workflow.set_monitor_selection(monitor_ids) - except ValueError as e: - logger.exception( - "bad request", extra={ATTRIBUTE_NAMES.HTTP_RESPONSE_STATUS_CODE: 400} - ) - if isinstance(e, WorkflowUpdateError): - return Response({"detail": str(e)}, status=400) - return Response(status=400) - config = workflow.get_config() - - return Response(self.serialize(request, organization, config)) - - -def serialize_workflow_configs( - configs: Sequence[SeerWorkflowConfig], - *, - accessible_project_ids: Sequence[int], - has_global_access: bool, -) -> SeerWorkflowConfigsResponse: - selected_projects = get_selected_project_ids( - configs, accessible_project_ids=accessible_project_ids - ) - connected_monitors = get_config_monitors_bulk( - configs, accessible_project_ids=accessible_project_ids - ) - last_runs = get_config_last_runs( - configs, - accessible_project_ids=accessible_project_ids, - has_global_access=has_global_access, - ) - last_runs_by_config_id = {run.config_id: run for run in last_runs} - return { - "configs": [ - serialize_workflow_config( - config, - selected_project_ids=selected_projects[config.id], - connected_monitors=connected_monitors.get(config.id, []), - last_run=last_runs_by_config_id.get(config.id), - ) - for config in configs - ] - } - - -def get_selected_project_ids( - configs: Sequence[SeerWorkflowConfig], *, accessible_project_ids: Sequence[int] -) -> defaultdict[int, list[int]]: - selected_projects: defaultdict[int, list[int]] = defaultdict(list) - links = SeerWorkflowConfigProject.objects.filter( - config__in=configs, project_id__in=accessible_project_ids - ).order_by("project_id") - for config_id, project_id in links.values_list("config_id", "project_id"): - selected_projects[config_id].append(project_id) - return selected_projects - - -def get_config_last_runs( - configs: Sequence[SeerWorkflowConfig], - *, - accessible_project_ids: Sequence[int], - has_global_access: bool, -) -> list[WorkflowConfigLastRun]: - runs = SeerWorkflowRun.objects.filter( - workflow_config_id=OuterRef("id"), organization_id=OuterRef("organization_id") - ) - if not has_global_access: - runs = runs.filter(extras__snapshot__project_ids__contained_by=list(accessible_project_ids)) - latest_run = runs.order_by("-date_added", "-id") - last_runs = ( - SeerWorkflowConfig.objects.filter(id__in=[config.id for config in configs]) - .annotate( - last_run_at=Subquery(latest_run.values("date_added")[:1]), - last_run_status=Subquery(latest_run.values("status")[:1]), - ) - .values_list("id", "last_run_at", "last_run_status") - ) - return [ - WorkflowConfigLastRun( - config_id=config_id, last_run_at=last_run_at, last_run_status=last_run_status - ) - for config_id, last_run_at, last_run_status in last_runs - ] - - -def serialize_workflow_config( - config: SeerWorkflowConfig, - *, - selected_project_ids: list[int], - connected_monitors: list[tuple[int, int]], - last_run: WorkflowConfigLastRun | None, -) -> SeerWorkflowConfigResponse: - filters = SeerWorkflowFilters.parse_obj(config.filters) - monitor_ids = [monitor_id for monitor_id, _ in connected_monitors] - if config.strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - # Metric automation projects come from their connected monitors. - project_ids = sorted({project_id for _, project_id in connected_monitors}) - elif filters.all_projects: - project_ids = [ALL_ACCESS_PROJECT_ID] - else: - project_ids = selected_project_ids - result: SeerWorkflowConfigResponse = { - "id": str(config.id), - "strategy": config.strategy, - "enabled": config.enabled, - "schedule": config.schedule, - "version": config.version, - "projects": project_ids, - "monitorIds": monitor_ids, - "environments": filters.environments, - "filters": convert_dict_key_case( - { - key: value - for key, value in config.filters.items() - if key not in {"all_projects", "environments"} - }, - snake_to_camel_case, - ), - "outputPolicy": config.output_policy, - "seededDefault": SeerWorkflowConfigExtras.parse_obj(config.extras).seeded_default, - "dateAdded": config.date_added, - "dateUpdated": config.date_updated, - } - if last_run is not None and last_run.last_run_at is not None: - result["dateLastRun"] = last_run.last_run_at - result["lastRunStatus"] = last_run.last_run_status - name = SeerWorkflowConfigExtras.parse_obj(config.extras).name - if name is not None: - result["name"] = name - return result diff --git a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py b/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py deleted file mode 100644 index 06e91926c..000000000 --- a/packages/evals/fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py +++ /dev/null @@ -1,430 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Collection, Iterator, Mapping, Sequence -from contextlib import contextmanager -from dataclasses import dataclass -from functools import partial -from typing import Any, assert_never - -from django.db import router, transaction -from django.db.models import Count, Q, QuerySet -from pydantic import BaseModel - -from sentry.constants import ALL_ACCESS_PROJECTS, ObjectStatus -from sentry.models.organization import Organization -from sentry.models.project import Project -from sentry.seer.models.workflow import ( - MAX_WORKFLOW_CONFIGS_PER_ORG, - SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, - SeerWorkflowConfig, - SeerWorkflowConfigExtras, - SeerWorkflowConfigProject, - SeerWorkflowFilters, - SeerWorkflowRun, - SeerWorkflowRunExtras, - SeerWorkflowRunStatus, - SeerWorkflowSchedule, - SeerWorkflowStrategy, - WorkflowRunSnapshot, -) -from sentry.seer.workflows.runs import SeerWorkflowRunLifecycle -from sentry.service_accounts.models import ServiceAccountReplica -from sentry.utils import json - -logger = logging.getLogger(__name__) - -SEER_DEFAULT_WORKFLOW_STRATEGIES = (SeerWorkflowStrategy.AGENTIC_TRIAGE_V2,) - - -class WorkflowUpdateError(ValueError): - """A rejected workflow update whose message is safe to show users.""" - - -@dataclass(frozen=True) -class AllProjects: - pass - - -@dataclass(frozen=True) -class SelectedProjects: - project_ids: tuple[int, ...] - - -type ProjectSelection = AllProjects | SelectedProjects - - -@dataclass(frozen=True, kw_only=True) -class SeerWorkflow: - """Apply each change to fresh state under the workflow config's row lock. - - Returned models do not carry a lock or refresh automatically. Configuration - changes, including project selection, should go through these methods. - """ - - config_id: int - organization_id: int - - @classmethod - def ensure_defaults(cls, organization_id: int) -> None: - """Seed defaults for strategies with no config; disabled configs count as existing.""" - database = router.db_for_write(SeerWorkflowConfig) - configs = SeerWorkflowConfig.objects.using(database).filter( - organization_id=organization_id, - strategy__in=[s.value for s in SEER_DEFAULT_WORKFLOW_STRATEGIES], - ) - existing = set(configs.values_list("strategy", flat=True)) - if all(s.value in existing for s in SEER_DEFAULT_WORKFLOW_STRATEGIES): - return - - with transaction.atomic(database): - # Prevent duplicate seeds without a uniqueness constraint on configurable strategies. - list( - Organization.objects.using(database).select_for_update().filter(id=organization_id) - ) - existing = set(configs.values_list("strategy", flat=True)) - SeerWorkflowConfig.objects.using(database).bulk_create( - [ - SeerWorkflowConfig( - organization_id=organization_id, - strategy=strategy.value, - filters=SeerWorkflowFilters(all_projects=True).dict(), - extras=SeerWorkflowConfigExtras(seeded_default=strategy).dict( - exclude_none=True - ), - ) - for strategy in SEER_DEFAULT_WORKFLOW_STRATEGIES - if strategy.value not in existing - ] - ) - - @classmethod - def _create_config( - cls, - *, - database: str, - organization_id: int, - strategy: SeerWorkflowStrategy, - enabled: bool, - filters: SeerWorkflowFilters, - name: str | None = None, - ) -> SeerWorkflowConfig: - """ - Create a config within the org's automation limit. Call inside a transaction on - ``database`` so the limit check and any related writes are atomic. - """ - existing = ( - SeerWorkflowConfig.objects.using(database) - .filter( - organization_id=organization_id, - strategy__in=SEER_CONFIGURABLE_WORKFLOW_STRATEGIES, - ) - .count() - ) - if existing >= MAX_WORKFLOW_CONFIGS_PER_ORG: - raise WorkflowUpdateError( - f"Organizations can have at most {MAX_WORKFLOW_CONFIGS_PER_ORG} automations" - ) - - return SeerWorkflowConfig.objects.using(database).create( - organization_id=organization_id, - strategy=strategy, - enabled=enabled, - filters=filters.dict(), - extras=SeerWorkflowConfigExtras(name=name).dict(exclude_none=True), - ) - - @classmethod - def create( - cls, - *, - organization_id: int, - strategy: SeerWorkflowStrategy, - enabled: bool = False, - schedule: SeerWorkflowSchedule, - project_ids: Collection[int], - environments: Sequence[str] = (), - name: str | None = None, - ) -> SeerWorkflow: - """ - Create a user-defined automation. Use ``project_ids=[-1]`` to select all - projects and ``environments=[]`` for all environments. - """ - if strategy not in SEER_CONFIGURABLE_WORKFLOW_STRATEGIES: - raise WorkflowUpdateError(f"{strategy} automations cannot be configured") - if strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - raise ValueError("Use create_with_monitors for metric workflows") - - project_set = set(project_ids) - all_projects = project_set == ALL_ACCESS_PROJECTS - if all_projects: - # [-1] is not stored as project links, only in config.filters - project_set = set() - cls._validate_project_ids(organization_id, project_set) - - database = router.db_for_write(SeerWorkflowConfig) - with transaction.atomic(database): - config = cls._create_config( - database=database, - organization_id=organization_id, - strategy=strategy, - enabled=enabled, - filters=SeerWorkflowFilters( - all_projects=all_projects, - environments=list(environments), - ), - name=name, - ) - config.schedule = schedule - config.save(using=database, update_fields=["schedule"]) - if project_set: - SeerWorkflowConfigProject.objects.using(database).bulk_create( - [ - SeerWorkflowConfigProject(config=config, project_id=project_id) - for project_id in sorted(project_set) - ] - ) - transaction.on_commit( - partial( - logger.info, - "seer.workflow.created", - extra={ - "organization_id": organization_id, - "config_id": config.id, - "config_version": config.version, - "strategy": config.strategy, - "all_projects": all_projects, - "project_ids": list(project_set), - "project_count": len(project_set), - }, - ), - using=database, - ) - return cls(config_id=config.id, organization_id=organization_id) - - def get_config(self) -> SeerWorkflowConfig: - """Fetch the config without acquiring a lock.""" - return self._config_queryset().get() - - @property - def extras(self) -> SeerWorkflowConfigExtras: - return SeerWorkflowConfigExtras.parse_obj(self.get_config().extras) - - @property - def filters(self) -> SeerWorkflowFilters: - return SeerWorkflowFilters.parse_obj(self.get_config().filters) - - def get_or_create_run( - self, - *, - trigger_key: str, - inputs: BaseModel, - extras: Mapping[str, Any] | None = None, - ) -> WorkflowRunCreationResult: - """Retries preserve the original configuration and inputs.""" - if not trigger_key or len(trigger_key) > 256: - raise ValueError("Trigger keys must contain between 1 and 256 characters") - - with self._lock_config() as config: - existing = self.get_runs().filter(trigger_key=trigger_key).first() - if existing is not None: - return WorkflowRunCreationResult( - lifecycle=SeerWorkflowRunLifecycle( - run_id=existing.id, organization_id=existing.organization_id - ), - created=False, - ) - - snapshot = WorkflowRunSnapshot( - config_version=config.version, - strategy=SeerWorkflowStrategy(config.strategy), - filters=config.filters, - project_ids=tuple(self._get_project_ids(config)), - output_policy=config.output_policy, - inputs=json.loads(inputs.json()), - ) - run_extras = SeerWorkflowRunExtras.parse_obj({**(extras or {}), "snapshot": snapshot}) - run = SeerWorkflowRun.objects.create( - organization_id=config.organization_id, - workflow_config=config, - trigger_key=trigger_key, - status=SeerWorkflowRunStatus.QUEUED, - extras=json.loads(run_extras.json()), - ) - return WorkflowRunCreationResult( - lifecycle=SeerWorkflowRunLifecycle( - run_id=run.id, organization_id=run.organization_id - ), - created=True, - ) - - def get_runs(self) -> QuerySet[SeerWorkflowRun]: - return ( - SeerWorkflowRun.objects.using(router.db_for_write(SeerWorkflowRun)) - .filter(organization_id=self.organization_id, workflow_config_id=self.config_id) - .order_by("-date_added", "-id") - ) - - def matches_scope(self, project_id: int, environment: str | None) -> bool: - """Check the automation's scope before recording an admission decision.""" - return self._matches_scope(self.get_config(), project_id, environment) - - def has_quota_available(self) -> bool: - """Allow runs until automation quota enforcement is implemented.""" - # TODO(telkins): Enforce automation quota before accepting a new run. - return True - - def get_project_ids(self) -> list[int]: - """Return active project IDs matching this workflow's filters, in ascending order.""" - return self._get_project_ids(self.get_config()) - - def has_all_projects(self) -> bool: - """ - Return whether this workflow's scope covers every active project in its organization. - Edge case: always false for empty selections, even if the org has no active projects. - """ - config = self.get_config() - selected_projects = self._get_projects(config).values("id") - counts = Project.objects.filter( - organization_id=config.organization_id, status=ObjectStatus.ACTIVE - ).aggregate( - total=Count("id"), - selected=Count("id", filter=Q(id__in=selected_projects)), - ) - return counts["selected"] > 0 and counts["selected"] == counts["total"] - - def set_service_account(self, service_account_id: int | None) -> None: - """Store the control account ID after validating its current cell replica.""" - with self._lock_config() as config: - if service_account_id is not None: - try: - replica = ServiceAccountReplica.objects.get( - service_account_id=service_account_id, - organization_id=config.organization_id, - ) - except ServiceAccountReplica.DoesNotExist as exc: - raise ValueError( - "Service account is unavailable in the workflow's organization" - ) from exc - if not replica.is_active: - raise ValueError("Service account is inactive") - - if config.service_account_id != service_account_id: - config.service_account_id = service_account_id - config.version += 1 - config.save(update_fields=["service_account_id", "version", "date_updated"]) - - def resolve_service_account(self) -> ServiceAccountReplica: - """Resolve authoritative identity against current cell state on every call.""" - config = self.get_config() - if config.service_account_id is None: - raise ValueError("Workflow has no service account") - try: - replica = ServiceAccountReplica.objects.get( - service_account_id=config.service_account_id, - organization_id=config.organization_id, - ) - except ServiceAccountReplica.DoesNotExist as exc: - raise ValueError( - "Service account is unavailable in the workflow's organization" - ) from exc - if not replica.is_active: - raise ValueError("Service account is inactive") - return replica - - def set_project_selection(self, selection: ProjectSelection) -> None: - """Replace the selected projects, bumping the version only when the selection changes. - - Callers must authorize the change; this only checks that the projects belong to the - workflow's organization. - """ - match selection: - case AllProjects(): - all_projects = True - project_ids: set[int] = set() - case SelectedProjects(project_ids=selected): - all_projects = False - project_ids = set(selected) - case _: - assert_never(selection) - - with self._lock_config() as config: - if config.strategy == SeerWorkflowStrategy.BREACHED_METRIC_INVESTIGATION: - raise WorkflowUpdateError( - "Breached metric projects are selected by connecting monitors" - ) - self._validate_project_ids(config.organization_id, project_ids) - - existing = set( - SeerWorkflowConfigProject.objects.filter(config=config).values_list( - "project_id", flat=True - ) - ) - removed = existing - project_ids - added = project_ids - existing - filters = SeerWorkflowFilters.parse_obj(config.filters) - if not removed and not added and filters.all_projects == all_projects: - return - - SeerWorkflowConfigProject.objects.filter(config=config, project_id__in=removed).delete() - SeerWorkflowConfigProject.objects.bulk_create( - [ - SeerWorkflowConfigProject(config=config, project_id=project_id) - for project_id in sorted(added) - ] - ) - filters.all_projects = all_projects - config.filters = config.filters | filters.dict() - config.version += 1 - config.save(update_fields=["filters", "version", "date_updated"]) - - @contextmanager - def _lock_config(self) -> Iterator[SeerWorkflowConfig]: - with transaction.atomic(router.db_for_write(SeerWorkflowConfig)): - yield self._config_queryset().select_for_update().get() - - def _config_queryset(self) -> QuerySet[SeerWorkflowConfig]: - return SeerWorkflowConfig.objects.using(router.db_for_write(SeerWorkflowConfig)).filter( - id=self.config_id, organization_id=self.organization_id - ) - - def _matches_scope( - self, config: SeerWorkflowConfig, project_id: int, environment: str | None - ) -> bool: - """Check scope using a config snapshot without modifying it or acquiring a lock.""" - filters = SeerWorkflowFilters.parse_obj(config.filters) - return ( - config.enabled - and (not filters.environments or environment in filters.environments) - and self._get_projects(config).filter(id=project_id).exists() - ) - - def _get_project_ids(self, config: SeerWorkflowConfig) -> list[int]: - return list(self._get_projects(config).order_by("id").values_list("id", flat=True)) - - @classmethod - def _validate_project_ids(cls, organization_id: int, project_ids: set[int]) -> None: - valid_project_ids = set( - Project.objects.filter(organization_id=organization_id, id__in=project_ids).values_list( - "id", flat=True - ) - ) - if project_ids != valid_project_ids: - raise WorkflowUpdateError("Selected projects must exist in the workflow's organization") - - def _get_projects(self, config: SeerWorkflowConfig) -> QuerySet[Project]: - """Return active projects in the config's scope; strategies may derive it differently.""" - projects = Project.objects.filter( - organization_id=config.organization_id, status=ObjectStatus.ACTIVE - ) - if not SeerWorkflowFilters.parse_obj(config.filters).all_projects: - projects = projects.filter( - id__in=SeerWorkflowConfigProject.objects.filter(config=config).values("project_id") - ) - return projects - - -@dataclass(frozen=True) -class WorkflowRunCreationResult: - lifecycle: SeerWorkflowRunLifecycle - created: bool diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py deleted file mode 100644 index d9ca1f347..000000000 --- a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py +++ /dev/null @@ -1,1533 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Collection, Iterable, Mapping -from dataclasses import dataclass -from functools import cached_property -from typing import Any - -from django.conf import settings -from django.contrib.auth.models import AnonymousUser -from django.http.request import HttpRequest -from rest_framework.request import Request -from sentry_sdk import traces - -from sentry import features, roles -from sentry.api.exceptions import DataSecrecyError -from sentry.auth.deprecated_scope_translation import translate_deprecated_scopes -from sentry.auth.principal import has_user -from sentry.auth.scope_declaration import check_scope_declaration, check_scope_declarations -from sentry.auth.services.access.service import access_service -from sentry.auth.services.auth import AuthenticatedToken, RpcAuthState, RpcMemberSsoState -from sentry.auth.staff import is_active_staff -from sentry.auth.superuser import ( - Superuser, - get_superuser_access_expiry, - get_superuser_scopes, - is_active_superuser, - resolve_superuser_access, -) -from sentry.auth.system import is_system_auth -from sentry.constants import ObjectStatus -from sentry.data_secrecy.logic import should_allow_superuser_access -from sentry.models.apiscopes import add_scope_hierarchy -from sentry.models.organization import Organization -from sentry.models.organizationmember import OrganizationMember -from sentry.models.organizationmemberteam import OrganizationMemberTeam -from sentry.models.project import Project -from sentry.models.team import Team, TeamStatus -from sentry.organizations.services.organization import ( - RpcTeamMember, - RpcUserOrganizationContext, - organization_service, -) -from sentry.organizations.services.organization.serial import summarize_member -from sentry.roles import organization_roles -from sentry.roles.manager import OrganizationRole, TeamRole -from sentry.seer.agent_token import is_agent_auth -from sentry.sentry_apps.models.sentry_app import SentryApp -from sentry.service_accounts.auth import is_service_account_auth -from sentry.service_accounts.models import ServiceAccountProjectReplica, ServiceAccountReplica -from sentry.types.request import _HttpRequestWithUser -from sentry.users.models.user import User -from sentry.users.services.user import RpcUser -from sentry.users.services.user.service import user_service -from sentry.utils import metrics -from sentry.viewer_context import SuperuserAccess, set_viewer_context_superuser - -__all__ = ( - "from_user", - "from_member", - "DEFAULT", - "from_user_and_rpc_user_org_context", - "from_rpc_member", -) - - -def has_role_in_organization(role: str, organization: Organization, user_id: int) -> bool: - query = OrganizationMember.objects.filter( - user_is_active=True, - user_id=user_id, - organization_id=organization.id, - role=role, - ) - return query.exists() - - -class Access(abc.ABC): - @property - @abc.abstractmethod - def sso_is_valid(self) -> bool: - pass - - @property - @abc.abstractmethod - def requires_sso(self) -> bool: - pass - - @property - @abc.abstractmethod - def has_open_membership(self) -> bool: - pass - - @property - @abc.abstractmethod - def has_global_access(self) -> bool: - pass - - @property - @abc.abstractmethod - def scopes(self) -> frozenset[str]: - pass - - @property - @abc.abstractmethod - def permissions(self) -> frozenset[str]: - pass - - # TODO(cathy): remove this - @property - @abc.abstractmethod - def role(self) -> str | None: - pass - - @property - @abc.abstractmethod - def team_ids_with_membership(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def accessible_team_ids(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def project_ids_with_team_membership(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def accessible_project_ids(self) -> frozenset[int]: - pass - - @property - def is_integration_token(self) -> bool: - return False - - def has_permission(self, permission: str) -> bool: - """ - Return bool representing if the user has the given permission. - >>> access.has_permission('broadcasts.admin') - """ - return permission in self.permissions - - def has_scope(self, scope: str) -> bool: - """ - Return bool representing if the user has the given scope. - >>> access.has_project('org:read') - """ - check_scope_declaration(scope) - return scope in self.scopes - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - """Whether adding ``scope`` to the current auth scope cap would grant it.""" - check_scope_declaration(scope) - return False - - def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: - """Whether adding ``scope`` to the current auth scope cap would grant it for a project.""" - check_scope_declaration(scope) - return False - - def get_organization_role(self) -> OrganizationRole | None: - if self.role is not None: - return organization_roles.get(self.role) - return None - - @abc.abstractmethod - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - pass - - @abc.abstractmethod - def has_team_access(self, team: Team) -> bool: - """ - Return bool representing if a user should have access to information for the given team. - >>> access.has_team_access(team) - """ - - @abc.abstractmethod - def has_team_scope(self, team: Team, scope: str) -> bool: - pass - - def has_team_membership(self, team: Team) -> bool: - return team.id in self.team_ids_with_membership - - @abc.abstractmethod - def get_team_role(self, team: Team) -> TeamRole | None: - pass - - @abc.abstractmethod - def has_project_access(self, project: Project) -> bool: - """ - Return bool representing if a user should have access to information for the given project. - >>> access.has_project_access(project) - """ - raise NotImplementedError - - def has_projects_access(self, projects: Iterable[Project]) -> bool: - """ - Returns bool representing if a user should have access to every requested project - """ - return all([self.has_project_access(project) for project in projects]) - - def has_project_membership(self, project: Project) -> bool: - """ - Return bool representing if a user has explicit membership for the given project. - >>> access.has_project_membership(project) - """ - return project.id in self.project_ids_with_team_membership - - def has_project_scope(self, project: Project, scope: str) -> bool: - """ - Return bool representing if a user should have access with the given scope to information - for the given project. - - >>> access.has_project_scope(project, 'project:read') - """ - return self.has_any_project_scope(project, [scope]) - - @abc.abstractmethod - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - pass - - -def _intersect_member_and_token_scopes( - member_scopes: Collection[str], token_scopes: Iterable[str] | None -) -> frozenset[str]: - member_scopes = frozenset(member_scopes) - if token_scopes is None: - return member_scopes - - token_scopes = frozenset(token_scopes) - token_only_scopes = token_scopes & settings.SENTRY_TOKEN_ONLY_SCOPES - - return (token_scopes & member_scopes) | token_only_scopes - - -@dataclass -class DbAccess(Access): - # TODO(dcramer): this is still a little gross, and ideally backend access - # would be based on the same scopes as API access so there's clarity in - # what things mean - - sso_is_valid: bool = False - requires_sso: bool = False - has_open_membership: bool = False - - # if has_global_access is True, then any project - # matching organization_id is valid. This is used for - # both `organization.allow_joinleave` and to indicate - # that the role is global / a user is an active superuser - has_global_access: bool = False - - scopes: frozenset[str] = frozenset() - scopes_upper_bound: frozenset[str] | None = None - permissions: frozenset[str] = frozenset() - - _member: OrganizationMember | None = None - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - return self._member.role if self._member else None - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - if self._member is None or self.scopes_upper_bound is None: - return False - candidate_scopes = _intersect_member_and_token_scopes( - self._member.get_scopes(), self.scopes_upper_bound | {scope} - ) - return scope in add_scope_hierarchy(list(candidate_scopes)) - - @cached_property - def _team_memberships(self) -> Mapping[Team, OrganizationMemberTeam]: - if self._member is None: - return {} - return { - omt.team: omt - for omt in OrganizationMemberTeam.objects.filter( - organizationmember=self._member, is_active=True, team__status=TeamStatus.ACTIVE - ).select_related("team") - } - - @cached_property - def team_ids_with_membership(self) -> frozenset[int]: - """Return the IDs of teams in which the user has actual membership. - - This represents the set of all teams for which `has_team_membership` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - - Compare to accessible_team_ids, which is equal to this property in the - typical case but represents a superset of IDs in case of superuser access. - """ - return frozenset(team.id for team in self._team_memberships.keys()) - - @property - def accessible_team_ids(self) -> frozenset[int]: - """Return the IDs of teams to which the user has access. - - This represents the set of all teams for which `has_team_access` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - """ - return self.team_ids_with_membership - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - """Return the IDs of projects to which the user has access via actual team membership. - - This represents the set of all projects for which `has_project_membership` - returns true. Use that method where possible and use this property only when - you need to iterate or query for all such teams. - - Compare to accessible_project_ids, which is equal to this property in the - typical case but represents a superset of IDs in case of superuser access. - """ - teams = self._team_memberships.keys() - if not teams: - return frozenset() - - with traces.start_span( - name="get_project_access_in_teams", - attributes={"sentry.op": "get_project_access_in_teams"}, - ) as span: - projects = frozenset( - Project.objects.filter(status=ObjectStatus.ACTIVE, teams__in=teams) - .distinct() - .values_list("id", flat=True) - ) - span.set_attribute("Project Count", len(projects)) - span.set_attribute("Team Count", len(teams)) - - return projects - - @property - def accessible_project_ids(self) -> frozenset[int]: - """Return the IDs of projects to which the user has access. - - This represents the set of all teams for which `has_project_access` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - """ - return self.project_ids_with_team_membership - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - if self._member and self._member.user_id is not None: - return has_role_in_organization( - role=role, organization=organization, user_id=self._member.user_id - ) - return False - - def has_team_scope(self, team: Team, scope: str) -> bool: - """ - Return bool representing if a user should have access with the given scope to information - for the given team. - - >>> access.has_team_scope(team, 'team:read') - """ - check_scope_declaration(scope) - if not self.has_team_access(team): - return False - if self.has_scope(scope): - return True - - membership = self._team_memberships.get(team) - if not membership: - return False - - team_scopes = membership.get_scopes() - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - if membership and scope in team_scopes: - metrics.incr( - "team_roles.pass_by_team_scope", - tags={"team_role": membership.role, "scope": scope}, - ) - return True - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - team_member = self._team_memberships.get(team) - if team_member: - return team_member.get_team_role() - else: - return None - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - """ - Represent if a user should have access with any one of the given scopes to - information for the given project. - - For performance's sake, prefer this over multiple calls to `has_project_scope`. - """ - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - if any(self.has_scope(scope) for scope in scopes): - return True - - if self._member and features.has("organizations:team-roles", self._member.organization): - with traces.start_span( - name="check_access_for_all_project_teams", - attributes={"sentry.op": "check_access_for_all_project_teams"}, - ) as span: - memberships = [ - self._team_memberships[team] - for team in project.teams.all() - if team in self._team_memberships - ] - span.set_attribute("organization", self._member.organization.id) - span.set_attribute("organization.slug", self._member.organization.slug) - span.set_attribute("membership_count", len(memberships)) - - for membership in memberships: - team_scopes = membership.get_scopes() - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - for scope in scopes: - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_project_scope", - tags={"team_role": membership.role, "scope": scope}, - ) - return True - return False - - -@dataclass -class SingularRpcAccessOrgOptimization: - access: RpcBackedAccess - - -def maybe_singular_rpc_access_org_context( - access: Access, org_ids: set[int] -) -> SingularRpcAccessOrgOptimization | None: - if ( - isinstance(access, RpcBackedAccess) - and len(org_ids) == 1 - and access.rpc_user_organization_context.organization.id in org_ids - ): - return SingularRpcAccessOrgOptimization(access) - return None - - -maybe_singular_api_access_org_context = maybe_singular_rpc_access_org_context - - -@dataclass -class RpcBackedAccess(Access): - rpc_user_organization_context: RpcUserOrganizationContext - scopes_upper_bound: frozenset[str] | None - auth_state: RpcAuthState - - # TODO: remove once getsentry has updated to use the new names. - @property - def api_user_organization_context(self) -> RpcUserOrganizationContext: - return self.rpc_user_organization_context - - @cached_property - def permissions(self) -> frozenset[str]: - return frozenset(self.auth_state.permissions) - - @property - def sso_is_valid(self) -> bool: - return self.auth_state.sso_state.is_valid - - @property - def requires_sso(self) -> bool: - return self.auth_state.sso_state.is_required - - @property - def has_open_membership(self) -> bool: - return self.rpc_user_organization_context.organization.flags.allow_joinleave - - @property - def has_global_access(self) -> bool: - if self.has_open_membership: - return True - - if ( - self.rpc_user_organization_context.member - and roles.get(self.rpc_user_organization_context.member.role).is_global - ): - return True - - return False - - @cached_property - def scopes(self) -> frozenset[str]: - if self.rpc_user_organization_context.member is None: - return frozenset(self.scopes_upper_bound or []) - - if self.scopes_upper_bound is None: - return frozenset(self.rpc_user_organization_context.member.scopes) - - return _intersect_member_and_token_scopes( - self.rpc_user_organization_context.member.scopes, - self.scopes_upper_bound, - ) - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - member = self.rpc_user_organization_context.member - if member is None or self.scopes_upper_bound is None: - return False - candidate_scopes = _intersect_member_and_token_scopes( - member.scopes, self.scopes_upper_bound | {scope} - ) - return scope in add_scope_hierarchy(list(candidate_scopes)) - - def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: - """ - Return whether adding a scope to the token would grant access for this project. - - Check both organization-level scopes and scopes from the member's role on - a project team. This does not modify the token or access instance. - """ - if not self.has_project_access(project): - return False - if self.would_have_scope_with_added_auth_scope(scope): - return True - - # Team roles can only grant project scopes for member-backed token access. - member = self.rpc_user_organization_context.member - if ( - member is None - or self.scopes_upper_bound is None - or not features.has( - "organizations:team-roles", self.rpc_user_organization_context.organization - ) - ): - return False - - # A role on any team linked to the project may grant the requested scope... - project_team_ids = set(project.teams.values_list("id", flat=True)) - return any( - member_team.role - and member_team.team_id in project_team_ids - and scope in member_team.role.scopes - for member_team in member.member_teams - ) - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - if self.rpc_user_organization_context.member is None: - return None - return self.rpc_user_organization_context.member.role - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - member = self.rpc_user_organization_context.member - if member and member.user_id: - return has_role_in_organization( - role=role, organization=organization, user_id=member.user_id - ) - return False - - @cached_property - def team_ids_with_membership(self) -> frozenset[int]: - if self.rpc_user_organization_context.member is None: - return frozenset() - return frozenset( - mt.team_id for mt in self.rpc_user_organization_context.member.member_teams - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return self.team_ids_with_membership - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - if self.rpc_user_organization_context.member is None: - return frozenset() - return frozenset(self.rpc_user_organization_context.member.project_ids) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return self.project_ids_with_team_membership - - def has_team_access(self, team: Team) -> bool: - if team.status != TeamStatus.ACTIVE: - return False - if ( - self.has_global_access - and self.rpc_user_organization_context.organization.id == team.organization_id - ): - return True - return team.id in self.team_ids_with_membership - - def get_team_membership(self, team_id: int) -> RpcTeamMember | None: - if self.rpc_user_organization_context.member is None: - return None - - for team_membership in self.rpc_user_organization_context.member.member_teams: - if team_membership.team_id == team_id: - return team_membership - return None - - def has_team_scope(self, team: Team, scope: str) -> bool: - check_scope_declaration(scope) - if not self.has_team_access(team): - return False - if self.has_scope(scope): - return True - - if self.rpc_user_organization_context.member is None: - return False - - team_membership = self.get_team_membership(team.id) - if not team_membership: - return False - - team_scopes = frozenset(team_membership.scopes) - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_team_scope", - tags={"team_role": f"{team_membership.role}", "scope": scope}, - ) - return True - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - team_member = self.get_team_membership(team.id) - if team_member: - return team_member.role - return None - - def has_project_access(self, project: Project) -> bool: - if project.status != ObjectStatus.ACTIVE: - return False - if ( - self.has_global_access - and self.rpc_user_organization_context.member - and self.rpc_user_organization_context.organization.id == project.organization_id - ): - return True - return project.id in self.project_ids_with_team_membership - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - """ - Represent if a user should have access with any one of the given scopes to - information for the given project. - - For performance's sake, prefer this over multiple calls to `has_project_scope`. - """ - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - if any(self.has_scope(scope) for scope in scopes): - return True - - if self.rpc_user_organization_context.member and features.has( - "organizations:team-roles", self.rpc_user_organization_context.organization - ): - with traces.start_span( - name="check_access_for_all_project_teams", - attributes={"sentry.op": "check_access_for_all_project_teams"}, - ) as span: - project_teams_id = set(project.teams.values_list("id", flat=True)) - orgmember_teams = self.rpc_user_organization_context.member.member_teams - span.set_attribute( - "organization", self.rpc_user_organization_context.organization.id - ) - span.set_attribute( - "organization.slug", self.rpc_user_organization_context.organization.slug - ) - span.set_attribute("membership_count", len(orgmember_teams)) - - for member_team in orgmember_teams: - if not member_team.role: - continue - if member_team.team_id not in project_teams_id: - continue - - team_scopes = member_team.role.scopes - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - for scope in scopes: - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_project_scope", - tags={"team_role": f"{member_team.role.id}", "scope": scope}, - ) - return True - return False - - -def _wrap_scopes(scopes_upper_bound: Iterable[str] | None) -> frozenset[str] | None: - if scopes_upper_bound is not None: - return frozenset(scopes_upper_bound) - return None - - -class OrganizationMemberAccess(DbAccess): - def __init__( - self, - member: OrganizationMember, - scopes: Iterable[str], - permissions: Iterable[str], - scopes_upper_bound: Iterable[str] | None, - ) -> None: - auth_state = access_service.get_user_auth_state( - organization_id=member.organization_id, - is_superuser=False, - is_staff=False, - org_member=summarize_member(member), - user_id=member.user_id, - ) - sso_state = auth_state.sso_state - has_global_access = ( - bool(member.organization.flags.allow_joinleave) or roles.get(member.role).is_global - ) - - super().__init__( - _member=member, - sso_is_valid=sso_state.is_valid, - requires_sso=sso_state.is_required, - has_global_access=has_global_access, - scopes=frozenset(scopes), - permissions=frozenset(permissions), - scopes_upper_bound=_wrap_scopes(scopes_upper_bound), - ) - - def has_team_access(self, team: Team) -> bool: - assert self._member is not None - if team.status != TeamStatus.ACTIVE: - return False - if self.has_global_access and self._member.organization.id == team.organization_id: - return True - return team.id in self.team_ids_with_membership - - def has_project_access(self, project: Project) -> bool: - assert self._member is not None - if project.status != ObjectStatus.ACTIVE: - return False - if self.has_global_access and self._member.organization.id == project.organization_id: - return True - return project.id in self.project_ids_with_team_membership - - -class OrganizationGlobalAccess(DbAccess): - """Access to all an organization's teams and projects.""" - - def __init__( - self, organization: Organization | int, scopes: Iterable[str], **kwargs: Any - ) -> None: - self._organization_id = ( - organization.id if isinstance(organization, Organization) else organization - ) - super().__init__(has_global_access=True, scopes=frozenset(scopes), **kwargs) - - def has_team_access(self, team: Team) -> bool: - return bool( - team.organization_id == self._organization_id and team.status == TeamStatus.ACTIVE - ) - - def has_project_access(self, project: Project) -> bool: - return bool( - project.organization_id == self._organization_id - and project.status == ObjectStatus.ACTIVE - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset( - Team.objects.filter( - organization_id=self._organization_id, status=TeamStatus.ACTIVE - ).values_list("id", flat=True) - ) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset( - Project.objects.filter( - organization_id=self._organization_id, status=ObjectStatus.ACTIVE - ).values_list("id", flat=True) - ) - - -class ServiceAccountAccess(DbAccess): - """ - A service account's access to its organization. It has no membership, so its - scopes are the account's own. With `all_projects` it reaches every active project - in the org. Without it, it reaches only the active projects linked to the - account. It never reaches teams. Open membership is a setting for people, so - it doesn't apply. - """ - - def __init__( - self, - *, - service_account_id: int, - organization_id: int, - scopes: Iterable[str], - all_projects: bool, - ) -> None: - self._service_account_id = service_account_id - self._organization_id = organization_id - super().__init__( - sso_is_valid=True, - has_global_access=all_projects, - scopes=frozenset(scopes), - ) - - def has_team_access(self, team: Team) -> bool: - return False - - def has_project_access(self, project: Project) -> bool: - if ( - project.organization_id != self._organization_id - or project.status != ObjectStatus.ACTIVE - ): - return False - return self.has_global_access or project.id in self.accessible_project_ids - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - # With no teams, the account's project links stand in for membership, so - # endpoints that default to "my projects" see the projects it can reach. - return self.accessible_project_ids - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset() - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - if self.has_global_access: - return frozenset( - Project.objects.filter( - organization_id=self._organization_id, status=ObjectStatus.ACTIVE - ).values_list("id", flat=True) - ) - # A replica link can briefly outlive a transfer of its project to another org. - return frozenset( - ServiceAccountProjectReplica.objects.filter( - service_account__service_account_id=self._service_account_id, - project__organization_id=self._organization_id, - project__status=ObjectStatus.ACTIVE, - ).values_list("project_id", flat=True) - ) - - -class ApiBackedOrganizationGlobalAccess(RpcBackedAccess): - """Access to all an organization's teams and projects.""" - - def __init__( - self, - *, - rpc_user_organization_context: RpcUserOrganizationContext, - auth_state: RpcAuthState, - scopes: Iterable[str] | None, - ): - super().__init__( - rpc_user_organization_context=rpc_user_organization_context, - auth_state=auth_state, - scopes_upper_bound=_wrap_scopes(scopes), - ) - - @cached_property - def scopes(self) -> frozenset[str]: - return frozenset(self.scopes_upper_bound or []) - - @property - def has_global_access(self) -> bool: - return True - - def has_team_access(self, team: Team) -> bool: - return bool( - team.organization_id == self.rpc_user_organization_context.organization.id - and team.status == TeamStatus.ACTIVE - ) - - def has_project_access(self, project: Project) -> bool: - return bool( - project.organization_id == self.rpc_user_organization_context.organization.id - and project.status == ObjectStatus.ACTIVE - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset( - t.id - for t in self.rpc_user_organization_context.organization.teams - if t.status == TeamStatus.ACTIVE - ) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset( - p.id - for p in self.rpc_user_organization_context.organization.projects - if p.status == ObjectStatus.ACTIVE - ) - - -class OrganizationGlobalMembership(OrganizationGlobalAccess): - """Access to all an organization's teams and projects with simulated membership.""" - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return self.accessible_team_ids - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return self.accessible_project_ids - - def has_team_membership(self, team: Team) -> bool: - return self.has_team_access(team) - - def has_project_membership(self, project: Project) -> bool: - return self.has_project_access(project) - - -class ApiOrganizationGlobalMembership(ApiBackedOrganizationGlobalAccess): - """Access to all an organization's teams and projects with simulated membership.""" - - @property - def is_integration_token(self) -> bool: - return True - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return self.accessible_team_ids - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return self.accessible_project_ids - - def has_team_membership(self, team: Team) -> bool: - return self.has_team_access(team) - - def has_project_membership(self, project: Project) -> bool: - return self.has_project_access(project) - - -@dataclass -class OrganizationlessAccess(Access): - auth_state: RpcAuthState - - @cached_property - def permissions(self) -> frozenset[str]: - return frozenset(self.auth_state.permissions) - - def has_team_access(self, team: Team) -> bool: - return False - - def has_project_access(self, project: Project) -> bool: - return False - - @property - def sso_is_valid(self) -> bool: - return self.auth_state.sso_state.is_valid - - @property - def requires_sso(self) -> bool: - return self.auth_state.sso_state.is_required - - @property - def has_open_membership(self) -> bool: - return False - - @property - def has_global_access(self) -> bool: - return False - - @property - def scopes(self) -> frozenset[str]: - return frozenset() - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - return None - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - if user_id: - return has_role_in_organization(role=role, organization=organization, user_id=user_id) - return False - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset() - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset() - - def has_team_scope(self, team: Team, scope: str) -> bool: - check_scope_declaration(scope) - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - return None - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - - return any(self.has_scope(scope) for scope in scopes) - - -class SystemAccess(OrganizationlessAccess): - def __init__(self) -> None: - super().__init__( - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_required=False, is_valid=False), - permissions=[], - ), - ) - - @property - def has_global_access(self) -> bool: - return True - - def has_permission(self, permission: str) -> bool: - return True - - def has_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - return True - - def has_team_access(self, team: Team) -> bool: - return True - - def has_project_access(self, project: Project) -> bool: - return True - - # The semantically correct behavior for accessible_(team|project)_ids would be to - # query for all teams or projects in the system, which we don't want to attempt. - # Code paths that may have SystemAccess must avoid looking at these properties. - @property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset() - - -class NoAccess(OrganizationlessAccess): - def __init__(self) -> None: - super().__init__( - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_required=False, is_valid=True), - permissions=[], - ), - ) - - -def _propagate_superuser_access( - request: _HttpRequestWithUser, - org_context: RpcUserOrganizationContext, - su: Superuser | None, -) -> None: - # Members use their normal callback permissions and user flags; adding this - # claim would instead force read-only access and a five-minute VC lifetime. - if org_context.member is not None: - return - if request.auth is not None or su is None or not su.is_active: - return - if su.requires_org_auth(org_context.organization): - return - expires_at = get_superuser_access_expiry(su, org_context.organization.id) - if expires_at is not None: - set_viewer_context_superuser( - user_id=request.user.id, - organization_id=org_context.organization.id, - superuser=SuperuserAccess(expires_at=expires_at), - ) - - -def from_request_org_and_scopes( - *, - request: HttpRequest, - rpc_user_org_context: RpcUserOrganizationContext | None = None, - scopes: Iterable[str] | None = None, -) -> Access: - """ - Note that `scopes` is usually None because request.auth is not set at `get_authorization_header` - when the request is made from the frontend using cookies - """ - if isinstance(request.auth, AuthenticatedToken) and request.auth.superuser is not None: - if rpc_user_org_context is None: - return DEFAULT - return from_superuser_access(request.auth, rpc_user_org_context) - if is_agent_auth(request.auth): - if rpc_user_org_context is None: - return DEFAULT - return from_agent_auth(request.auth, rpc_user_org_context) - - is_staff = is_active_staff(request) - - if not rpc_user_org_context: - return from_user_and_rpc_user_org_context( - user=request.user, - rpc_user_org_context=rpc_user_org_context, - is_superuser=is_active_superuser(request), - is_staff=is_staff, - scopes=scopes, - ) - - if getattr(request, "actual_user", None) is not None: - if not should_allow_superuser_access(rpc_user_org_context): - raise DataSecrecyError() - - if getattr(request.user, "is_sentry_app", False): - return _from_rpc_sentry_app(rpc_user_org_context) - - if is_active_superuser(request): - su = getattr(request, "superuser", None) - if su is None and request.auth is None: - su = Superuser(request) - requires_org_auth = su is not None and su.requires_org_auth( - rpc_user_org_context.organization - ) - if requires_org_auth and rpc_user_org_context.member is None: - setattr(request, "_superuser_needs_org_auth", rpc_user_org_context.organization.slug) - - member = rpc_user_org_context.member - auth_state = access_service.get_user_auth_state( - user_id=request.user.id, - organization_id=rpc_user_org_context.organization.id, - is_superuser=True, - is_staff=is_staff, - org_member=member, - ) - - superuser_scopes = get_superuser_scopes(auth_state, rpc_user_org_context) - if scopes: - superuser_scopes = superuser_scopes.union(set(scopes)) - if member and member.scopes: - superuser_scopes = superuser_scopes.union(set(member.scopes)) - - _propagate_superuser_access(request, rpc_user_org_context, su) - - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=auth_state, - scopes=superuser_scopes, - ) - - if request.auth is not None and not has_user(request.user): - return from_rpc_auth(request.auth, rpc_user_org_context) - - return from_user_and_rpc_user_org_context( - user=request.user, - rpc_user_org_context=rpc_user_org_context, - is_superuser=False, - is_staff=is_staff, - scopes=scopes, - ) - - -def organizationless_access( - user: User | RpcUser | AnonymousUser, is_superuser: bool, is_staff: bool -) -> Access: - return OrganizationlessAccess( - auth_state=access_service.get_user_auth_state( - user_id=user.id, - is_superuser=is_superuser, - is_staff=is_staff, - organization_id=None, - org_member=None, - ) - ) - - -def normalize_valid_user(user: User | RpcUser | AnonymousUser | None) -> User | RpcUser | None: - if not user or not has_user(user) or not user.is_active: - return None - return user - - -def from_user_and_rpc_user_org_context( - *, - user: User | AnonymousUser | RpcUser | None, - rpc_user_org_context: RpcUserOrganizationContext | None = None, - is_superuser: bool = False, - is_staff: bool = False, - scopes: Iterable[str] | None = None, - auth_state: RpcAuthState | None = None, -) -> Access: - if (user := normalize_valid_user(user)) is None: - return DEFAULT - - if not rpc_user_org_context or not rpc_user_org_context.member: - return organizationless_access(user, is_superuser, is_staff) - - return from_rpc_member( - rpc_user_organization_context=rpc_user_org_context, - scopes=scopes, - is_superuser=is_superuser, - is_staff=is_staff, - auth_state=auth_state, - ) - - -def from_request( - request: Request, organization: Organization | None = None, scopes: Iterable[str] | None = None -) -> Access: - if isinstance(request.auth, AuthenticatedToken) and ( - is_agent_auth(request.auth) or request.auth.superuser is not None - ): - if organization is None: - return DEFAULT - return from_auth(request.auth, organization) - - is_staff = is_active_staff(request) - - if not organization: - return from_user( - request.user, - organization=organization, - scopes=scopes, - is_superuser=is_active_superuser(request), - is_staff=is_staff, - ) - - if getattr(request, "actual_user", None) is not None: - if not should_allow_superuser_access(organization): - raise DataSecrecyError() - - if request.user.is_authenticated and request.user.is_sentry_app: - return _from_sentry_app(request.user, organization=organization) - - if is_active_superuser(request): - member: OrganizationMember | None = None - try: - member = OrganizationMember.objects.get( - user_id=request.user.id, organization_id=organization.id - ) - except OrganizationMember.DoesNotExist: - pass - auth_state = access_service.get_user_auth_state( - user_id=request.user.id, - organization_id=organization.id, - is_superuser=True, - is_staff=is_staff, - org_member=(summarize_member(member) if member is not None else None), - ) - sso_state = auth_state.sso_state - - superuser_scopes = get_superuser_scopes(auth_state, organization) - if scopes: - superuser_scopes = superuser_scopes.union(set(scopes)) - if member and (member_scopes := member.get_scopes()): - superuser_scopes = superuser_scopes.union(set(member_scopes)) - - return OrganizationGlobalAccess( - organization=organization, - _member=member, - scopes=superuser_scopes, - sso_is_valid=sso_state.is_valid, - requires_sso=sso_state.is_required, - permissions=access_service.get_permissions_for_user(request.user.id), - ) - - if request.auth is not None and not has_user(request.user): - return from_auth(request.auth, organization) - - return from_user(user=request.user, organization=organization, scopes=scopes, is_staff=is_staff) - - -# only used internally -def _from_sentry_app(user: User, organization: Organization | None = None) -> Access: - if not organization: - return NoAccess() - - sentry_app_query = SentryApp.objects.filter(proxy_user=user) - - if not sentry_app_query.exists(): - return NoAccess() - - sentry_app = sentry_app_query.get() - - if not sentry_app.is_installed_on(organization): - return NoAccess() - - return OrganizationGlobalMembership( - organization, translate_deprecated_scopes(sentry_app.scope_list), sso_is_valid=True - ) - - -def _from_rpc_sentry_app(context: RpcUserOrganizationContext | None = None) -> Access: - from sentry.sentry_apps.services.app import app_service - - if not context or context.user_id is None: - return NoAccess() - - installation = app_service.find_installation_by_proxy_user( - proxy_user_id=context.user_id, organization_id=context.organization.id - ) - if installation is None: - return NoAccess() - - return ApiOrganizationGlobalMembership( - rpc_user_organization_context=context, - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState( - is_valid=True, - is_required=False, - ), - permissions=[], - ), - scopes=translate_deprecated_scopes(installation.sentry_app.scope_list), - ) - - -def from_user( - user: User | RpcUser | AnonymousUser | None, - organization: Organization | None = None, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, -) -> Access: - if (user := normalize_valid_user(user)) is None: - return DEFAULT - - if not organization: - return organizationless_access(user, is_superuser, is_staff) - - try: - om = OrganizationMember.objects.get(user_id=user.id, organization_id=organization.id) - except OrganizationMember.DoesNotExist: - return organizationless_access(user, is_superuser, is_staff) - - # ensure cached relation - om.organization = organization - - return from_member(om, scopes=scopes, is_superuser=is_superuser, is_staff=is_staff) - - -def from_member( - member: OrganizationMember, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, -) -> Access: - scope_intersection = _intersect_member_and_token_scopes(member.get_scopes(), scopes) - - if (is_superuser or is_staff) and member.user_id is not None: - # "permissions" is a bit of a misnomer -- these are all admin level permissions, and the intent is that if you - # have them, you can only use them when you are acting, as a superuser or staff. This is intentional. - permissions = access_service.get_permissions_for_user(member.user_id) - else: - permissions = frozenset() - - return OrganizationMemberAccess(member, scope_intersection, permissions, scopes) - - -def from_rpc_member( - rpc_user_organization_context: RpcUserOrganizationContext, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, - auth_state: RpcAuthState | None = None, -) -> Access: - if rpc_user_organization_context.user_id is None: - return DEFAULT - - return RpcBackedAccess( - rpc_user_organization_context=rpc_user_organization_context, - scopes_upper_bound=_wrap_scopes(scopes), - auth_state=auth_state - or access_service.get_user_auth_state( - user_id=rpc_user_organization_context.user_id, - organization_id=rpc_user_organization_context.organization.id, - is_superuser=is_superuser, - is_staff=is_staff, - org_member=rpc_user_organization_context.member, - ), - ) - - -def from_auth(auth: AuthenticatedToken, organization: Organization) -> Access: - if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: - context = organization_service.get_organization_by_id( - id=organization.id, user_id=auth.user_id - ) - return from_superuser_access(auth, context) if context is not None else DEFAULT - if is_system_auth(auth): - return SystemAccess() - if is_service_account_auth(auth): - return from_service_account_auth(auth, organization.id) - if is_agent_auth(auth): - access: Access = DEFAULT - if auth.user_id is not None and auth.organization_id == organization.id: - try: - member = OrganizationMember.objects.get( - user_id=auth.user_id, organization_id=organization.id - ) - except OrganizationMember.DoesNotExist: - pass - else: - member.organization = organization - access = from_member(member, scopes=auth.get_scopes()) - return access - auth_organization_id = auth.organization_id - if auth_organization_id is not None and auth_organization_id == organization.id: - return OrganizationGlobalAccess( - auth_organization_id, settings.SENTRY_SCOPES, sso_is_valid=True - ) - else: - return DEFAULT - - -def from_rpc_auth( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: - return from_superuser_access(auth, rpc_user_org_context) - if is_system_auth(auth): - return SystemAccess() - if is_service_account_auth(auth): - return from_service_account_auth(auth, rpc_user_org_context.organization.id) - if is_agent_auth(auth): - # Agents are non-user actors with member-derived, capped authority -- never the - # org-global access an org token gets. Dispatched here so the cap holds at the - # shared userless-auth choke, not only in determine_access. - return from_agent_auth(auth, rpc_user_org_context) - if auth.organization_id == rpc_user_org_context.organization.id: - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=RpcAuthState( - permissions=[], - sso_state=RpcMemberSsoState( - is_valid=True, - is_required=False, - ), - ), - scopes=settings.SENTRY_SCOPES, - ) - else: - return DEFAULT - - -def from_agent_auth( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - """Access for a Seer agent capability token: a non-user actor acting on behalf of a - member. Unlike an org token, the agent is not org-global — its authority is the - delegating member's, capped by the token's scopes, so project access follows the - member's teams. The context MUST be resolved for the delegating user_id.""" - # Bound to the org it was minted for; never honored elsewhere, even if the - # delegating user is also a member of the requested org. - if auth.organization_id != rpc_user_org_context.organization.id: - return DEFAULT - if auth.user_id != rpc_user_org_context.user_id: - return DEFAULT - # No membership (never a member, or revoked since mint) -> no access. Required - # explicitly because RpcBackedAccess would otherwise hand back the full token - # scopes uncapped when member is None. - if rpc_user_org_context.member is None: - return DEFAULT - return from_rpc_member(rpc_user_org_context, scopes=auth.get_scopes()) - - -def from_service_account_auth(auth: AuthenticatedToken, organization_id: int) -> Access: - # Bound to the account's own org, like agent tokens. - if auth.entity_id is None or auth.organization_id != organization_id: - return DEFAULT - # The token doesn't carry `all_projects`, so read it from the replica, like the - # project links. An account deactivated since authentication gets no access. - all_projects = ( - ServiceAccountReplica.objects.filter( - service_account_id=auth.entity_id, organization_id=organization_id, is_active=True - ) - .values_list("all_projects", flat=True) - .first() - ) - if all_projects is None: - return DEFAULT - return ServiceAccountAccess( - service_account_id=auth.entity_id, - organization_id=organization_id, - scopes=auth.get_scopes(), - all_projects=all_projects, - ) - - -def from_superuser_access( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - if ( - auth.superuser is None - or auth.organization_id != rpc_user_org_context.organization.id - or auth.user_id != rpc_user_org_context.user_id - ): - return DEFAULT - users = user_service.get_many(filter={"user_ids": [auth.user_id]}) - user = users[0] if users else None - if user is None: - return DEFAULT - delegated = resolve_superuser_access(auth.superuser, user, rpc_user_org_context) - if delegated is None: - return DEFAULT - scopes, _ = delegated - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_valid=True, is_required=False), permissions=[] - ), - scopes=scopes & set(auth.get_scopes()), - ) - - -DEFAULT = NoAccess() diff --git a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py b/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py deleted file mode 100644 index 54d7541cf..000000000 --- a/packages/evals/fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py +++ /dev/null @@ -1,593 +0,0 @@ -from collections.abc import Mapping -from typing import Any, Literal, TypedDict - -from drf_spectacular.utils import OpenApiResponse, extend_schema -from rest_framework import serializers, status -from rest_framework.exceptions import ValidationError -from rest_framework.request import Request -from rest_framework.response import Response - -from sentry import audit_log, features, roles -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import cell_silo_endpoint -from sentry.api.bases import OrganizationMemberEndpoint -from sentry.api.bases.organization import OrganizationPermission -from sentry.api.exceptions import ResourceDoesNotExist -from sentry.api.serializers import Serializer, serialize -from sentry.api.serializers.models.team import ( - BaseTeamSerializer, - TeamSerializer, - TeamSerializerResponse, -) -from sentry.apidocs.constants import ( - RESPONSE_ACCEPTED, - RESPONSE_BAD_REQUEST, - RESPONSE_NO_CONTENT, - RESPONSE_NOT_FOUND, - RESPONSE_UNAUTHORIZED, -) -from sentry.apidocs.examples.team_examples import TeamExamples -from sentry.apidocs.omissions import sentry_schema_serializer -from sentry.apidocs.parameters import GlobalParams -from sentry.apidocs.response_types import DetailResponse -from sentry.auth.access import Access -from sentry.auth.superuser import superuser_has_permission -from sentry.core.endpoints.organization_member_utils import can_admin_team, can_set_team_role -from sentry.models.groupassignee import GroupAssignee -from sentry.models.groupsubscription import GroupSubscription -from sentry.models.organization import Organization -from sentry.models.organizationaccessrequest import OrganizationAccessRequest -from sentry.models.organizationmember import OrganizationMember -from sentry.models.organizationmemberteam import OrganizationMemberTeam -from sentry.models.team import Team -from sentry.notifications.types import GroupSubscriptionReason -from sentry.roles import organization_roles, team_roles -from sentry.roles.manager import TeamRole -from sentry.utils import metrics - -ERR_INSUFFICIENT_ROLE = "You do not have permission to edit that user's membership." - - -class OrganizationMemberTeamSerializerResponse(TypedDict): - isActive: bool - # This must be manually kept up to date, because we cannot dynamically - # unpack into static type annotations. See https://github.com/microsoft/pylance-release/issues/4084 - teamRole: Literal["contributor", "admin"] - - -@sentry_schema_serializer( - omit_from_public_schema={ - "isActive": "Reported in responses, but the update handler only applies teamRole; " - "a value sent here is validated and discarded.", - } -) -class OrganizationMemberTeamSerializer(serializers.Serializer[dict[str, Any]]): - isActive = serializers.BooleanField() - teamRole = serializers.ChoiceField( - choices=team_roles.get_descriptions(), - default=team_roles.get_default().id, - help_text="The team-level role to switch to. Valid roles include:", - # choices will follow in the docs - ) - - -class OrganizationMemberTeamDetailsSerializer(Serializer[OrganizationMemberTeamSerializerResponse]): - def serialize( - self, obj: OrganizationMemberTeam, attrs: Mapping[Any, Any], user: Any, **kwargs: Any - ) -> OrganizationMemberTeamSerializerResponse: - return { - "isActive": obj.is_active, - "teamRole": obj.role, # type:ignore[typeddict-item] - } - - -class OrganizationTeamMemberPermission(OrganizationPermission): - scope_map = { - "GET": [ - "org:read", - "org:write", - "org:admin", - "member:read", - "member:write", - "member:admin", - ], - "POST": ["org:read", "org:write", "team:write"], - "PUT": [ - "org:read", - "org:write", - "org:admin", - "member:read", - "member:write", - "member:admin", - "team:admin", - "team:write", - ], - "DELETE": ["org:read", "org:write", "org:admin", "team:admin"], - } - - -def _has_elevated_scope(access: Access) -> bool: - """ - Validate that the token has more than just org:read - """ - return access.has_scope("org:write") or access.has_scope("team:write") - - -def _is_org_owner_or_manager(access: Access) -> bool: - role = access.get_organization_role() - # only org owners and managers have org:write scope - return "org:write" in role.scopes if role else False - - -@extend_schema(tags=["Teams"]) -@cell_silo_endpoint -class OrganizationMemberTeamDetailsEndpoint(OrganizationMemberEndpoint): - def convert_args( - self, - request: Request, - organization_id_or_slug: int | str | None = None, - *args: Any, - **kwargs: Any, - ) -> tuple[tuple[Any, ...], dict[str, Any]]: - args, kwargs = super().convert_args(request, organization_id_or_slug, *args, **kwargs) - - team_id_or_slug = kwargs.pop("team_id_or_slug") - organization = kwargs["organization"] - member = kwargs["member"] - - if request.method == "GET": - try: - omt = OrganizationMemberTeam.objects.get( - team__slug__id_or_slug=team_id_or_slug, organizationmember=member - ) - except OrganizationMemberTeam.DoesNotExist: - raise ResourceDoesNotExist - - kwargs["omt"] = omt - - else: - try: - team = Team.objects.get( - organization__slug__id_or_slug=organization.slug, - slug__id_or_slug=team_id_or_slug, - ) - except Team.DoesNotExist: - raise ResourceDoesNotExist - kwargs["team"] = team - - return (args, kwargs) - - publish_status = { - "DELETE": ApiPublishStatus.PUBLIC, - "GET": ApiPublishStatus.PRIVATE, - "PUT": ApiPublishStatus.PUBLIC, - "POST": ApiPublishStatus.PUBLIC, - } - owner = ApiOwner.FOUNDATIONS - permission_classes = (OrganizationTeamMemberPermission,) - - def _can_create_team_member(self, request: Request, team: Team) -> bool: - """ - User can join or add a member to a team: - - * If they are an active superuser - * If they are a team admin or have global write access - * If the open membership organization setting is enabled - """ - access = request.access - - # When open membership is disabled, we need to check if the token has elevated permissions - # in order to ensure integration tokens with only "org:read" scope cannot add members. This check - # comes first because access.has_global_access is True for all integration tokens - if access.is_integration_token and not access.has_open_membership: - return _has_elevated_scope(access) - return access.has_global_access or can_admin_team(access, team) - - def _can_delete( - self, - request: Request, - member: OrganizationMember, - team: Team, - ) -> bool: - """ - User can remove a member from a team: - - * If they are an active superuser - * If they are removing their own membership - * If they are a team admin or have global write access - """ - if superuser_has_permission(request): - return True - - if not request.user.is_authenticated: - return False - - if request.user.id == member.user_id: - return True - - # There is an edge case where org owners/managers cannot remove a member from a team they - # are not part of using team:admin. We cannot explicitly check for team:admin b/c org admins - # also have it but are only allowed to remove members from teams they are on. - if _is_org_owner_or_manager(request.access): - return True - - return can_admin_team(request.access, team) - - def _create_access_request( - self, request: Request, team: Team, member: OrganizationMember - ) -> None: - omt, created = OrganizationAccessRequest.objects.get_or_create(team=team, member=member) - - if not created: - return - - requester = request.user.id if request.user.id != member.user_id else None - if requester: - omt.update(requester_id=requester) - - omt.send_request_email() - - def get( - self, - request: Request, - organization: Organization, - member: OrganizationMember, - omt: OrganizationMemberTeam, - ) -> Response: - return Response( - serialize(omt, request.user, OrganizationMemberTeamDetailsSerializer()), status=200 - ) - - @extend_schema( - operation_id="addOrganizationMemberTeam", - summary="Add an Organization Member to a Team", - parameters=[ - GlobalParams.ORG_ID_OR_SLUG, - GlobalParams.member_id("The ID of the organization member to add to the team"), - GlobalParams.TEAM_ID_OR_SLUG, - ], - request=None, - responses={ - 201: BaseTeamSerializer, - 202: RESPONSE_ACCEPTED, - 204: RESPONSE_NO_CONTENT, - 401: RESPONSE_UNAUTHORIZED, - 403: OpenApiResponse( - description="This team is managed through your organization's identity provider" - ), - 404: RESPONSE_NOT_FOUND, - }, - examples=TeamExamples.ADD_TO_TEAM, - ) - def post( - self, - request: Request, - organization: Organization, - member: OrganizationMember, - team: Team, - ) -> Response[TeamSerializerResponse] | Response[None] | Response[DetailResponse]: - # NOTE: Required to use HTML for table b/c this markdown version doesn't support colspan. - r""" - This request can return various success codes depending on the context of the team: - - **`201`**: The member has been successfully added. - - **`202`**: The member needs permission to join the team and an access request - has been generated. - - **`204`**: The member is already on the team. - - If the team is provisioned through an identity provider, the member cannot join the - team through Sentry. - - Note the permission scopes vary depending on the organization setting `"Open Membership"` - and the type of authorization token. The following table outlines the accepted scopes. - - - - - - - - - - - - - - - - - - - - - - - - -
Open Membership
OnOff
Internal Integration Token -
    -
  • • org:read
    • -
- 		-
    -
  • • org:write
    • -
  • • team:write
    • -
-
User Auth Token -
    -
  • • org:read
    • -
- 		-
    -
  • • org:read*
    • -
  • • org:write
    • -
  • • org:read +
    • -
  •    team:write**
    • -
-
- - - *Organization members are restricted to this scope. When sending a request, it will always - return a 202 and request an invite to the team. - - - \*\*Team Admins must have both **`org:read`** and **`team:write`** scopes in their user - authorization token to add members to their teams. - """ - if not request.user.is_authenticated: - return Response(status=status.HTTP_401_UNAUTHORIZED) - - if not organization_roles.get(member.role).is_team_roles_allowed: - return Response( - { - "detail": f"The user with a '{member.role}' role cannot have team-level permissions." - }, - status=403, - ) - - if OrganizationMemberTeam.objects.filter(team=team, organizationmember=member).exists(): - return Response(status=204) - - if team.idp_provisioned: - return Response( - {"detail": "This team is managed through your organization's identity provider."}, - status=403, - ) - - if not self._can_create_team_member(request, team): - self._create_access_request(request, team, member) - return Response(status=202) - - omt = OrganizationMemberTeam.objects.create(team=team, organizationmember=member) - - self.create_audit_entry( - request=request, - organization=organization, - target_object=omt.id, - target_user_id=member.user_id, - event=audit_log.get_event_id("MEMBER_JOIN_TEAM"), - data=omt.get_audit_log_data(), - ) - - body: TeamSerializerResponse = serialize(team, request.user, TeamSerializer()) - return Response(body, status=201) - - @extend_schema( - operation_id="updateOrganizationMemberTeam", - summary="Update an Organization Member's Team Role", - parameters=[ - GlobalParams.ORG_ID_OR_SLUG, - GlobalParams.member_id("The ID of the organization member to change"), - GlobalParams.TEAM_ID_OR_SLUG, - ], - request=OrganizationMemberTeamSerializer, - responses={ - 200: OrganizationMemberTeamDetailsSerializer, - 400: RESPONSE_BAD_REQUEST, - 404: RESPONSE_NOT_FOUND, - }, - examples=TeamExamples.UPDATE_TEAM_ROLE, - ) - def put( - self, - request: Request, - organization: Organization, - member: OrganizationMember, - team: Team, - ) -> ( - Response[OrganizationMemberTeamSerializerResponse] - | Response[None] - | Response[DetailResponse] - ): - """ - The relevant organization member must already be a part of the team. - - Note that for organization admins, managers, and owners, they are - automatically granted a minimum team role of `admin` on all teams they - are part of. Read more about [team roles](https://docs.sentry.io/product/teams/roles/). - """ - try: - omt = OrganizationMemberTeam.objects.get(team=team, organizationmember=member) - except OrganizationMemberTeam.DoesNotExist: - raise ResourceDoesNotExist - - serializer = OrganizationMemberTeamSerializer(data=request.data, partial=True) - if not serializer.is_valid(): - raise ValidationError(serializer.errors) - result = serializer.validated_data - - if "teamRole" in result and features.has("organizations:team-roles", organization): - new_role_id = result["teamRole"] - try: - new_role = team_roles.get(new_role_id) - except KeyError: - return Response(status=400) - can_set_new_role = can_set_team_role(request, team, new_role) - - try: - old_role = team_roles.get(omt.role) if omt.role else None - except KeyError: - old_role = None - if old_role is None: - old_role = roles.get_minimum_team_role(member.role) - can_set_old_role = can_set_team_role(request, team, old_role) - - # Verify that the request is allowed to set both the old and new role to prevent role downgrades by low-privilege users - if not (can_set_new_role and can_set_old_role): - return Response({"detail": ERR_INSUFFICIENT_ROLE}, status=400) - - self._change_team_member_role(omt, new_role) - - self.create_audit_entry( - request=request, - organization=organization, - target_object=omt.id, - target_user_id=member.user_id, - event=audit_log.get_event_id("MEMBER_EDIT"), - data={ - "email": member.get_email(), - "role": member.role, - "team_slug": team.slug, - "team_role": new_role_id, - }, - ) - - put_body: OrganizationMemberTeamSerializerResponse = serialize( - omt, request.user, OrganizationMemberTeamDetailsSerializer() - ) - return Response(put_body, status=200) - - @staticmethod - def _change_team_member_role( - team_membership: OrganizationMemberTeam, team_role: TeamRole - ) -> None: - """Modify a member's team-level role.""" - minimum_team_role = roles.get_minimum_team_role(team_membership.organizationmember.role) - if team_role.priority > minimum_team_role.priority: - applying_minimum = False - team_membership.update(role=team_role.id) - else: - # The new team role is redundant to the role that this member would - # receive as their minimum team role anyway. This makes it effectively - # invisible in the UI, and it would be surprising if it were suddenly - # left over after the user's org-level role is demoted. So, write a null - # value to the database and let the minimum team role take over. - applying_minimum = True - team_membership.update(role=None) - - metrics.incr( - "team_roles.assign", - tags={"target_team_role": team_role.id, "applying_minimum": str(applying_minimum)}, - ) - - @extend_schema( - operation_id="deleteOrganizationMemberTeam", - summary="Delete an Organization Member from a Team", - parameters=[ - GlobalParams.ORG_ID_OR_SLUG, - GlobalParams.member_id("The ID of the organization member to delete from the team"), - GlobalParams.TEAM_ID_OR_SLUG, - ], - responses={ - 200: BaseTeamSerializer, - 400: RESPONSE_BAD_REQUEST, - 403: OpenApiResponse( - description="This team is managed through your organization's identity provider" - ), - 404: RESPONSE_NOT_FOUND, - }, - examples=TeamExamples.DELETE_FROM_TEAM, - ) - def delete( - self, - request: Request, - organization: Organization, - member: OrganizationMember, - team: Team, - ) -> Response[TeamSerializerResponse] | Response[DetailResponse]: - r""" - Delete an organization member from a team. - - Note the permission scopes vary depending on the type of authorization token. The following - table outlines the accepted scopes. - - - - - - - - - -
Org Auth Token -
    -
  • • org:write
    • -
  • • org:admin
    • -
  • • team:admin
    • -
-
User Auth Token -
    -
  • • org:read*
    • -
  • • org:write
    • -
  • • org:admin
    • -
  • • team:admin
    • -
  • • org:read + team:admin**
    • -
-
- - - \***`org:read`** can only be used to remove yourself from the teams you are a member of. - - - \*\*Team Admins must have both **`org:read`** and **`team:admin`** scopes in their user - authorization token to delete members from their teams. - """ - - if not self._can_delete(request, member, team): - return Response({"detail": ERR_INSUFFICIENT_ROLE}, status=400) - - if team.idp_provisioned: - return Response( - {"detail": "This team is managed through your organization's identity provider."}, - status=403, - ) - - omt = None - try: - omt = OrganizationMemberTeam.objects.get(team=team, organizationmember=member) - except OrganizationMemberTeam.DoesNotExist: - pass - - else: - self.create_audit_entry( - request=request, - organization=organization, - target_object=omt.id, - target_user_id=member.user_id, - event=audit_log.get_event_id("MEMBER_LEAVE_TEAM"), - data=omt.get_audit_log_data(), - ) - omt.delete() - - self._unsubscribe_issues(team, member) - - delete_body: TeamSerializerResponse = serialize(team, request.user, TeamSerializer()) - return Response(delete_body, status=200) - - @staticmethod - def _unsubscribe_issues(team: Team, member: OrganizationMember) -> None: - """ - Unsubscribe user from issues the team is subscribed to - """ - team_assigned_groups = GroupAssignee.objects.filter(team_id=team.id).values_list( - "group_id", flat=True - ) - team_subscribed_groups = GroupSubscription.objects.filter( - team_id=team.id, reason=GroupSubscriptionReason.assigned - ).values_list("group_id", flat=True) - group_ids_to_unsubscribe = set(team_assigned_groups) | set(team_subscribed_groups) - GroupSubscription.objects.filter( - group_id__in=group_ids_to_unsubscribe, user_id=member.user_id - ).delete() diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py deleted file mode 100644 index d5fa08b5d..000000000 --- a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py +++ /dev/null @@ -1,1537 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Collection, Iterable, Mapping -from dataclasses import dataclass -from functools import cached_property -from typing import Any - -from django.conf import settings -from django.contrib.auth.models import AnonymousUser -from django.http.request import HttpRequest -from rest_framework.request import Request -from sentry_sdk import traces - -from sentry import features, roles -from sentry.api.exceptions import DataSecrecyError -from sentry.auth.deprecated_scope_translation import translate_deprecated_scopes -from sentry.auth.principal import has_user -from sentry.auth.scope_declaration import check_scope_declaration, check_scope_declarations -from sentry.auth.services.access.service import access_service -from sentry.auth.services.auth import AuthenticatedToken, RpcAuthState, RpcMemberSsoState -from sentry.auth.staff import is_active_staff -from sentry.auth.superuser import ( - Superuser, - get_superuser_access_expiry, - get_superuser_scopes, - is_active_superuser, - resolve_superuser_access, -) -from sentry.auth.system import is_system_auth -from sentry.constants import ObjectStatus -from sentry.data_secrecy.logic import should_allow_superuser_access -from sentry.models.apiscopes import add_scope_hierarchy -from sentry.models.organization import Organization -from sentry.models.organizationmember import OrganizationMember -from sentry.models.organizationmemberteam import OrganizationMemberTeam -from sentry.models.project import Project -from sentry.models.team import Team, TeamStatus -from sentry.organizations.services.organization import ( - RpcTeamMember, - RpcUserOrganizationContext, - organization_service, -) -from sentry.organizations.services.organization.serial import summarize_member -from sentry.roles import organization_roles -from sentry.roles.manager import OrganizationRole, TeamRole -from sentry.seer.agent_token import is_agent_auth -from sentry.sentry_apps.models.sentry_app import SentryApp -from sentry.service_accounts.auth import is_service_account_auth -from sentry.service_accounts.models import ServiceAccountProjectReplica -from sentry.types.request import _HttpRequestWithUser -from sentry.users.models.user import User -from sentry.users.services.user import RpcUser -from sentry.users.services.user.service import user_service -from sentry.utils import metrics -from sentry.viewer_context import SuperuserAccess, set_viewer_context_superuser - -__all__ = ( - "from_user", - "from_member", - "DEFAULT", - "from_user_and_rpc_user_org_context", - "from_rpc_member", -) - - -def has_role_in_organization(role: str, organization: Organization, user_id: int) -> bool: - query = OrganizationMember.objects.filter( - user_is_active=True, - user_id=user_id, - organization_id=organization.id, - role=role, - ) - return query.exists() - - -class Access(abc.ABC): - @property - @abc.abstractmethod - def sso_is_valid(self) -> bool: - pass - - @property - @abc.abstractmethod - def requires_sso(self) -> bool: - pass - - @property - @abc.abstractmethod - def has_open_membership(self) -> bool: - pass - - @property - @abc.abstractmethod - def has_global_access(self) -> bool: - pass - - @property - @abc.abstractmethod - def scopes(self) -> frozenset[str]: - pass - - @property - @abc.abstractmethod - def permissions(self) -> frozenset[str]: - pass - - # TODO(cathy): remove this - @property - @abc.abstractmethod - def role(self) -> str | None: - pass - - @property - @abc.abstractmethod - def team_ids_with_membership(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def accessible_team_ids(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def project_ids_with_team_membership(self) -> frozenset[int]: - pass - - @property - @abc.abstractmethod - def accessible_project_ids(self) -> frozenset[int]: - pass - - @property - def is_integration_token(self) -> bool: - return False - - def has_permission(self, permission: str) -> bool: - """ - Return bool representing if the user has the given permission. - >>> access.has_permission('broadcasts.admin') - """ - return permission in self.permissions - - def has_scope(self, scope: str) -> bool: - """ - Return bool representing if the user has the given scope. - >>> access.has_project('org:read') - """ - check_scope_declaration(scope) - return scope in self.scopes - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - """Whether adding ``scope`` to the current auth scope cap would grant it.""" - check_scope_declaration(scope) - return False - - def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: - """Whether adding ``scope`` to the current auth scope cap would grant it for a project.""" - check_scope_declaration(scope) - return False - - def get_organization_role(self) -> OrganizationRole | None: - if self.role is not None: - return organization_roles.get(self.role) - return None - - @abc.abstractmethod - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - pass - - @abc.abstractmethod - def has_team_access(self, team: Team) -> bool: - """ - Return bool representing if a user should have access to information for the given team. - >>> access.has_team_access(team) - """ - - @abc.abstractmethod - def has_team_scope(self, team: Team, scope: str) -> bool: - pass - - def has_team_membership(self, team: Team) -> bool: - return team.id in self.team_ids_with_membership - - @abc.abstractmethod - def get_team_role(self, team: Team) -> TeamRole | None: - pass - - @abc.abstractmethod - def has_project_access(self, project: Project) -> bool: - """ - Return bool representing if a user should have access to information for the given project. - >>> access.has_project_access(project) - """ - raise NotImplementedError - - def has_projects_access(self, projects: Iterable[Project]) -> bool: - """ - Returns bool representing if a user should have access to every requested project - """ - return all([self.has_project_access(project) for project in projects]) - - def has_project_membership(self, project: Project) -> bool: - """ - Return bool representing if a user has explicit membership for the given project. - >>> access.has_project_membership(project) - """ - return project.id in self.project_ids_with_team_membership - - def has_project_scope(self, project: Project, scope: str) -> bool: - """ - Return bool representing if a user should have access with the given scope to information - for the given project. - - >>> access.has_project_scope(project, 'project:read') - """ - return self.has_any_project_scope(project, [scope]) - - @abc.abstractmethod - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - pass - - -def _intersect_member_and_token_scopes( - member_scopes: Collection[str], token_scopes: Iterable[str] | None -) -> frozenset[str]: - member_scopes = frozenset(member_scopes) - if token_scopes is None: - return member_scopes - - token_scopes = frozenset(token_scopes) - token_only_scopes = token_scopes & settings.SENTRY_TOKEN_ONLY_SCOPES - - return (token_scopes & member_scopes) | token_only_scopes - - -@dataclass -class DbAccess(Access): - # TODO(dcramer): this is still a little gross, and ideally backend access - # would be based on the same scopes as API access so there's clarity in - # what things mean - - sso_is_valid: bool = False - requires_sso: bool = False - has_open_membership: bool = False - - # if has_global_access is True, then any project - # matching organization_id is valid. This is used for - # both `organization.allow_joinleave` and to indicate - # that the role is global / a user is an active superuser - has_global_access: bool = False - - scopes: frozenset[str] = frozenset() - scopes_upper_bound: frozenset[str] | None = None - permissions: frozenset[str] = frozenset() - - _member: OrganizationMember | None = None - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - return self._member.role if self._member else None - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - if self._member is None or self.scopes_upper_bound is None: - return False - candidate_scopes = _intersect_member_and_token_scopes( - self._member.get_scopes(), self.scopes_upper_bound | {scope} - ) - return scope in add_scope_hierarchy(list(candidate_scopes)) - - @cached_property - def _team_memberships(self) -> Mapping[Team, OrganizationMemberTeam]: - if self._member is None: - return {} - return { - omt.team: omt - for omt in OrganizationMemberTeam.objects.filter( - organizationmember=self._member, is_active=True, team__status=TeamStatus.ACTIVE - ).select_related("team") - } - - @cached_property - def team_ids_with_membership(self) -> frozenset[int]: - """Return the IDs of teams in which the user has actual membership. - - This represents the set of all teams for which `has_team_membership` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - - Compare to accessible_team_ids, which is equal to this property in the - typical case but represents a superset of IDs in case of superuser access. - """ - return frozenset(team.id for team in self._team_memberships.keys()) - - @property - def accessible_team_ids(self) -> frozenset[int]: - """Return the IDs of teams to which the user has access. - - This represents the set of all teams for which `has_team_access` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - """ - return self.team_ids_with_membership - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - """Return the IDs of projects to which the user has access via actual team membership. - - This represents the set of all projects for which `has_project_membership` - returns true. Use that method where possible and use this property only when - you need to iterate or query for all such teams. - - Compare to accessible_project_ids, which is equal to this property in the - typical case but represents a superset of IDs in case of superuser access. - """ - teams = self._team_memberships.keys() - if not teams: - return frozenset() - - with traces.start_span( - name="get_project_access_in_teams", - attributes={"sentry.op": "get_project_access_in_teams"}, - ) as span: - projects = frozenset( - Project.objects.filter(status=ObjectStatus.ACTIVE, teams__in=teams) - .distinct() - .values_list("id", flat=True) - ) - span.set_attribute("Project Count", len(projects)) - span.set_attribute("Team Count", len(teams)) - - return projects - - @property - def accessible_project_ids(self) -> frozenset[int]: - """Return the IDs of projects to which the user has access. - - This represents the set of all teams for which `has_project_access` returns - true. Use that method where possible and use this property only when you need - to iterate or query for all such teams. - """ - return self.project_ids_with_team_membership - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - if self._member and self._member.user_id is not None: - return has_role_in_organization( - role=role, organization=organization, user_id=self._member.user_id - ) - return False - - def has_team_scope(self, team: Team, scope: str) -> bool: - """ - Return bool representing if a user should have access with the given scope to information - for the given team. - - >>> access.has_team_scope(team, 'team:read') - """ - check_scope_declaration(scope) - if not self.has_team_access(team): - return False - if self.has_scope(scope): - return True - - membership = self._team_memberships.get(team) - if not membership: - return False - - team_scopes = membership.get_scopes() - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - if membership and scope in team_scopes: - metrics.incr( - "team_roles.pass_by_team_scope", - tags={"team_role": membership.role, "scope": scope}, - ) - return True - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - team_member = self._team_memberships.get(team) - if team_member: - return team_member.get_team_role() - else: - return None - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - """ - Represent if a user should have access with any one of the given scopes to - information for the given project. - - For performance's sake, prefer this over multiple calls to `has_project_scope`. - """ - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - if any(self.has_scope(scope) for scope in scopes): - return True - - if self._member and features.has("organizations:team-roles", self._member.organization): - with traces.start_span( - name="check_access_for_all_project_teams", - attributes={"sentry.op": "check_access_for_all_project_teams"}, - ) as span: - memberships = [ - self._team_memberships[team] - for team in project.teams.all() - if team in self._team_memberships - ] - span.set_attribute("organization", self._member.organization.id) - span.set_attribute("organization.slug", self._member.organization.slug) - span.set_attribute("membership_count", len(memberships)) - - for membership in memberships: - team_scopes = membership.get_scopes() - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - for scope in scopes: - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_project_scope", - tags={"team_role": membership.role, "scope": scope}, - ) - return True - return False - - -@dataclass -class SingularRpcAccessOrgOptimization: - access: RpcBackedAccess - - -def maybe_singular_rpc_access_org_context( - access: Access, org_ids: set[int] -) -> SingularRpcAccessOrgOptimization | None: - if ( - isinstance(access, RpcBackedAccess) - and len(org_ids) == 1 - and access.rpc_user_organization_context.organization.id in org_ids - ): - return SingularRpcAccessOrgOptimization(access) - return None - - -maybe_singular_api_access_org_context = maybe_singular_rpc_access_org_context - - -@dataclass -class RpcBackedAccess(Access): - rpc_user_organization_context: RpcUserOrganizationContext - scopes_upper_bound: frozenset[str] | None - auth_state: RpcAuthState - - # TODO: remove once getsentry has updated to use the new names. - @property - def api_user_organization_context(self) -> RpcUserOrganizationContext: - return self.rpc_user_organization_context - - @cached_property - def permissions(self) -> frozenset[str]: - return frozenset(self.auth_state.permissions) - - @property - def sso_is_valid(self) -> bool: - return self.auth_state.sso_state.is_valid - - @property - def requires_sso(self) -> bool: - return self.auth_state.sso_state.is_required - - @property - def has_open_membership(self) -> bool: - return self.rpc_user_organization_context.organization.flags.allow_joinleave - - @property - def has_global_access(self) -> bool: - if self.has_open_membership: - return True - - if ( - self.rpc_user_organization_context.member - and roles.get(self.rpc_user_organization_context.member.role).is_global - ): - return True - - return False - - @cached_property - def scopes(self) -> frozenset[str]: - if self.rpc_user_organization_context.member is None: - return frozenset(self.scopes_upper_bound or []) - - if self.scopes_upper_bound is None: - return frozenset(self.rpc_user_organization_context.member.scopes) - - return _intersect_member_and_token_scopes( - self.rpc_user_organization_context.member.scopes, - self.scopes_upper_bound, - ) - - def would_have_scope_with_added_auth_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - member = self.rpc_user_organization_context.member - if member is None or self.scopes_upper_bound is None: - return False - candidate_scopes = _intersect_member_and_token_scopes( - member.scopes, self.scopes_upper_bound | {scope} - ) - return scope in add_scope_hierarchy(list(candidate_scopes)) - - def would_have_project_scope_with_added_auth_scope(self, project: Project, scope: str) -> bool: - """ - Return whether adding a scope to the token would grant access for this project. - - Check both organization-level scopes and scopes from the member's role on - a project team. This does not modify the token or access instance. - """ - if not self.has_project_access(project): - return False - if self.would_have_scope_with_added_auth_scope(scope): - return True - - # Team roles can only grant project scopes for member-backed token access. - member = self.rpc_user_organization_context.member - if ( - member is None - or self.scopes_upper_bound is None - or not features.has( - "organizations:team-roles", self.rpc_user_organization_context.organization - ) - ): - return False - - # A role on any team linked to the project may grant the requested scope... - project_team_ids = set(project.teams.values_list("id", flat=True)) - return any( - member_team.role - and member_team.team_id in project_team_ids - and scope in member_team.role.scopes - for member_team in member.member_teams - ) - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - if self.rpc_user_organization_context.member is None: - return None - return self.rpc_user_organization_context.member.role - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - member = self.rpc_user_organization_context.member - if member and member.user_id: - return has_role_in_organization( - role=role, organization=organization, user_id=member.user_id - ) - return False - - @cached_property - def team_ids_with_membership(self) -> frozenset[int]: - if self.rpc_user_organization_context.member is None: - return frozenset() - return frozenset( - mt.team_id for mt in self.rpc_user_organization_context.member.member_teams - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return self.team_ids_with_membership - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - if self.rpc_user_organization_context.member is None: - return frozenset() - return frozenset(self.rpc_user_organization_context.member.project_ids) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return self.project_ids_with_team_membership - - def has_team_access(self, team: Team) -> bool: - if team.status != TeamStatus.ACTIVE: - return False - if ( - self.has_global_access - and self.rpc_user_organization_context.organization.id == team.organization_id - ): - return True - return team.id in self.team_ids_with_membership - - def get_team_membership(self, team_id: int) -> RpcTeamMember | None: - if self.rpc_user_organization_context.member is None: - return None - - for team_membership in self.rpc_user_organization_context.member.member_teams: - if team_membership.team_id == team_id: - return team_membership - return None - - def has_team_scope(self, team: Team, scope: str) -> bool: - check_scope_declaration(scope) - if not self.has_team_access(team): - return False - if self.has_scope(scope): - return True - - if self.rpc_user_organization_context.member is None: - return False - - team_membership = self.get_team_membership(team.id) - if not team_membership: - return False - - team_scopes = frozenset(team_membership.scopes) - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_team_scope", - tags={"team_role": f"{team_membership.role}", "scope": scope}, - ) - return True - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - team_member = self.get_team_membership(team.id) - if team_member: - return team_member.role - return None - - def has_project_access(self, project: Project) -> bool: - if project.status != ObjectStatus.ACTIVE: - return False - if ( - self.has_global_access - and self.rpc_user_organization_context.member - and self.rpc_user_organization_context.organization.id == project.organization_id - ): - return True - return project.id in self.project_ids_with_team_membership - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - """ - Represent if a user should have access with any one of the given scopes to - information for the given project. - - For performance's sake, prefer this over multiple calls to `has_project_scope`. - """ - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - if any(self.has_scope(scope) for scope in scopes): - return True - - if self.rpc_user_organization_context.member and features.has( - "organizations:team-roles", self.rpc_user_organization_context.organization - ): - with traces.start_span( - name="check_access_for_all_project_teams", - attributes={"sentry.op": "check_access_for_all_project_teams"}, - ) as span: - project_teams_id = set(project.teams.values_list("id", flat=True)) - orgmember_teams = self.rpc_user_organization_context.member.member_teams - span.set_attribute( - "organization", self.rpc_user_organization_context.organization.id - ) - span.set_attribute( - "organization.slug", self.rpc_user_organization_context.organization.slug - ) - span.set_attribute("membership_count", len(orgmember_teams)) - - for member_team in orgmember_teams: - if not member_team.role: - continue - if member_team.team_id not in project_teams_id: - continue - - team_scopes = member_team.role.scopes - if self.scopes_upper_bound: - team_scopes = team_scopes & self.scopes_upper_bound - - for scope in scopes: - if scope in team_scopes: - metrics.incr( - "team_roles.pass_by_project_scope", - tags={"team_role": f"{member_team.role.id}", "scope": scope}, - ) - return True - return False - - -def _wrap_scopes(scopes_upper_bound: Iterable[str] | None) -> frozenset[str] | None: - if scopes_upper_bound is not None: - return frozenset(scopes_upper_bound) - return None - - -class OrganizationMemberAccess(DbAccess): - def __init__( - self, - member: OrganizationMember, - scopes: Iterable[str], - permissions: Iterable[str], - scopes_upper_bound: Iterable[str] | None, - ) -> None: - auth_state = access_service.get_user_auth_state( - organization_id=member.organization_id, - is_superuser=False, - is_staff=False, - org_member=summarize_member(member), - user_id=member.user_id, - ) - sso_state = auth_state.sso_state - has_global_access = ( - bool(member.organization.flags.allow_joinleave) or roles.get(member.role).is_global - ) - - super().__init__( - _member=member, - sso_is_valid=sso_state.is_valid, - requires_sso=sso_state.is_required, - has_global_access=has_global_access, - scopes=frozenset(scopes), - permissions=frozenset(permissions), - scopes_upper_bound=_wrap_scopes(scopes_upper_bound), - ) - - def has_team_access(self, team: Team) -> bool: - assert self._member is not None - if team.status != TeamStatus.ACTIVE: - return False - if self.has_global_access and self._member.organization.id == team.organization_id: - return True - return team.id in self.team_ids_with_membership - - def has_project_access(self, project: Project) -> bool: - assert self._member is not None - if project.status != ObjectStatus.ACTIVE: - return False - if self.has_global_access and self._member.organization.id == project.organization_id: - return True - return project.id in self.project_ids_with_team_membership - - -class OrganizationGlobalAccess(DbAccess): - """Access to all an organization's teams and projects.""" - - def __init__( - self, organization: Organization | int, scopes: Iterable[str], **kwargs: Any - ) -> None: - self._organization_id = ( - organization.id if isinstance(organization, Organization) else organization - ) - super().__init__(has_global_access=True, scopes=frozenset(scopes), **kwargs) - - def has_team_access(self, team: Team) -> bool: - return bool( - team.organization_id == self._organization_id and team.status == TeamStatus.ACTIVE - ) - - def has_project_access(self, project: Project) -> bool: - return bool( - project.organization_id == self._organization_id - and project.status == ObjectStatus.ACTIVE - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset( - Team.objects.filter( - organization_id=self._organization_id, status=TeamStatus.ACTIVE - ).values_list("id", flat=True) - ) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset( - Project.objects.filter( - organization_id=self._organization_id, status=ObjectStatus.ACTIVE - ).values_list("id", flat=True) - ) - - -class ServiceAccountAccess(DbAccess): - """ - A service account's access to its organization. It has no membership, so its - scopes are the account's own. With open membership it reaches every active team - and project, like `OrganizationGlobalAccess`. Without it, it reaches only the - active projects linked to the account, and no teams. - """ - - def __init__( - self, - *, - service_account_id: int, - organization_id: int, - scopes: Iterable[str], - has_open_membership: bool, - ) -> None: - self._service_account_id = service_account_id - self._organization_id = organization_id - super().__init__( - sso_is_valid=True, - has_open_membership=has_open_membership, - has_global_access=has_open_membership, - scopes=frozenset(scopes), - ) - - def has_team_access(self, team: Team) -> bool: - return bool( - self.has_global_access - and team.organization_id == self._organization_id - and team.status == TeamStatus.ACTIVE - ) - - def has_project_access(self, project: Project) -> bool: - if ( - project.organization_id != self._organization_id - or project.status != ObjectStatus.ACTIVE - ): - return False - return self.has_global_access or project.id in self.accessible_project_ids - - @cached_property - def project_ids_with_team_membership(self) -> frozenset[int]: - # With no teams, the account's project links stand in for membership, so - # endpoints that default to "my projects" see the projects it can reach. - return self.accessible_project_ids - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - if not self.has_global_access: - return frozenset() - return frozenset( - Team.objects.filter( - organization_id=self._organization_id, status=TeamStatus.ACTIVE - ).values_list("id", flat=True) - ) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - if self.has_global_access: - return frozenset( - Project.objects.filter( - organization_id=self._organization_id, status=ObjectStatus.ACTIVE - ).values_list("id", flat=True) - ) - # A replica link can briefly outlive a transfer of its project to another org. - return frozenset( - ServiceAccountProjectReplica.objects.filter( - service_account__service_account_id=self._service_account_id, - project__organization_id=self._organization_id, - project__status=ObjectStatus.ACTIVE, - ).values_list("project_id", flat=True) - ) - - -class ApiBackedOrganizationGlobalAccess(RpcBackedAccess): - """Access to all an organization's teams and projects.""" - - def __init__( - self, - *, - rpc_user_organization_context: RpcUserOrganizationContext, - auth_state: RpcAuthState, - scopes: Iterable[str] | None, - ): - super().__init__( - rpc_user_organization_context=rpc_user_organization_context, - auth_state=auth_state, - scopes_upper_bound=_wrap_scopes(scopes), - ) - - @cached_property - def scopes(self) -> frozenset[str]: - return frozenset(self.scopes_upper_bound or []) - - @property - def has_global_access(self) -> bool: - return True - - def has_team_access(self, team: Team) -> bool: - return bool( - team.organization_id == self.rpc_user_organization_context.organization.id - and team.status == TeamStatus.ACTIVE - ) - - def has_project_access(self, project: Project) -> bool: - return bool( - project.organization_id == self.rpc_user_organization_context.organization.id - and project.status == ObjectStatus.ACTIVE - ) - - @cached_property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset( - t.id - for t in self.rpc_user_organization_context.organization.teams - if t.status == TeamStatus.ACTIVE - ) - - @cached_property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset( - p.id - for p in self.rpc_user_organization_context.organization.projects - if p.status == ObjectStatus.ACTIVE - ) - - -class OrganizationGlobalMembership(OrganizationGlobalAccess): - """Access to all an organization's teams and projects with simulated membership.""" - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return self.accessible_team_ids - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return self.accessible_project_ids - - def has_team_membership(self, team: Team) -> bool: - return self.has_team_access(team) - - def has_project_membership(self, project: Project) -> bool: - return self.has_project_access(project) - - -class ApiOrganizationGlobalMembership(ApiBackedOrganizationGlobalAccess): - """Access to all an organization's teams and projects with simulated membership.""" - - @property - def is_integration_token(self) -> bool: - return True - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return self.accessible_team_ids - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return self.accessible_project_ids - - def has_team_membership(self, team: Team) -> bool: - return self.has_team_access(team) - - def has_project_membership(self, project: Project) -> bool: - return self.has_project_access(project) - - -@dataclass -class OrganizationlessAccess(Access): - auth_state: RpcAuthState - - @cached_property - def permissions(self) -> frozenset[str]: - return frozenset(self.auth_state.permissions) - - def has_team_access(self, team: Team) -> bool: - return False - - def has_project_access(self, project: Project) -> bool: - return False - - @property - def sso_is_valid(self) -> bool: - return self.auth_state.sso_state.is_valid - - @property - def requires_sso(self) -> bool: - return self.auth_state.sso_state.is_required - - @property - def has_open_membership(self) -> bool: - return False - - @property - def has_global_access(self) -> bool: - return False - - @property - def scopes(self) -> frozenset[str]: - return frozenset() - - # TODO(cathy): remove this - @property - def role(self) -> str | None: - return None - - def has_role_in_organization( - self, role: str, organization: Organization, user_id: int | None - ) -> bool: - if user_id: - return has_role_in_organization(role=role, organization=organization, user_id=user_id) - return False - - @property - def team_ids_with_membership(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset() - - @property - def project_ids_with_team_membership(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset() - - def has_team_scope(self, team: Team, scope: str) -> bool: - check_scope_declaration(scope) - return False - - def get_team_role(self, team: Team) -> TeamRole | None: - return None - - def has_any_project_scope(self, project: Project, scopes: Collection[str]) -> bool: - check_scope_declarations(scopes) - if not self.has_project_access(project): - return False - - return any(self.has_scope(scope) for scope in scopes) - - -class SystemAccess(OrganizationlessAccess): - def __init__(self) -> None: - super().__init__( - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_required=False, is_valid=False), - permissions=[], - ), - ) - - @property - def has_global_access(self) -> bool: - return True - - def has_permission(self, permission: str) -> bool: - return True - - def has_scope(self, scope: str) -> bool: - check_scope_declaration(scope) - return True - - def has_team_access(self, team: Team) -> bool: - return True - - def has_project_access(self, project: Project) -> bool: - return True - - # The semantically correct behavior for accessible_(team|project)_ids would be to - # query for all teams or projects in the system, which we don't want to attempt. - # Code paths that may have SystemAccess must avoid looking at these properties. - @property - def accessible_team_ids(self) -> frozenset[int]: - return frozenset() - - @property - def accessible_project_ids(self) -> frozenset[int]: - return frozenset() - - -class NoAccess(OrganizationlessAccess): - def __init__(self) -> None: - super().__init__( - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_required=False, is_valid=True), - permissions=[], - ), - ) - - -def _propagate_superuser_access( - request: _HttpRequestWithUser, - org_context: RpcUserOrganizationContext, - su: Superuser | None, -) -> None: - # Members use their normal callback permissions and user flags; adding this - # claim would instead force read-only access and a five-minute VC lifetime. - if org_context.member is not None: - return - if request.auth is not None or su is None or not su.is_active: - return - if su.requires_org_auth(org_context.organization): - return - expires_at = get_superuser_access_expiry(su, org_context.organization.id) - if expires_at is not None: - set_viewer_context_superuser( - user_id=request.user.id, - organization_id=org_context.organization.id, - superuser=SuperuserAccess(expires_at=expires_at), - ) - - -def from_request_org_and_scopes( - *, - request: HttpRequest, - rpc_user_org_context: RpcUserOrganizationContext | None = None, - scopes: Iterable[str] | None = None, -) -> Access: - """ - Note that `scopes` is usually None because request.auth is not set at `get_authorization_header` - when the request is made from the frontend using cookies - """ - if isinstance(request.auth, AuthenticatedToken) and request.auth.superuser is not None: - if rpc_user_org_context is None: - return DEFAULT - return from_superuser_access(request.auth, rpc_user_org_context) - if is_agent_auth(request.auth): - if rpc_user_org_context is None: - return DEFAULT - return from_agent_auth(request.auth, rpc_user_org_context) - - is_staff = is_active_staff(request) - - if not rpc_user_org_context: - return from_user_and_rpc_user_org_context( - user=request.user, - rpc_user_org_context=rpc_user_org_context, - is_superuser=is_active_superuser(request), - is_staff=is_staff, - scopes=scopes, - ) - - if getattr(request, "actual_user", None) is not None: - if not should_allow_superuser_access(rpc_user_org_context): - raise DataSecrecyError() - - if getattr(request.user, "is_sentry_app", False): - return _from_rpc_sentry_app(rpc_user_org_context) - - if is_active_superuser(request): - su = getattr(request, "superuser", None) - if su is None and request.auth is None: - su = Superuser(request) - requires_org_auth = su is not None and su.requires_org_auth( - rpc_user_org_context.organization - ) - if requires_org_auth and rpc_user_org_context.member is None: - setattr(request, "_superuser_needs_org_auth", rpc_user_org_context.organization.slug) - - member = rpc_user_org_context.member - auth_state = access_service.get_user_auth_state( - user_id=request.user.id, - organization_id=rpc_user_org_context.organization.id, - is_superuser=True, - is_staff=is_staff, - org_member=member, - ) - - superuser_scopes = get_superuser_scopes(auth_state, rpc_user_org_context) - if scopes: - superuser_scopes = superuser_scopes.union(set(scopes)) - if member and member.scopes: - superuser_scopes = superuser_scopes.union(set(member.scopes)) - - _propagate_superuser_access(request, rpc_user_org_context, su) - - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=auth_state, - scopes=superuser_scopes, - ) - - if request.auth is not None and not has_user(request.user): - return from_rpc_auth(request.auth, rpc_user_org_context) - - return from_user_and_rpc_user_org_context( - user=request.user, - rpc_user_org_context=rpc_user_org_context, - is_superuser=False, - is_staff=is_staff, - scopes=scopes, - ) - - -def organizationless_access( - user: User | RpcUser | AnonymousUser, is_superuser: bool, is_staff: bool -) -> Access: - return OrganizationlessAccess( - auth_state=access_service.get_user_auth_state( - user_id=user.id, - is_superuser=is_superuser, - is_staff=is_staff, - organization_id=None, - org_member=None, - ) - ) - - -def normalize_valid_user(user: User | RpcUser | AnonymousUser | None) -> User | RpcUser | None: - if not user or not has_user(user) or not user.is_active: - return None - return user - - -def from_user_and_rpc_user_org_context( - *, - user: User | AnonymousUser | RpcUser | None, - rpc_user_org_context: RpcUserOrganizationContext | None = None, - is_superuser: bool = False, - is_staff: bool = False, - scopes: Iterable[str] | None = None, - auth_state: RpcAuthState | None = None, -) -> Access: - if (user := normalize_valid_user(user)) is None: - return DEFAULT - - if not rpc_user_org_context or not rpc_user_org_context.member: - return organizationless_access(user, is_superuser, is_staff) - - return from_rpc_member( - rpc_user_organization_context=rpc_user_org_context, - scopes=scopes, - is_superuser=is_superuser, - is_staff=is_staff, - auth_state=auth_state, - ) - - -def from_request( - request: Request, organization: Organization | None = None, scopes: Iterable[str] | None = None -) -> Access: - if isinstance(request.auth, AuthenticatedToken) and ( - is_agent_auth(request.auth) or request.auth.superuser is not None - ): - if organization is None: - return DEFAULT - return from_auth(request.auth, organization) - - is_staff = is_active_staff(request) - - if not organization: - return from_user( - request.user, - organization=organization, - scopes=scopes, - is_superuser=is_active_superuser(request), - is_staff=is_staff, - ) - - if getattr(request, "actual_user", None) is not None: - if not should_allow_superuser_access(organization): - raise DataSecrecyError() - - if request.user.is_authenticated and request.user.is_sentry_app: - return _from_sentry_app(request.user, organization=organization) - - if is_active_superuser(request): - member: OrganizationMember | None = None - try: - member = OrganizationMember.objects.get( - user_id=request.user.id, organization_id=organization.id - ) - except OrganizationMember.DoesNotExist: - pass - auth_state = access_service.get_user_auth_state( - user_id=request.user.id, - organization_id=organization.id, - is_superuser=True, - is_staff=is_staff, - org_member=(summarize_member(member) if member is not None else None), - ) - sso_state = auth_state.sso_state - - superuser_scopes = get_superuser_scopes(auth_state, organization) - if scopes: - superuser_scopes = superuser_scopes.union(set(scopes)) - if member and (member_scopes := member.get_scopes()): - superuser_scopes = superuser_scopes.union(set(member_scopes)) - - return OrganizationGlobalAccess( - organization=organization, - _member=member, - scopes=superuser_scopes, - sso_is_valid=sso_state.is_valid, - requires_sso=sso_state.is_required, - permissions=access_service.get_permissions_for_user(request.user.id), - ) - - if request.auth is not None and not has_user(request.user): - return from_auth(request.auth, organization) - - return from_user(user=request.user, organization=organization, scopes=scopes, is_staff=is_staff) - - -# only used internally -def _from_sentry_app(user: User, organization: Organization | None = None) -> Access: - if not organization: - return NoAccess() - - sentry_app_query = SentryApp.objects.filter(proxy_user=user) - - if not sentry_app_query.exists(): - return NoAccess() - - sentry_app = sentry_app_query.get() - - if not sentry_app.is_installed_on(organization): - return NoAccess() - - return OrganizationGlobalMembership( - organization, translate_deprecated_scopes(sentry_app.scope_list), sso_is_valid=True - ) - - -def _from_rpc_sentry_app(context: RpcUserOrganizationContext | None = None) -> Access: - from sentry.sentry_apps.services.app import app_service - - if not context or context.user_id is None: - return NoAccess() - - installation = app_service.find_installation_by_proxy_user( - proxy_user_id=context.user_id, organization_id=context.organization.id - ) - if installation is None: - return NoAccess() - - return ApiOrganizationGlobalMembership( - rpc_user_organization_context=context, - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState( - is_valid=True, - is_required=False, - ), - permissions=[], - ), - scopes=translate_deprecated_scopes(installation.sentry_app.scope_list), - ) - - -def from_user( - user: User | RpcUser | AnonymousUser | None, - organization: Organization | None = None, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, -) -> Access: - if (user := normalize_valid_user(user)) is None: - return DEFAULT - - if not organization: - return organizationless_access(user, is_superuser, is_staff) - - try: - om = OrganizationMember.objects.get(user_id=user.id, organization_id=organization.id) - except OrganizationMember.DoesNotExist: - return organizationless_access(user, is_superuser, is_staff) - - # ensure cached relation - om.organization = organization - - return from_member(om, scopes=scopes, is_superuser=is_superuser, is_staff=is_staff) - - -def from_member( - member: OrganizationMember, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, -) -> Access: - scope_intersection = _intersect_member_and_token_scopes(member.get_scopes(), scopes) - - if (is_superuser or is_staff) and member.user_id is not None: - # "permissions" is a bit of a misnomer -- these are all admin level permissions, and the intent is that if you - # have them, you can only use them when you are acting, as a superuser or staff. This is intentional. - permissions = access_service.get_permissions_for_user(member.user_id) - else: - permissions = frozenset() - - return OrganizationMemberAccess(member, scope_intersection, permissions, scopes) - - -def from_rpc_member( - rpc_user_organization_context: RpcUserOrganizationContext, - scopes: Iterable[str] | None = None, - is_superuser: bool = False, - is_staff: bool = False, - auth_state: RpcAuthState | None = None, -) -> Access: - if rpc_user_organization_context.user_id is None: - return DEFAULT - - return RpcBackedAccess( - rpc_user_organization_context=rpc_user_organization_context, - scopes_upper_bound=_wrap_scopes(scopes), - auth_state=auth_state - or access_service.get_user_auth_state( - user_id=rpc_user_organization_context.user_id, - organization_id=rpc_user_organization_context.organization.id, - is_superuser=is_superuser, - is_staff=is_staff, - org_member=rpc_user_organization_context.member, - ), - ) - - -def from_auth(auth: AuthenticatedToken, organization: Organization) -> Access: - if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: - context = organization_service.get_organization_by_id( - id=organization.id, user_id=auth.user_id - ) - return from_superuser_access(auth, context) if context is not None else DEFAULT - if is_system_auth(auth): - return SystemAccess() - if is_service_account_auth(auth): - return from_service_account_auth( - auth, organization.id, bool(organization.flags.allow_joinleave) - ) - if is_agent_auth(auth): - access: Access = DEFAULT - if auth.user_id is not None and auth.organization_id == organization.id: - try: - member = OrganizationMember.objects.get( - user_id=auth.user_id, organization_id=organization.id - ) - except OrganizationMember.DoesNotExist: - pass - else: - member.organization = organization - access = from_member(member, scopes=auth.get_scopes()) - return access - auth_organization_id = auth.organization_id - if auth_organization_id is not None and auth_organization_id == organization.id: - return OrganizationGlobalAccess( - auth_organization_id, settings.SENTRY_SCOPES, sso_is_valid=True - ) - else: - return DEFAULT - - -def from_rpc_auth( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - if isinstance(auth, AuthenticatedToken) and auth.superuser is not None: - return from_superuser_access(auth, rpc_user_org_context) - if is_system_auth(auth): - return SystemAccess() - if is_service_account_auth(auth): - organization = rpc_user_org_context.organization - return from_service_account_auth(auth, organization.id, organization.flags.allow_joinleave) - if is_agent_auth(auth): - # Agents are non-user actors with member-derived, capped authority -- never the - # org-global access an org token gets. Dispatched here so the cap holds at the - # shared userless-auth choke, not only in determine_access. - return from_agent_auth(auth, rpc_user_org_context) - if auth.organization_id == rpc_user_org_context.organization.id: - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=RpcAuthState( - permissions=[], - sso_state=RpcMemberSsoState( - is_valid=True, - is_required=False, - ), - ), - scopes=settings.SENTRY_SCOPES, - ) - else: - return DEFAULT - - -def from_agent_auth( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - """Access for a Seer agent capability token: a non-user actor acting on behalf of a - member. Unlike an org token, the agent is not org-global — its authority is the - delegating member's, capped by the token's scopes, so project access follows the - member's teams. The context MUST be resolved for the delegating user_id.""" - # Bound to the org it was minted for; never honored elsewhere, even if the - # delegating user is also a member of the requested org. - if auth.organization_id != rpc_user_org_context.organization.id: - return DEFAULT - if auth.user_id != rpc_user_org_context.user_id: - return DEFAULT - # No membership (never a member, or revoked since mint) -> no access. Required - # explicitly because RpcBackedAccess would otherwise hand back the full token - # scopes uncapped when member is None. - if rpc_user_org_context.member is None: - return DEFAULT - return from_rpc_member(rpc_user_org_context, scopes=auth.get_scopes()) - - -def from_service_account_auth( - auth: AuthenticatedToken, organization_id: int, has_open_membership: bool -) -> Access: - # Bound to the account's own org, like agent tokens. - if auth.entity_id is None or auth.organization_id != organization_id: - return DEFAULT - return ServiceAccountAccess( - service_account_id=auth.entity_id, - organization_id=organization_id, - scopes=auth.get_scopes(), - has_open_membership=has_open_membership, - ) - - -def from_superuser_access( - auth: AuthenticatedToken, rpc_user_org_context: RpcUserOrganizationContext -) -> Access: - if ( - auth.superuser is None - or auth.organization_id != rpc_user_org_context.organization.id - or auth.user_id != rpc_user_org_context.user_id - ): - return DEFAULT - users = user_service.get_many(filter={"user_ids": [auth.user_id]}) - user = users[0] if users else None - if user is None: - return DEFAULT - delegated = resolve_superuser_access(auth.superuser, user, rpc_user_org_context) - if delegated is None: - return DEFAULT - scopes, _ = delegated - return ApiBackedOrganizationGlobalAccess( - rpc_user_organization_context=rpc_user_org_context, - auth_state=RpcAuthState( - sso_state=RpcMemberSsoState(is_valid=True, is_required=False), permissions=[] - ), - scopes=scopes & set(auth.get_scopes()), - ) - - -DEFAULT = NoAccess() diff --git a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py b/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py deleted file mode 100644 index ce7956c0e..000000000 --- a/packages/evals/fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py +++ /dev/null @@ -1,483 +0,0 @@ -import logging -import random -import string -from email.headerregistry import Address -from typing import Any, TypedDict, TypeIs - -from django.contrib.auth.models import AnonymousUser -from django.db import IntegrityError, router, transaction -from django.db.models import Q -from django.db.models.query import QuerySet -from django.utils.text import slugify -from drf_spectacular.utils import extend_schema -from rest_framework.exceptions import NotAuthenticated, ParseError, PermissionDenied -from rest_framework.request import Request -from rest_framework.response import Response -from rest_framework.serializers import ValidationError - -from sentry import audit_log -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import cell_silo_endpoint -from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission -from sentry.api.exceptions import ConflictError -from sentry.api.helpers.environments import get_environment_id -from sentry.api.paginator import OffsetPaginator -from sentry.api.permissions import StaffPermissionMixin -from sentry.api.serializers import serialize -from sentry.api.serializers.models.project import ( - OrganizationProjectResponse, - ProjectSummarySerializer, -) -from sentry.apidocs.constants import ( - RESPONSE_BAD_REQUEST, - RESPONSE_CONFLICT, - RESPONSE_FORBIDDEN, - RESPONSE_NOT_FOUND, - RESPONSE_UNAUTHORIZED, -) -from sentry.apidocs.examples.organization_examples import OrganizationExamples -from sentry.apidocs.examples.project_examples import ProjectExamples -from sentry.apidocs.parameters import ( - CursorQueryParam, - GlobalParams, - OrganizationParams, - VisibilityParams, -) -from sentry.apidocs.response_types import DetailResponse -from sentry.apidocs.utils import inline_sentry_response_serializer -from sentry.auth.principal import has_user, require_user -from sentry.constants import ObjectStatus -from sentry.core.endpoints.team_projects import ( - AuditData, - ProjectPostSerializer, - apply_default_project_settings, -) -from sentry.models.organization import Organization -from sentry.models.organizationmember import OrganizationMember -from sentry.models.organizationmemberteam import OrganizationMemberTeam -from sentry.models.project import Project -from sentry.models.team import Team -from sentry.search.utils import tokenize_query -from sentry.seer.agent_token import is_agent_auth -from sentry.signals import project_created, team_created -from sentry.snuba import discover, metrics_enhanced_performance -from sentry.users.models.user import User -from sentry.utils.snowflake import MaxSnowflakeRetryError - -ERR_INVALID_STATS_PERIOD = ( - "Invalid stats_period. Valid choices are '', '1h', '24h', '7d', '14d', '30d', and '90d'" -) - - -class _LegacyParamErrorResponse(TypedDict): - # Legacy nested error envelope used by this endpoint's stats_period / id - # validation; kept as-is for wire compatibility. - error: dict[str, dict[str, dict[str, str]]] - - -class _OrganizationProjectCreateResponse(OrganizationProjectResponse): - """Project-creation response: the base `OrganizationProjectResponse` shape - plus a `team_slug` key that the endpoint appends post-serialize to surface - the auto-created personal team.""" - - team_slug: str - - -# Only applies to the error `stats`; `transactionStats` always queries spans. Kept for -# now until we have the frontend stop sending it, though every option resolves to discover for the error query anyway since the -# metrics datasets is not compatible with `!event.type:transaction`. -DATASETS = { - "": discover, # in case they pass an empty query string fall back on default - "discover": discover, - "metricsEnhanced": metrics_enhanced_performance, - "metrics": metrics_enhanced_performance, -} - -CONFLICTING_TEAM_SLUG_ERROR = "A team with this slug already exists." -DISABLED_FEATURE_ERROR_STRING = "Your organization has disabled this feature for members." - - -def get_dataset(dataset_label: str) -> Any: - if dataset_label not in DATASETS: - raise ParseError(detail=f"dataset must be one of: {', '.join(DATASETS.keys())}") - return DATASETS[dataset_label] - - -def _generate_suffix() -> str: - letters = string.ascii_lowercase - return "".join(random.choice(letters) for _ in range(3)) - - -def fetch_slugifed_email_username(email: str) -> str: - return slugify(Address(addr_spec=email).username) - - -class OrganizationProjectsPermission(StaffPermissionMixin, OrganizationPermission): - scope_map = { - "GET": ["org:read", "org:write", "org:admin"], - # Intentionally lowered: org members can create projects when - # allowMemberProjectCreation is enabled on the org. - "POST": ["project:read", "project:write", "project:admin"], - } - - -@extend_schema(tags=["Organizations"]) -@cell_silo_endpoint -class OrganizationProjectsEndpoint(OrganizationEndpoint): - publish_status = { - "GET": ApiPublishStatus.PUBLIC, - "POST": ApiPublishStatus.PUBLIC, - } - permission_classes = (OrganizationProjectsPermission,) - owner = ApiOwner.FOUNDATIONS - logger = logging.getLogger("team-project.create") - - @extend_schema( - operation_id="listOrganizationProjects", - summary="List an Organization's Projects", - parameters=[ - GlobalParams.ORG_ID_OR_SLUG, - CursorQueryParam, - VisibilityParams.PER_PAGE, - OrganizationParams.PROJECT_QUERY, - ], - request=None, - responses={ - 200: inline_sentry_response_serializer( - "OrganizationProjectResponseDict", list[OrganizationProjectResponse] - ), - 401: RESPONSE_UNAUTHORIZED, - 403: RESPONSE_FORBIDDEN, - 404: RESPONSE_NOT_FOUND, - }, - examples=OrganizationExamples.LIST_PROJECTS, - ) - def get( - self, request: Request, organization: Organization - ) -> ( - Response[list[OrganizationProjectResponse]] - | Response[DetailResponse] - | Response[_LegacyParamErrorResponse] - ): - """ - Return a list of projects bound to a organization. - """ - stats_period = request.GET.get("statsPeriod") - collapse = request.GET.getlist("collapse", []) - if stats_period not in (None, "", "1h", "24h", "7d", "14d", "30d", "90d"): - return Response( - {"error": {"params": {"stats_period": {"message": ERR_INVALID_STATS_PERIOD}}}}, - status=400, - ) - elif not stats_period: - # disable stats - stats_period = None - - datasetName = request.GET.get("dataset", "discover") - dataset = get_dataset(datasetName) - - queryset: QuerySet[Project] - if is_agent_auth(request.auth): - queryset = Project.objects.filter(organization=organization) - if not request.access.has_global_access: - queryset = queryset.filter(id__in=request.access.accessible_project_ids) - elif request.auth and not has_user(request.user): - # TODO: remove this, no longer supported probably - if hasattr(request.auth, "project"): - queryset = Project.objects.filter(id=request.auth.project.id) - elif request.auth.organization_id is not None: - org = request.auth.organization_id - team_list = list(Team.objects.filter(organization_id=org)) - queryset = Project.objects.filter(teams__in=team_list) - else: - return Response( - {"detail": "Current access does not point to organization."}, status=400 - ) - else: - queryset = Project.objects.filter(organization=organization) - - order_by = ["slug"] - - if has_user(request.user): - queryset = queryset.extra( - select={ - "is_bookmarked": """exists ( - select * - from sentry_projectbookmark spb - where spb.project_id = sentry_project.id and spb.user_id = %s - )""" - }, - select_params=(request.user.id,), - ) - order_by.insert(0, "-is_bookmarked") - - query = request.GET.get("query") - if query: - tokens = tokenize_query(query) - for key, value in tokens.items(): - if key == "query": - value_s = " ".join(value) - queryset = queryset.filter( - Q(name__icontains=value_s) | Q(slug__icontains=value_s) - ) - elif key == "id": - if all(v.isdigit() for v in value): - queryset = queryset.filter(id__in=value) - else: - return Response( - { - "error": { - "params": { - "stats_period": { - "message": "All 'id' values must be integers." - } - } - } - }, - status=400, - ) - elif key == "slug": - queryset = queryset.filter(slug__in=value) - elif key == "team": - team_list = list(Team.objects.filter(organization=organization, slug__in=value)) - queryset = queryset.filter(teams__in=team_list) - elif key == "!team": - team_list = list(Team.objects.filter(organization=organization, slug__in=value)) - queryset = queryset.exclude(teams__in=team_list) - elif key == "is_member": - queryset = queryset.filter(teams__organizationmember__user_id=request.user.id) - else: - queryset = queryset.none() - - queryset = queryset.filter(status=ObjectStatus.ACTIVE).distinct() - - # TODO(davidenwang): remove this after frontend requires only paginated projects - get_all_projects = request.GET.get("all_projects") == "1" - - if get_all_projects: - queryset = queryset.order_by("slug").select_related("organization") - return Response( - serialize( - list(queryset), - request.user, - ProjectSummarySerializer(collapse=collapse, dataset=dataset), - ) - ) - else: - expand = set() - if request.GET.get("transactionStats"): - expand.add("transaction_stats") - if request.GET.get("sessionStats"): - expand.add("session_stats") - - expand_context = {"options": request.GET.getlist("options") or []} - if expand_context: - expand.add("options") - - def serialize_on_result(result): - environment_id = get_environment_id(request, organization.id) - serializer = ProjectSummarySerializer( - environment_id=environment_id, - stats_period=stats_period, - expand=expand, - expand_context=expand_context, - collapse=collapse, - dataset=dataset, - ) - return serialize(result, request.user, serializer) - - return self.paginate( - request=request, - queryset=queryset, - order_by=order_by, - on_results=serialize_on_result, - paginator_cls=OffsetPaginator, - ) - - def should_add_creator_to_team(self, user: User | AnonymousUser) -> TypeIs[User]: - return has_user(user) - - @extend_schema( - tags=["Projects"], - operation_id="createOrganizationProject", - summary="Create a Project for an Organization", - parameters=[GlobalParams.ORG_ID_OR_SLUG], - request=ProjectPostSerializer, - responses={ - 201: ProjectSummarySerializer, - 400: RESPONSE_BAD_REQUEST, - 401: RESPONSE_UNAUTHORIZED, - 403: RESPONSE_FORBIDDEN, - 404: RESPONSE_NOT_FOUND, - 409: RESPONSE_CONFLICT, - }, - examples=ProjectExamples.CREATE_PROJECT, - description=( - "Create a new project for an organization. A personal team (`team-{username}`) " - "is automatically created for the caller with Team Admin role, and the project is " - "bound to it. If the org has member project creation disabled " - "(`disable_member_project_creation`), `org:write` scope is required." - ), - ) - def post( - self, request: Request, organization: Organization - ) -> Response[_OrganizationProjectCreateResponse]: - """ - Create a new project for an organization. - - Auto-creates a personal team (``team-{username}``) for the caller with Team Admin - role, then creates the project under that team. A random suffix is appended to the - team slug if the default name is already taken (up to five attempts). - """ - serializer = ProjectPostSerializer( - data=request.data, context={"organization": organization} - ) - - if not serializer.is_valid(): - raise ValidationError(serializer.errors) - # The new project's default team is the creator's own, which needs a person. - require_user(request) - if not self.should_add_creator_to_team(request.user): - raise NotAuthenticated("User is not authenticated") - - result = serializer.validated_data - - if organization.flags.disable_member_project_creation and not request.access.has_scope( - "org:write" - ): - raise PermissionDenied(detail=DISABLED_FEATURE_ERROR_STRING) - - # parse the email to retrieve the username before the "@" - parsed_email = fetch_slugifed_email_username(request.user.email) - - project_name = result["name"] - default_team_slug = f"team-{parsed_email}" - suffixed_team_slug = default_team_slug - - # attempt to a maximum of 5 times to add a suffix to team slug until it is unique - for _ in range(5): - if not Team.objects.filter(organization=organization, slug=suffixed_team_slug).exists(): - break - suffixed_team_slug = f"{default_team_slug}-{_generate_suffix()}" - else: - raise ConflictError( - { - "detail": "Unable to create a default team for this user. Please try again.", - } - ) - default_team_slug = suffixed_team_slug - - try: - with transaction.atomic(router.db_for_write(Team)): - team = Team.objects.create( - name=default_team_slug, - slug=default_team_slug, - idp_provisioned=result.get("idp_provisioned", False), - organization=organization, - ) - member = OrganizationMember.objects.get( - user_id=request.user.id, organization=organization - ) - OrganizationMemberTeam.objects.create( - team=team, - organizationmember=member, - role="admin", - ) - project = Project.objects.create( - name=project_name, - # slug is *not* set so we get an automatic one - organization=organization, - platform=result.get("platform"), - ) - project.add_team(team) - except (IntegrityError, MaxSnowflakeRetryError): - raise ConflictError( - { - "non_field_errors": [CONFLICTING_TEAM_SLUG_ERROR], - "detail": CONFLICTING_TEAM_SLUG_ERROR, - } - ) - except OrganizationMember.DoesNotExist: - raise PermissionDenied( - detail="You must be a member of the organization to join a new team as a Team Admin" - ) - - team_created.send_robust( - organization=organization, - user=request.user, - team=team, - sender=self.__class__, - ) - self.create_audit_entry( - request=request, - organization=organization, - target_object=team.id, - event=audit_log.get_event_id("TEAM_ADD"), - data=team.get_audit_log_data(), - ) - - common_audit_data: AuditData = { - "request": request, - "organization": team.organization, - "target_object": project.id, - } - origin = request.data.get("origin") - if origin: - self.create_audit_entry( - **common_audit_data, - event=audit_log.get_event_id("PROJECT_ADD_WITH_ORIGIN"), - data={ - **project.get_audit_log_data(), - "origin": origin, - }, - ) - else: - self.create_audit_entry( - **common_audit_data, - event=audit_log.get_event_id("PROJECT_ADD"), - data={**project.get_audit_log_data()}, - ) - - apply_default_project_settings(organization, project) - - project_created.send_robust( - project=project, - user=request.user, - default_rules=result.get("default_rules", True), - origin=origin, - sender=self, - ) - self.create_audit_entry( - request=request, - organization=team.organization, - event=audit_log.get_event_id("TEAM_AND_PROJECT_CREATED"), - data={"team_slug": default_team_slug, "project_slug": project_name}, - ) - self.logger.info( - "created team through project creation flow", - extra={"team_slug": default_team_slug, "project_slug": project_name}, - ) - base: OrganizationProjectResponse = serialize( - project, request.user, ProjectSummarySerializer(collapse=["unusedFeatures"]) - ) - serialized_response: _OrganizationProjectCreateResponse = { - **base, - "team_slug": team.slug, - } - return Response(serialized_response, status=201) - - -@cell_silo_endpoint -class OrganizationProjectsCountEndpoint(OrganizationEndpoint): - publish_status = { - "GET": ApiPublishStatus.PRIVATE, - } - - def get(self, request: Request, organization: Organization) -> Response: - queryset = Project.objects.filter(organization=organization) - - all_projects = queryset.count() - my_projects = queryset.filter(teams__organizationmember__user_id=request.user.id).count() - - return Response({"allProjects": all_projects, "myProjects": my_projects}) diff --git a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py b/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py deleted file mode 100644 index bddc4209e..000000000 --- a/packages/evals/fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py +++ /dev/null @@ -1,142 +0,0 @@ -import logging - -from django.conf import settings -from django.db import router, transaction -from rest_framework.request import Request -from rest_framework.response import Response - -import sentry -from sentry import options -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import Endpoint, all_silo_endpoint -from sentry.api.permissions import SuperuserPermission -from sentry.options.store import Key -from sentry.utils.email import is_smtp_enabled - -logger = logging.getLogger("sentry") - -SYSTEM_OPTIONS_ALLOWLIST = frozenset( - [ - # Used during setup before the superadmin role with the options.admin permission is authed - "system.admin-email" - ] -) - - -def _is_secret(k: Key) -> bool: - keywords = ["secret", "private", "token"] - return ((k.flags & options.FLAG_CREDENTIAL) != 0) or any( - [keyword in k.name for keyword in keywords] - ) - - -@all_silo_endpoint -class SystemOptionsEndpoint(Endpoint): - publish_status = { - "GET": ApiPublishStatus.PRIVATE, - "PUT": ApiPublishStatus.PRIVATE, - } - owner = ApiOwner.HYBRID_CLOUD - permission_classes = (SuperuserPermission,) - - def get(self, request: Request) -> Response: - query = request.GET.get("query") - if query == "is:required": - option_list = options.filter(flag=options.FLAG_REQUIRED) - elif query: - return Response(f"{query} is not a supported search query", status=400) - else: - option_list = options.all() - - smtp_disabled = not is_smtp_enabled() - - results = {} - for k in option_list: - disabled, disabled_reason = False, None - - if smtp_disabled and k.name[:5] == "mail.": - disabled_reason, disabled = "smtpDisabled", True - elif bool( - k.flags & options.FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(k.name) - ): - # TODO(mattrobenolt): Expose this as a property on Key. - disabled_reason, disabled = "diskPriority", True - - # TODO(mattrobenolt): help, placeholder, title, type - results[k.name] = { - "value": options.get(k.name) if not _is_secret(k) else "[redacted]", - "field": { - "default": k.default(), - "required": bool(k.flags & options.FLAG_REQUIRED), - "disabled": disabled, - "disabledReason": disabled_reason, - "isSet": options.isset(k.name), - "allowEmpty": bool(k.flags & options.FLAG_ALLOW_EMPTY), - }, - } - - return Response(results) - - def has_permission(self, request: Request) -> bool: - if settings.SENTRY_SELF_HOSTED and request.user.is_superuser: - return True - if not request.access.has_permission("options.admin"): - # We ignore options.admin permission if all keys in the update match the allowlist. - return bool(request.data) and all( - [k in SYSTEM_OPTIONS_ALLOWLIST for k in request.data.keys()] - ) - - return True - - def put(self, request: Request) -> Response: - if not self.has_permission(request): - return Response(status=403) - - # TODO(dcramer): this should validate options before saving them - for k, v in request.data.items(): - if v and isinstance(v, str): - v = v.strip() - try: - option = options.lookup_key(k) - except options.UnknownOption: - # TODO(dcramer): unify API errors - return Response( - {"error": "unknown_option", "errorDetail": {"option": k}}, status=400 - ) - - try: - with transaction.atomic(router.db_for_write(options.default_store.model)): - if not (option.flags & options.FLAG_ALLOW_EMPTY) and not v: - options.delete(k) - else: - options.set(k, v, channel=options.UpdateChannel.APPLICATION) - - logger.info( - "options.update", - extra={ - "ip_address": request.META["REMOTE_ADDR"], - "user_id": request.user.id, - "option_key": k, - "option_value": "[redacted]" if _is_secret(option) else v, - }, - ) - except (TypeError, AssertionError) as e: - # TODO(chadwhitacre): Use a custom exception for the - # immutability case, especially since asserts disappear with - # `python -O`. - return Response( - { - "error": "invalid_type" if type(e) is TypeError else "immutable_option", - "errorDetail": {"option": k, "message": str(e)}, - }, - status=400, - ) - # TODO(dcramer): this has nothing to do with configuring options and - # should not be set here - options.set( - "sentry:version-configured", - sentry.get_version(), - channel=options.UpdateChannel.APPLICATION, - ) - return Response(status=200) diff --git a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py b/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py deleted file mode 100644 index af2fd93ec..000000000 --- a/packages/evals/fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py +++ /dev/null @@ -1,141 +0,0 @@ -import logging -from typing import Any - -from django.conf import settings -from django.db import router, transaction -from rest_framework.request import Request -from rest_framework.response import Response - -import sentry -from sentry import options -from sentry.api.api_owners import ApiOwner -from sentry.api.api_publish_status import ApiPublishStatus -from sentry.api.base import Endpoint, all_silo_endpoint -from sentry.api.permissions import SuperuserPermission -from sentry.utils.email import is_smtp_enabled - -logger = logging.getLogger("sentry") - -SYSTEM_OPTIONS_ALLOWLIST = frozenset( - [ - # Used during setup before the superadmin role with the options.admin permission is authed - "system.admin-email" - ] -) - - -@all_silo_endpoint -class SystemOptionsEndpoint(Endpoint): - publish_status = { - "GET": ApiPublishStatus.PRIVATE, - "PUT": ApiPublishStatus.PRIVATE, - } - owner = ApiOwner.HYBRID_CLOUD - permission_classes = (SuperuserPermission,) - - def get(self, request: Request) -> Response: - query = request.GET.get("query") - if query == "is:required": - option_list = options.filter(flag=options.FLAG_REQUIRED) - elif query: - return Response(f"{query} is not a supported search query", status=400) - else: - option_list = options.all() - - smtp_disabled = not is_smtp_enabled() - - results = {} - for k in option_list: - disabled, disabled_reason = False, None - - if smtp_disabled and k.name[:5] == "mail.": - disabled_reason, disabled = "smtpDisabled", True - elif bool( - k.flags & options.FLAG_PRIORITIZE_DISK and settings.SENTRY_OPTIONS.get(k.name) - ): - # TODO(mattrobenolt): Expose this as a property on Key. - disabled_reason, disabled = "diskPriority", True - - # TODO(mattrobenolt): help, placeholder, title, type - results[k.name] = { - "value": options.get(k.name) if not self.__is_secret(k) else "[redacted]", - "field": { - "default": k.default(), - "required": bool(k.flags & options.FLAG_REQUIRED), - "disabled": disabled, - "disabledReason": disabled_reason, - "isSet": options.isset(k.name), - "allowEmpty": bool(k.flags & options.FLAG_ALLOW_EMPTY), - }, - } - - return Response(results) - - def __is_secret(self, k: Any) -> bool: - keywords = ["secret", "private", "token"] - return (k.flags & options.FLAG_CREDENTIAL) or any( - [keyword in k.name for keyword in keywords] - ) - - def has_permission(self, request: Request) -> bool: - if settings.SENTRY_SELF_HOSTED and request.user.is_superuser: - return True - if not request.access.has_permission("options.admin"): - # We ignore options.admin permission if all keys in the update match the allowlist. - return bool(request.data) and all( - [k in SYSTEM_OPTIONS_ALLOWLIST for k in request.data.keys()] - ) - - return True - - def put(self, request: Request) -> Response: - if not self.has_permission(request): - return Response(status=403) - - # TODO(dcramer): this should validate options before saving them - for k, v in request.data.items(): - if v and isinstance(v, str): - v = v.strip() - try: - option = options.lookup_key(k) - except options.UnknownOption: - # TODO(dcramer): unify API errors - return Response( - {"error": "unknown_option", "errorDetail": {"option": k}}, status=400 - ) - - try: - with transaction.atomic(router.db_for_write(options.default_store.model)): - if not (option.flags & options.FLAG_ALLOW_EMPTY) and not v: - options.delete(k) - else: - options.set(k, v, channel=options.UpdateChannel.APPLICATION) - - logger.info( - "options.update", - extra={ - "ip_address": request.META["REMOTE_ADDR"], - "user_id": request.user.id, - "option_key": k, - "option_value": v, - }, - ) - except (TypeError, AssertionError) as e: - # TODO(chadwhitacre): Use a custom exception for the - # immutability case, especially since asserts disappear with - # `python -O`. - return Response( - { - "error": "invalid_type" if type(e) is TypeError else "immutable_option", - "errorDetail": {"option": k, "message": str(e)}, - }, - status=400, - ) - # TODO(dcramer): this has nothing to do with configuring options and - # should not be set here - options.set( - "sentry:version-configured", - sentry.get_version(), - channel=options.UpdateChannel.APPLICATION, - ) - return Response(status=200) diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py b/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py deleted file mode 100644 index 05f089e24..000000000 --- a/packages/evals/fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py +++ /dev/null @@ -1,303 +0,0 @@ -import datetime -import logging -from datetime import timedelta -from urllib.parse import urljoin -from urllib.robotparser import RobotFileParser - -from dateutil.parser import parse as parse_datetime -from django.utils import timezone - -from sentry import features, options -from sentry.exceptions import RestrictedIPAddress -from sentry.http import safe_urlopen -from sentry.locks import locks -from sentry.models.organization import Organization -from sentry.models.project import Project -from sentry.tasks.base import instrumented_task -from sentry.taskworker.namespaces import uptime_tasks -from sentry.uptime.autodetect.ranking import ( - delete_candidate_projects_for_org, - delete_candidate_urls_for_project, - delete_organization_bucket, - get_candidate_projects_for_org, - get_candidate_urls_for_project, - get_organization_bucket, - should_autodetect_for_organization, - should_autodetect_for_project, -) -from sentry.uptime.subscriptions.subscriptions import ( - create_uptime_detector, - delete_uptime_detector, - get_auto_monitored_detectors_for_project, - is_url_auto_monitored_for_project, -) -from sentry.uptime.types import DEFAULT_2XX_STATUS_ASSERTION, UptimeMonitorMode -from sentry.uptime.utils import get_cluster -from sentry.utils import metrics -from sentry.utils.hashlib import md5_text -from sentry.utils.locking import UnableToAcquireLock -from sentry.workflow_engine.models.detector import Detector - -UPTIME_USER_AGENT = "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)" -LAST_PROCESSED_KEY = "uptime_detector_last_processed" -SCHEDULER_LOCK_KEY = "uptime_detector_scheduler_lock" -FAILED_URL_RETRY_FREQ = timedelta(days=7) -URL_MIN_TIMES_SEEN = 5 -URL_MIN_PERCENT = 0.05 -# Default value for how often we should run these subscriptions when onboarding them -ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS = int(timedelta(minutes=60).total_seconds()) -# Default timeout for auto-detected uptime monitors -ONBOARDING_SUBSCRIPTION_TIMEOUT_MS = 10_000 - -logger = logging.getLogger("sentry.uptime-url-autodetection") - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.schedule_detections", - namespace=uptime_tasks, - processing_deadline_duration=60, -) -def schedule_autodetections(): - """ - Runs regularly and fires off a task for each autodetection bucket that needs to be run since - this task last ran. - """ - lock = locks.get( - SCHEDULER_LOCK_KEY, - duration=60, - name="uptime.detection.schedule_detections", - ) - try: - with lock.acquire(): - cluster = get_cluster() - last_processed = cluster.get(LAST_PROCESSED_KEY) - if last_processed is None: - last_processed = timezone.now().replace(second=0, microsecond=0) - else: - last_processed = datetime.datetime.fromtimestamp( - int(last_processed), tz=datetime.UTC - ) - - minutes_since_last_processed = int( - (timezone.now() - last_processed) / timedelta(minutes=1) - ) - for _ in range(minutes_since_last_processed): - metrics.incr("uptime.detectors.scheduler.scheduled_bucket") - last_processed = last_processed + timedelta(minutes=1) - process_autodetection_bucket.delay(last_processed.isoformat()) - - cluster.set(LAST_PROCESSED_KEY, int(last_processed.timestamp()), timedelta(hours=1)) - except UnableToAcquireLock: - # If we can't acquire the lock it just means another task is already handling scheduling, - # so just exit - metrics.incr("uptime.detectors.scheduler.unable_to_acquire_lock") - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.process_detection_bucket", - namespace=uptime_tasks, -) -def process_autodetection_bucket(bucket: str): - """ - Schedules url autodetection for all projects in this time bucket that saw promising urls. - """ - date_bucket = parse_datetime(bucket) - - for organization_id in get_organization_bucket(date_bucket): - metrics.incr("uptime.detectors.scheduler.scheduled_organization") - process_organization_url_ranking.delay(organization_id) - delete_organization_bucket(date_bucket) - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.process_organization_url_ranking", - namespace=uptime_tasks, - processing_deadline_duration=20, -) -def process_organization_url_ranking(organization_id: int): - org = Organization.objects.get_from_cache(id=organization_id) - logger.info( - "uptime.process_organization", - extra={"organization_id": org.id}, - ) - should_autodetect = should_autodetect_for_organization(org) - - for project_id, project_count in get_candidate_projects_for_org(org): - project = Project.objects.get_from_cache(id=project_id) - if not should_autodetect: - # We still want to clear up these urls even if we're no longer autodetecting - delete_candidate_urls_for_project(project) - else: - if process_project_url_ranking(project, project_count): - metrics.incr("uptime.detectors.scheduler.detected_url_for_organization") - should_autodetect = False - - delete_candidate_projects_for_org(org) - - -def process_project_url_ranking(project: Project, project_url_count: int) -> bool: - """ - Looks at candidate urls for a project and determines whether we should start monitoring them - """ - logger.info( - "uptime.process_project", - extra={ - "project_id": project.id, - "project_url_count": project_url_count, - }, - ) - if not should_autodetect_for_project(project): - metrics.incr("uptime.detectors.project_detection_skipped") - return False - - found_url = False - - for url, url_count in get_candidate_urls_for_project(project, limit=5): - if process_candidate_url(project, project_url_count, url, url_count): - found_url = True - break - else: - # TODO: If we don't find any urls to monitor, we want to increment a counter in redis and check the value. - # After a number of failures, we want to stop checking for this project for a while. - pass - - delete_candidate_urls_for_project(project) - return found_url - - -def process_candidate_url( - project: Project, project_url_count: int, url: str, url_count: int -) -> bool: - """ - Takes a candidate url for a project and determines whether we should create an uptime subscription for it. - Checks that: - - URL has been seen at least `URL_MIN_TIMES_SEEN` times and is seen in at least `URL_MIN_PERCENT` of events with urls - - URL hasn't already been checked and failed recently - - Whether we are already monitoring this url for this project - - Whether the url's robots.txt will allow us to monitor this url - - If the url passes, and we don't already have a subscription for it, then create a new remote subscription for the - url and delete any existing automated monitors. - """ - logger.info( - "uptime.process_project_url", - extra={ - "project_id": project.id, - "project_url_count": project_url_count, - "url": url, - "url_count": url_count, - }, - ) - # The url has to be seen a minimum number of times, and make up at least - # a certain percentage of all urls seen in this project - if url_count < URL_MIN_TIMES_SEEN or url_count / project_url_count < URL_MIN_PERCENT: - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "below_thresholds"}) - return False - - # See if we're already auto monitoring this url on this project - if is_url_auto_monitored_for_project(project, url): - # Just mark this successful so `process_project_url_ranking` will choose to not process urls for this project - # for a week - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "already_monitored"}) - return True - - # Check whether we've recently attempted to monitor this url recently and failed. - if is_failed_url(url): - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "previously_failed"}) - return False - - # Check robots.txt to see if it's ok for us to attempt to monitor this url - if not check_url_robots_txt(url): - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "robots_txt"}) - logger.info( - "uptime.url_failed_robots_txt_check", - extra={ - "url": url, - "project": project.id, - }, - ) - set_failed_url(url) - return False - - logger.info( - "uptime.url_autodetected", - extra={ - "url": url, - "project": project.id, - }, - ) - if options.get("uptime.automatic-subscription-creation") and features.has( - "organizations:uptime", project.organization - ): - # If we hit this point, then the url looks worth monitoring. Create an uptime subscription in monitor mode. - monitor_url_for_project(project, url) - # Disable autodetection on this project and organization now that we've successfully found a hostname - project.update_option("sentry:uptime_autodetection", False) - project.organization.update_option("sentry:uptime_autodetection", False) - - metrics.incr("uptime.detectors.candidate_url.succeeded", sample_rate=1.0) - return True - - -def monitor_url_for_project(project: Project, url: str) -> Detector: - """ - Start monitoring a url for a project. Creates a subscription using our onboarding interval and links the project to - it. Also deletes any other auto-detected monitors since this one should replace them. - """ - for uptime_detector in get_auto_monitored_detectors_for_project(project): - delete_uptime_detector(uptime_detector) - metrics.incr("uptime.detectors.candidate_url.monitor_created", sample_rate=1.0) - return create_uptime_detector( - project, - # TODO(epurkhiser): This is where we would put the environment object - # from autodetection if we decide to do that. - environment=None, - url=url, - interval_seconds=ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS, - timeout_ms=ONBOARDING_SUBSCRIPTION_TIMEOUT_MS, - name=f"Uptime Monitoring for {url}", - mode=UptimeMonitorMode.AUTO_DETECTED_ONBOARDING, - assertion=DEFAULT_2XX_STATUS_ASSERTION, - ) - - -def is_failed_url(url: str) -> bool: - key = get_failed_url_key(url) - return get_cluster().exists(key) == 1 - - -def set_failed_url(url: str) -> None: - """ - If we failed to monitor a url for some reason, skip processing it for FAILED_URL_RETRY_FREQ - """ - key = get_failed_url_key(url) - # TODO: Jitter the expiry here, so we don't retry all at the same time. - get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ) - - -def get_failed_url_key(url: str) -> str: - return f"f:u:{md5_text(url).hexdigest()}" - - -def check_url_robots_txt(url: str) -> bool: - try: - return get_robots_txt_parser(url).can_fetch(UPTIME_USER_AGENT, url) - except RestrictedIPAddress: - logger.warning("Blocked robots.txt fetch to restricted IP", exc_info=True) - return False - except Exception: - logger.warning("Failed to check robots.txt", exc_info=True) - return True - - -def get_robots_txt_parser(url: str) -> RobotFileParser: - robots_url = urljoin(url, "robots.txt") - response = safe_urlopen(robots_url, timeout=10, allow_redirects=True) - robot_parser = RobotFileParser(url=robots_url) - if response.status_code in (401, 403): - robot_parser.parse(["User-agent: *", "Disallow: /"]) - elif response.status_code >= 400: - robot_parser.parse([]) - else: - robot_parser.parse(response.text.splitlines()) - return robot_parser diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md deleted file mode 100644 index f97b1d1f6..000000000 --- a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md +++ /dev/null @@ -1,105 +0,0 @@ -# Functional Source License, Version 1.1, Apache 2.0 Future License - -## Abbreviation - -FSL-1.1-Apache-2.0 - -## Notice - -Copyright 2008-2024 Functional Software, Inc. dba Sentry - -## Terms and Conditions - -### Licensor ("We") - -The party offering the Software under these Terms and Conditions. - -### The Software - -The "Software" is each version of the software that we make available under -these Terms and Conditions, as indicated by our inclusion of these Terms and -Conditions with the Software. - -### License Grant - -Subject to your compliance with this License Grant and the Patents, -Redistribution and Trademark clauses below, we hereby grant you the right to -use, copy, modify, create derivative works, publicly perform, publicly display -and redistribute the Software for any Permitted Purpose identified below. - -### Permitted Purpose - -A Permitted Purpose is any purpose other than a Competing Use. A Competing Use -means making the Software available to others in a commercial product or -service that: - -1. substitutes for the Software; - -2. substitutes for any other product or service we offer using the Software - that exists as of the date we make the Software available; or - -3. offers the same or substantially similar functionality as the Software. - -Permitted Purposes specifically include using the Software: - -1. for your internal use and access; - -2. for non-commercial education; - -3. for non-commercial research; and - -4. in connection with professional services that you provide to a licensee - using the Software in accordance with these Terms and Conditions. - -### Patents - -To the extent your use for a Permitted Purpose would necessarily infringe our -patents, the license grant above includes a license under our patents. If you -make a claim against any party that the Software infringes or contributes to -the infringement of any patent, then your patent license to the Software ends -immediately. - -### Redistribution - -The Terms and Conditions apply to all copies, modifications and derivatives of -the Software. - -If you redistribute any copies, modifications or derivatives of the Software, -you must include a copy of or a link to these Terms and Conditions and not -remove any copyright notices provided in or with the Software. - -### Disclaimer - -THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR -PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - -IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE -SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, -EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. - -### Trademarks - -Except for displaying the License Details and identifying us as the origin of -the Software, you have no right under these Terms and Conditions to use our -trademarks, trade names, service marks or product names. - -## Grant of Future License - -We hereby irrevocably grant you an additional license to use the Software under -the Apache License, Version 2.0 that is effective on the second anniversary of -the date we make the Software available. On or after that date, you may use the -Software under the Apache License, Version 2.0, in which case the following -will apply: - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. diff --git a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py b/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py deleted file mode 100644 index 9a30cfe54..000000000 --- a/packages/evals/fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py +++ /dev/null @@ -1,291 +0,0 @@ -import datetime -import logging -from datetime import timedelta -from urllib.parse import urljoin -from urllib.robotparser import RobotFileParser - -from dateutil.parser import parse as parse_datetime -from django.utils import timezone - -from sentry import features, options -from sentry.locks import locks -from sentry.models.organization import Organization -from sentry.models.project import Project -from sentry.tasks.base import instrumented_task -from sentry.taskworker.namespaces import uptime_tasks -from sentry.uptime.autodetect.ranking import ( - delete_candidate_projects_for_org, - delete_candidate_urls_for_project, - delete_organization_bucket, - get_candidate_projects_for_org, - get_candidate_urls_for_project, - get_organization_bucket, - should_autodetect_for_organization, - should_autodetect_for_project, -) -from sentry.uptime.subscriptions.subscriptions import ( - create_uptime_detector, - delete_uptime_detector, - get_auto_monitored_detectors_for_project, - is_url_auto_monitored_for_project, -) -from sentry.uptime.types import DEFAULT_2XX_STATUS_ASSERTION, UptimeMonitorMode -from sentry.uptime.utils import get_cluster -from sentry.utils import metrics -from sentry.utils.hashlib import md5_text -from sentry.utils.locking import UnableToAcquireLock -from sentry.workflow_engine.models.detector import Detector - -UPTIME_USER_AGENT = "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)" -LAST_PROCESSED_KEY = "uptime_detector_last_processed" -SCHEDULER_LOCK_KEY = "uptime_detector_scheduler_lock" -FAILED_URL_RETRY_FREQ = timedelta(days=7) -URL_MIN_TIMES_SEEN = 5 -URL_MIN_PERCENT = 0.05 -# Default value for how often we should run these subscriptions when onboarding them -ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS = int(timedelta(minutes=60).total_seconds()) -# Default timeout for auto-detected uptime monitors -ONBOARDING_SUBSCRIPTION_TIMEOUT_MS = 10_000 - -logger = logging.getLogger("sentry.uptime-url-autodetection") - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.schedule_detections", - namespace=uptime_tasks, - processing_deadline_duration=60, -) -def schedule_autodetections(): - """ - Runs regularly and fires off a task for each autodetection bucket that needs to be run since - this task last ran. - """ - lock = locks.get( - SCHEDULER_LOCK_KEY, - duration=60, - name="uptime.detection.schedule_detections", - ) - try: - with lock.acquire(): - cluster = get_cluster() - last_processed = cluster.get(LAST_PROCESSED_KEY) - if last_processed is None: - last_processed = timezone.now().replace(second=0, microsecond=0) - else: - last_processed = datetime.datetime.fromtimestamp( - int(last_processed), tz=datetime.UTC - ) - - minutes_since_last_processed = int( - (timezone.now() - last_processed) / timedelta(minutes=1) - ) - for _ in range(minutes_since_last_processed): - metrics.incr("uptime.detectors.scheduler.scheduled_bucket") - last_processed = last_processed + timedelta(minutes=1) - process_autodetection_bucket.delay(last_processed.isoformat()) - - cluster.set(LAST_PROCESSED_KEY, int(last_processed.timestamp()), timedelta(hours=1)) - except UnableToAcquireLock: - # If we can't acquire the lock it just means another task is already handling scheduling, - # so just exit - metrics.incr("uptime.detectors.scheduler.unable_to_acquire_lock") - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.process_detection_bucket", - namespace=uptime_tasks, -) -def process_autodetection_bucket(bucket: str): - """ - Schedules url autodetection for all projects in this time bucket that saw promising urls. - """ - date_bucket = parse_datetime(bucket) - - for organization_id in get_organization_bucket(date_bucket): - metrics.incr("uptime.detectors.scheduler.scheduled_organization") - process_organization_url_ranking.delay(organization_id) - delete_organization_bucket(date_bucket) - - -@instrumented_task( - name="sentry.uptime.detectors.tasks.process_organization_url_ranking", - namespace=uptime_tasks, - processing_deadline_duration=20, -) -def process_organization_url_ranking(organization_id: int): - org = Organization.objects.get_from_cache(id=organization_id) - logger.info( - "uptime.process_organization", - extra={"organization_id": org.id}, - ) - should_autodetect = should_autodetect_for_organization(org) - - for project_id, project_count in get_candidate_projects_for_org(org): - project = Project.objects.get_from_cache(id=project_id) - if not should_autodetect: - # We still want to clear up these urls even if we're no longer autodetecting - delete_candidate_urls_for_project(project) - else: - if process_project_url_ranking(project, project_count): - metrics.incr("uptime.detectors.scheduler.detected_url_for_organization") - should_autodetect = False - - delete_candidate_projects_for_org(org) - - -def process_project_url_ranking(project: Project, project_url_count: int) -> bool: - """ - Looks at candidate urls for a project and determines whether we should start monitoring them - """ - logger.info( - "uptime.process_project", - extra={ - "project_id": project.id, - "project_url_count": project_url_count, - }, - ) - if not should_autodetect_for_project(project): - metrics.incr("uptime.detectors.project_detection_skipped") - return False - - found_url = False - - for url, url_count in get_candidate_urls_for_project(project, limit=5): - if process_candidate_url(project, project_url_count, url, url_count): - found_url = True - break - else: - # TODO: If we don't find any urls to monitor, we want to increment a counter in redis and check the value. - # After a number of failures, we want to stop checking for this project for a while. - pass - - delete_candidate_urls_for_project(project) - return found_url - - -def process_candidate_url( - project: Project, project_url_count: int, url: str, url_count: int -) -> bool: - """ - Takes a candidate url for a project and determines whether we should create an uptime subscription for it. - Checks that: - - URL has been seen at least `URL_MIN_TIMES_SEEN` times and is seen in at least `URL_MIN_PERCENT` of events with urls - - URL hasn't already been checked and failed recently - - Whether we are already monitoring this url for this project - - Whether the url's robots.txt will allow us to monitor this url - - If the url passes, and we don't already have a subscription for it, then create a new remote subscription for the - url and delete any existing automated monitors. - """ - logger.info( - "uptime.process_project_url", - extra={ - "project_id": project.id, - "project_url_count": project_url_count, - "url": url, - "url_count": url_count, - }, - ) - # The url has to be seen a minimum number of times, and make up at least - # a certain percentage of all urls seen in this project - if url_count < URL_MIN_TIMES_SEEN or url_count / project_url_count < URL_MIN_PERCENT: - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "below_thresholds"}) - return False - - # See if we're already auto monitoring this url on this project - if is_url_auto_monitored_for_project(project, url): - # Just mark this successful so `process_project_url_ranking` will choose to not process urls for this project - # for a week - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "already_monitored"}) - return True - - # Check whether we've recently attempted to monitor this url recently and failed. - if is_failed_url(url): - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "previously_failed"}) - return False - - # Check robots.txt to see if it's ok for us to attempt to monitor this url - if not check_url_robots_txt(url): - metrics.incr("uptime.detectors.candidate_url.failed", tags={"reason": "robots_txt"}) - logger.info( - "uptime.url_failed_robots_txt_check", - extra={ - "url": url, - "project": project.id, - }, - ) - set_failed_url(url) - return False - - logger.info( - "uptime.url_autodetected", - extra={ - "url": url, - "project": project.id, - }, - ) - if options.get("uptime.automatic-subscription-creation") and features.has( - "organizations:uptime", project.organization - ): - # If we hit this point, then the url looks worth monitoring. Create an uptime subscription in monitor mode. - monitor_url_for_project(project, url) - # Disable autodetection on this project and organization now that we've successfully found a hostname - project.update_option("sentry:uptime_autodetection", False) - project.organization.update_option("sentry:uptime_autodetection", False) - - metrics.incr("uptime.detectors.candidate_url.succeeded", sample_rate=1.0) - return True - - -def monitor_url_for_project(project: Project, url: str) -> Detector: - """ - Start monitoring a url for a project. Creates a subscription using our onboarding interval and links the project to - it. Also deletes any other auto-detected monitors since this one should replace them. - """ - for uptime_detector in get_auto_monitored_detectors_for_project(project): - delete_uptime_detector(uptime_detector) - metrics.incr("uptime.detectors.candidate_url.monitor_created", sample_rate=1.0) - return create_uptime_detector( - project, - # TODO(epurkhiser): This is where we would put the environment object - # from autodetection if we decide to do that. - environment=None, - url=url, - interval_seconds=ONBOARDING_SUBSCRIPTION_INTERVAL_SECONDS, - timeout_ms=ONBOARDING_SUBSCRIPTION_TIMEOUT_MS, - name=f"Uptime Monitoring for {url}", - mode=UptimeMonitorMode.AUTO_DETECTED_ONBOARDING, - assertion=DEFAULT_2XX_STATUS_ASSERTION, - ) - - -def is_failed_url(url: str) -> bool: - key = get_failed_url_key(url) - return get_cluster().exists(key) == 1 - - -def set_failed_url(url: str) -> None: - """ - If we failed to monitor a url for some reason, skip processing it for FAILED_URL_RETRY_FREQ - """ - key = get_failed_url_key(url) - # TODO: Jitter the expiry here, so we don't retry all at the same time. - get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ) - - -def get_failed_url_key(url: str) -> str: - return f"f:u:{md5_text(url).hexdigest()}" - - -def check_url_robots_txt(url: str) -> bool: - try: - return get_robots_txt_parser(url).can_fetch(UPTIME_USER_AGENT, url) - except Exception: - logger.warning("Failed to check robots.txt", exc_info=True) - return True - - -def get_robots_txt_parser(url: str) -> RobotFileParser: - robot_parser = RobotFileParser(url=urljoin(url, "robots.txt")) - robot_parser.read() - return robot_parser diff --git a/packages/evals/src/index.test.ts b/packages/evals/src/index.test.ts index e234e5d75..91de072c8 100644 --- a/packages/evals/src/index.test.ts +++ b/packages/evals/src/index.test.ts @@ -231,10 +231,6 @@ describe('standalone scenario files', () => { ...discoverEvalScenarioFiles('code-review', evalsDir), ...discoverEvalScenarioFiles('security-review', evalsDir), ...discoverEvalScenarioFiles('verification', evalsDir), - ...discoverEvalScenarioFiles('wrdn-authz', evalsDir), - ...discoverEvalScenarioFiles('wrdn-code-execution', evalsDir), - ...discoverEvalScenarioFiles('wrdn-data-exfil', evalsDir), - ...discoverEvalScenarioFiles('wrdn-dos-review', evalsDir), ]; const missingMetadata: string[] = []; diff --git a/packages/evals/src/security-matrix.eval.ts b/packages/evals/src/security-matrix.eval.ts deleted file mode 100644 index fa9b8d87c..000000000 --- a/packages/evals/src/security-matrix.eval.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { join } from 'node:path'; -import { afterAll, expect } from 'vitest'; -import { describeEval } from 'vitest-evals'; -import { resolveSkillAsync } from '@sentry/warden'; -import { - createWardenEvalHarness, - createWardenEvalJudge, - WardenEvalOutputSchema, -} from './harness.js'; -import { discoverEvalScenarios } from './index.js'; -import { formatEvalId, formatEvalTestName } from './names.js'; -import { resolveEvalProvider } from './provider.js'; -import { - matrixScenarioKind, - summarizeSecurityMatrix, - type MatrixLane, - type MatrixResult, -} from './security-matrix.js'; -import type { EvalMeta } from './types.js'; - -/** - * Pinned so a skill edit in warden-skills cannot silently move the matrix. - * Bump deliberately and rerun with `run-evals` to record new results. - */ -const WARDEN_SKILLS_REMOTE = 'getsentry/warden-skills@daf01f970a28450627a8d2d61d9915a742a735e2'; - -/** Remote security skills under test. Each skill's scenarios live in the category of the same name. */ -const MATRIX_SKILLS = ['wrdn-authz', 'wrdn-code-execution', 'wrdn-data-exfil', 'wrdn-dos-review'] as const; - -const SECURITY_REVIEW_SKILL = '../warden/src/builtin-skills/security-review/SKILL.md'; -const MATRIX_RUN_TIMEOUT_MS = 240_000; -const MATRIX_EVAL_TIMEOUT_MS = MATRIX_RUN_TIMEOUT_MS + 60_000; - -const provider = resolveEvalProvider(); -const remoteSkillPaths = new Map>(); -const results: MatrixResult[] = []; - -// Fetch lazily inside tests so suites without credentials never touch the network. -function resolveRemoteSkillPath(name: string): Promise { - let path = remoteSkillPaths.get(name); - if (!path) { - path = resolveSkillAsync(name, undefined, { remote: WARDEN_SKILLS_REMOTE }).then((skill) => { - if (!skill.rootDir) { - throw new Error(`Remote skill ${name} resolved without a rootDir`); - } - return join(skill.rootDir, 'SKILL.md'); - }); - remoteSkillPaths.set(name, path); - } - return path; -} - -function describeLane( - skill: string, - lane: MatrixLane, - evals: EvalMeta[], - resolveMeta: (meta: EvalMeta) => Promise, -): void { - const laneSkill = lane === 'skill' ? skill : 'security-review'; - describeEval( - `security-matrix/${skill}/${laneSkill}`, - { - harness: createWardenEvalHarness({ - apiKey: provider.apiKey, - model: provider.model, - runtime: provider.runtime, - auxiliaryModel: provider.auxiliaryModel, - timeoutMs: MATRIX_RUN_TIMEOUT_MS, - verbose: true, - }), - judges: [createWardenEvalJudge({ apiKey: provider.apiKey, model: provider.judgeModel })], - judgeThreshold: 1, - skipIf: () => !provider.enabled, - }, - (it) => { - for (const scenario of evals) { - it( - formatEvalTestName(scenario), - { timeout: MATRIX_EVAL_TIMEOUT_MS }, - async ({ run }) => { - const record = (passed: boolean) => results.push({ - skill, - scenario: scenario.name, - kind: matrixScenarioKind(scenario), - lane, - passed, - }); - const meta = await resolveMeta(scenario); - try { - // run() throws when the judge score misses the threshold. - const result = await run(meta); - const output = WardenEvalOutputSchema.parse(result.output); - expect(output.name).toBe(formatEvalId(meta)); - record(true); - console.log(`\n${output.summary}`); - console.log(` Findings: ${output.findings.length}`); - } catch (error) { - record(false); - throw error; - } - }, - ); - } - }, - ); -} - -// Each scenario runs under security-review and under the wrdn skill that owns the -// category. A bug only the wrdn lane catches is coverage security-review lacks. -for (const skill of MATRIX_SKILLS) { - const evals = discoverEvalScenarios({ - category: skill, - skill: SECURITY_REVIEW_SKILL, - runtime: 'pi', - model: 'anthropic/claude-sonnet-4-6', - }); - - describeLane(skill, 'security-review', evals, async (meta) => meta); - describeLane( - skill, - 'skill', - evals.map((meta) => ({ ...meta, skillName: skill })), - async (meta) => ({ ...meta, skillPath: await resolveRemoteSkillPath(skill) }), - ); -} - -afterAll(() => { - if (results.length > 0) { - console.log(`\n${summarizeSecurityMatrix(results)}`); - } -}); diff --git a/packages/evals/src/security-matrix.test.ts b/packages/evals/src/security-matrix.test.ts deleted file mode 100644 index 747e2f4ee..000000000 --- a/packages/evals/src/security-matrix.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { matrixScenarioKind, summarizeSecurityMatrix, type MatrixResult } from './security-matrix.js'; - -function result( - scenario: string, - lane: MatrixResult['lane'], - passed: boolean, - kind: MatrixResult['kind'] = 'bug', - skill = 'wrdn-authz', -): MatrixResult { - return { skill, scenario, kind, lane, passed }; -} - -describe('matrixScenarioKind', () => { - it('treats scenarios without a required should_find as clean', () => { - expect(matrixScenarioKind({ should_find: [{ finding: 'real bug', required: true }] })).toBe('bug'); - expect(matrixScenarioKind({ should_find: [{ finding: 'no bugs expected', required: false }] })).toBe('clean'); - }); -}); - -describe('summarizeSecurityMatrix', () => { - it('counts bugs only the wrdn skill caught and clean fixtures each lane flagged', () => { - const summary = summarizeSecurityMatrix([ - result('scim', 'skill', true), - result('scim', 'security-review', false), - result('projects', 'skill', true), - result('projects', 'security-review', true), - result('workflow', 'skill', false), - result('workflow', 'security-review', true), - result('fixed-scim', 'skill', false, 'clean'), - result('fixed-scim', 'security-review', true, 'clean'), - ]); - - expect(summary).toBe([ - 'Security matrix', - '', - 'wrdn-authz: 1 of 3 bugs caught only by wrdn-authz (scim)', - ' caught: wrdn-authz 2/3, security-review 2/3', - ' caught only by security-review: 1 (workflow)', - ' clean fixtures flagged: wrdn-authz 1/1, security-review 0/1', - ].join('\n')); - }); - - it('reports zero unique catches per skill and names scenarios missing a lane', () => { - const summary = summarizeSecurityMatrix([ - result('ssrf', 'skill', true, 'bug', 'wrdn-data-exfil'), - result('ssrf', 'security-review', true, 'bug', 'wrdn-data-exfil'), - result('logs', 'skill', false, 'bug', 'wrdn-data-exfil'), - ]); - - expect(summary).toContain('wrdn-data-exfil: 0 of 2 bugs caught only by wrdn-data-exfil'); - expect(summary).toContain(' incomplete, missing a lane: logs'); - }); -}); diff --git a/packages/evals/src/security-matrix.ts b/packages/evals/src/security-matrix.ts deleted file mode 100644 index a94ec6e10..000000000 --- a/packages/evals/src/security-matrix.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { EvalMeta } from './types.js'; - -/** Which skill ran a matrix scenario: builtin security-review or the wrdn skill that owns it. */ -export type MatrixLane = 'security-review' | 'skill'; - -/** A bug scenario expects a finding. A clean scenario only guards against false positives. */ -export type MatrixScenarioKind = 'bug' | 'clean'; - -/** One judged run of one scenario in one lane. */ -export interface MatrixResult { - skill: string; - scenario: string; - kind: MatrixScenarioKind; - lane: MatrixLane; - passed: boolean; -} - -/** Classify a scenario: clean when it has no required `should_find`. */ -export function matrixScenarioKind(meta: Pick): MatrixScenarioKind { - return meta.should_find.some((assertion) => assertion.required) ? 'bug' : 'clean'; -} - -function passedIn(results: MatrixResult[], scenario: string, lane: MatrixLane): boolean | undefined { - return results.find((result) => result.scenario === scenario && result.lane === lane)?.passed; -} - -function formatList(names: string[]): string { - return names.length > 0 ? ` (${names.join(', ')})` : ''; -} - -/** - * Render the per-skill coverage verdict. The headline number is bugs the wrdn - * skill caught that security-review missed; zero across a real fixture set is - * the evidence that the skill adds no coverage. - */ -export function summarizeSecurityMatrix(results: MatrixResult[]): string { - const skills = [...new Set(results.map((result) => result.skill))].sort(); - const lines: string[] = ['Security matrix']; - - for (const skill of skills) { - const skillResults = results.filter((result) => result.skill === skill); - const scenarios = [...new Set(skillResults.map((result) => result.scenario))].sort(); - const bugs = scenarios.filter((name) => skillResults.some((r) => r.scenario === name && r.kind === 'bug')); - const clean = scenarios.filter((name) => !bugs.includes(name)); - const incomplete = scenarios.filter( - (name) => passedIn(skillResults, name, 'skill') === undefined - || passedIn(skillResults, name, 'security-review') === undefined, - ); - - const caughtBy = (lane: MatrixLane) => bugs.filter((name) => passedIn(skillResults, name, lane) === true); - const flaggedBy = (lane: MatrixLane) => clean.filter((name) => passedIn(skillResults, name, lane) === false); - const skillCaught = caughtBy('skill'); - const reviewCaught = caughtBy('security-review'); - const onlySkill = skillCaught.filter((name) => !reviewCaught.includes(name)); - const onlyReview = reviewCaught.filter((name) => !skillCaught.includes(name)); - - lines.push( - '', - `${skill}: ${onlySkill.length} of ${bugs.length} bugs caught only by ${skill}${formatList(onlySkill)}`, - ` caught: ${skill} ${skillCaught.length}/${bugs.length}, security-review ${reviewCaught.length}/${bugs.length}`, - ` caught only by security-review: ${onlyReview.length}${formatList(onlyReview)}`, - ` clean fixtures flagged: ${skill} ${flaggedBy('skill').length}/${clean.length}, ` - + `security-review ${flaggedBy('security-review').length}/${clean.length}`, - ); - if (incomplete.length > 0) { - lines.push(` incomplete, missing a lane: ${incomplete.join(', ')}`); - } - } - - return lines.join('\n'); -} diff --git a/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json b/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json deleted file mode 100644 index b4bbd4280..000000000 --- a/packages/evals/wrdn-authz/sentry-scim-restriction-revert.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "SCIM permission class only checks that the org's auth provider has SCIM enabled, so any member or token with member/team scopes can call SCIM endpoints, not just the SCIM identity provider", - "files": [ - "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py" - ], - "should_find": [ - { - "finding": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams" - } - ], - "should_not_find": [ - "SCIM filter parsing", - "pagination parameter validation" - ], - "supporting_files": [ - "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/pull/130141", - "repository": "getsentry/sentry", - "source_ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2", - "source_files": [ - { - "fixturePath": "fixtures/sentry-scim-restriction-revert/github/getsentry/sentry/src/sentry/core/endpoints/scim/utils.py", - "sourcePath": "src/sentry/core/endpoints/scim/utils.py", - "ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2" - } - ] - } -} diff --git a/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json b/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json deleted file mode 100644 index 15b5127b2..000000000 --- a/packages/evals/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "given": "Seer workflow config PUT validates only the project IDs in the request and then replaces the whole project selection, so a limited member can unlink projects they cannot access or narrow an all-projects workflow", - "files": [ - "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py" - ], - "should_find": [ - { - "finding": "PUT replaces the workflow's project selection with the caller-supplied IDs after checking access only to those IDs, which deletes existing links to projects the caller cannot access and lets a member without global access convert an all_projects workflow to a narrower selection" - } - ], - "should_not_find": [ - "monitor selection type validation", - "config ID parsing" - ], - "supporting_files": [ - "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py", - "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/pull/129734", - "repository": "getsentry/sentry", - "source_ref": "d74d48356f2c822f69463c41922823908683e084", - "source_files": [ - { - "fixturePath": "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/endpoints/organization_seer_workflow_configs.py", - "sourcePath": "src/sentry/seer/endpoints/organization_seer_workflow_configs.py", - "ref": "d74d48356f2c822f69463c41922823908683e084" - }, - { - "fixturePath": "fixtures/sentry-seer-workflow-put-drops-inaccessible-projects/github/getsentry/sentry/src/sentry/seer/workflows/workflow.py", - "sourcePath": "src/sentry/seer/workflows/workflow.py", - "ref": "d74d48356f2c822f69463c41922823908683e084" - } - ] - } -} diff --git a/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json b/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json deleted file mode 100644 index 4f198c309..000000000 --- a/packages/evals/wrdn-authz/sentry-service-account-all-projects-team-write.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "given": "service account access maps all_projects to has_global_access, and team membership creation allows any caller with has_global_access, so an org:read service account can add members to any team", - "files": [ - "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py", - "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py" - ], - "should_find": [ - { - "finding": "A service account with all_projects gets has_global_access, and _can_create_team_member allows has_global_access callers, so an org:read service account can add an organization member to any team even though all_projects is meant to grant project access only" - } - ], - "should_not_find": [ - "team role escalation through team admin roles", - "member invite email handling" - ], - "supporting_files": [ - "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/pull/129343", - "repository": "getsentry/sentry", - "source_ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034", - "source_files": [ - { - "fixturePath": "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/auth/access.py", - "sourcePath": "src/sentry/auth/access.py", - "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034" - }, - { - "fixturePath": "fixtures/sentry-service-account-all-projects-team-write/github/getsentry/sentry/src/sentry/core/endpoints/organization_member_team_details.py", - "sourcePath": "src/sentry/core/endpoints/organization_member_team_details.py", - "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034" - } - ] - } -} diff --git a/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json b/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json deleted file mode 100644 index 2ae296348..000000000 --- a/packages/evals/wrdn-authz/sentry-service-account-org-projects-listing.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "given": "organization projects index sends non-human org-scoped auth through a branch that lists every project in the org, so a service account without all_projects sees projects it is not linked to", - "files": [ - "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py" - ], - "should_find": [ - { - "finding": "GET organization projects returns every project in the organization to a service account without global access, because the non-human auth branch does not filter by request.access.accessible_project_ids the way the agent-token branch does" - } - ], - "should_not_find": [ - "stats_period validation", - "project creation rate limits" - ], - "supporting_files": [ - "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py", - "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/pull/129154", - "repository": "getsentry/sentry", - "source_ref": "595efb30334e9427e572792031929c0298d24703", - "source_files": [ - { - "fixturePath": "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/core/endpoints/organization_projects.py", - "sourcePath": "src/sentry/core/endpoints/organization_projects.py", - "ref": "595efb30334e9427e572792031929c0298d24703" - }, - { - "fixturePath": "fixtures/sentry-service-account-org-projects-listing/github/getsentry/sentry/src/sentry/auth/access.py", - "sourcePath": "src/sentry/auth/access.py", - "ref": "595efb30334e9427e572792031929c0298d24703" - } - ] - } -} diff --git a/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json b/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json deleted file mode 100644 index 2ade58af5..000000000 --- a/packages/evals/wrdn-code-execution/sentry-analyze-styled-branch-shell-injection.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "styled-components history analysis script builds git shell commands with template strings that include the current branch name", - "files": [ - "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts" - ], - "should_find": [ - { - "finding": "originalBranch (the current git branch name) is interpolated into child_process.execSync shell command strings such as git show and git checkout, so a branch name containing shell metacharacters runs arbitrary commands" - } - ], - "should_not_find": [ - "slow synchronous git calls", - "analysis accuracy of styled-components detection" - ], - "supporting_files": [ - "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-061)", - "repository": "getsentry/sentry", - "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source_files": [ - { - "fixturePath": "fixtures/sentry-analyze-styled-branch-shell-injection/github/getsentry/sentry/scripts/analyze-styled.ts", - "sourcePath": "scripts/analyze-styled.ts", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - } - ] - } -} diff --git a/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json b/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json deleted file mode 100644 index 4acf4c6f4..000000000 --- a/packages/evals/wrdn-code-execution/sentry-getsentry-dispatch-github-output-injection.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "given": "PR workflow writes changed filenames from the GitHub API into $GITHUB_OUTPUT with a script that does not guard against newlines", - "files": [ - "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml", - "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py" - ], - "should_find": [ - { - "finding": "parse-pr-files.py prints files= into $GITHUB_OUTPUT; PR filenames are attacker-controlled and can contain newlines, so a fork PR can inject or override step outputs that the workflow forwards to the getsentry dispatch" - } - ], - "should_not_find": [ - "pinned action SHAs", - "token scope of GITHUB_TOKEN for the permissions step" - ], - "supporting_files": [ - "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-060)", - "repository": "getsentry/sentry", - "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source_files": [ - { - "fixturePath": "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/getsentry-dispatch.yml", - "sourcePath": ".github/workflows/getsentry-dispatch.yml", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - }, - { - "fixturePath": "fixtures/sentry-getsentry-dispatch-github-output-injection/github/getsentry/sentry/.github/workflows/scripts/parse-pr-files.py", - "sourcePath": ".github/workflows/scripts/parse-pr-files.py", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - } - ] - } -} diff --git a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json deleted file mode 100644 index 801cab3b1..000000000 --- a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "self-hosted feature flag endpoint writes request values into sentry.conf.py as Python source and reloads the config", - "files": [ - "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py" - ], - "should_find": [ - { - "finding": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server" - } - ], - "should_not_find": [ - "GET leaking the list of early features", - "missing CSRF protection" - ], - "supporting_files": [ - "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (pre-fix parent, getsentry/sentry#112980)", - "repository": "getsentry/sentry", - "source_ref": "101823f59fee6d71221feceb0cbcb28cc6024606", - "source_files": [ - { - "fixturePath": "fixtures/sentry-internal-feature-flags-config-injection/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py", - "sourcePath": "src/sentry/api/endpoints/internal/feature_flags.py", - "ref": "101823f59fee6d71221feceb0cbcb28cc6024606" - } - ] - } -} diff --git a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json b/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json deleted file mode 100644 index 2a01a9e5b..000000000 --- a/packages/evals/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "given": "feature flag endpoint after the fix: request values are normalized to booleans before being written to sentry.conf.py", - "files": [ - "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py" - ], - "should_find": [ - { - "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", - "required": false - } - ], - "should_not_find": [ - "code injection into sentry.conf.py through feature flag values", - "arbitrary Python executed by configure() from request data" - ], - "supporting_files": [ - "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (fixed state)", - "repository": "getsentry/sentry", - "source_ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866", - "source_files": [ - { - "fixturePath": "fixtures/sentry-internal-feature-flags-restricted-values/github/getsentry/sentry/src/sentry/api/endpoints/internal/feature_flags.py", - "sourcePath": "src/sentry/api/endpoints/internal/feature_flags.py", - "ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866" - } - ] - } -} diff --git a/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json b/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json deleted file mode 100644 index f9ad8368e..000000000 --- a/packages/evals/wrdn-data-exfil/sentry-data-forwarder-serializer-decrypted-config.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "given": "data forwarder API serializer returns the stored provider config, which holds decrypted Segment, AWS SQS, and Splunk credentials", - "files": [ - "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py" - ], - "should_find": [ - { - "finding": "DataForwarderSerializer returns obj.config (and project effectiveConfig) in API responses; config is an EncryptedJSONField decrypted on read, so stored provider secrets such as Segment write keys, AWS secret access keys, and Splunk tokens are exposed to API callers" - } - ], - "should_not_find": [ - "N+1 queries when loading project configs", - "missing pagination on enrolled projects" - ], - "supporting_files": [ - "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py", - "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py", - "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/warden/blob/main/packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json (sentry-vuln-052)", - "repository": "getsentry/sentry", - "source_ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source_files": [ - { - "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/serializers/models/data_forwarder.py", - "sourcePath": "src/sentry/integrations/api/serializers/models/data_forwarder.py", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - }, - { - "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/models/data_forwarder.py", - "sourcePath": "src/sentry/integrations/models/data_forwarder.py", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - }, - { - "fixturePath": "fixtures/sentry-data-forwarder-serializer-decrypted-config/github/getsentry/sentry/src/sentry/integrations/api/endpoints/data_forwarding_index.py", - "sourcePath": "src/sentry/integrations/api/endpoints/data_forwarding_index.py", - "ref": "7f41cc502faa1088108e896565409be72d92bc38" - } - ] - } -} diff --git a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json deleted file mode 100644 index c2189d3e4..000000000 --- a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-redacted.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "given": "system options endpoint after the fix: secret option values are redacted before logging", - "files": [ - "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py" - ], - "should_find": [ - { - "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", - "required": false - } - ], - "should_not_find": [ - "secret option values logged in cleartext", - "credentials written to logs by options.update" - ], - "supporting_files": [ - "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (fixed state)", - "repository": "getsentry/sentry", - "source_ref": "36ae5236a01ac174b5d3a432ac350fd469f05441", - "source_files": [ - { - "fixturePath": "fixtures/sentry-system-options-logs-redacted/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py", - "sourcePath": "src/sentry/api/endpoints/system_options.py", - "ref": "36ae5236a01ac174b5d3a432ac350fd469f05441" - } - ] - } -} diff --git a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json b/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json deleted file mode 100644 index e6797ccb1..000000000 --- a/packages/evals/wrdn-data-exfil/sentry-system-options-logs-secret-values.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "PUT /system/options/ logs every updated option key and value, including credential options, in cleartext", - "files": [ - "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py" - ], - "should_find": [ - { - "finding": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext" - } - ], - "should_not_find": [ - "superuser permission checks on the endpoint", - "option channel handling" - ], - "supporting_files": [ - "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (pre-fix parent, getsentry/sentry#114187)", - "repository": "getsentry/sentry", - "source_ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a", - "source_files": [ - { - "fixturePath": "fixtures/sentry-system-options-logs-secret-values/github/getsentry/sentry/src/sentry/api/endpoints/system_options.py", - "sourcePath": "src/sentry/api/endpoints/system_options.py", - "ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a" - } - ] - } -} diff --git a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json deleted file mode 100644 index 14faa0efc..000000000 --- a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "given": "uptime autodetect after the fix: robots.txt is fetched with safe_urlopen and parsed from the response body", - "files": [ - "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py" - ], - "should_find": [ - { - "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", - "required": false - } - ], - "should_not_find": [ - "SSRF in the robots.txt fetch", - "robots.txt fetched through raw urllib or RobotFileParser.read" - ], - "supporting_files": [ - "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (fixed state)", - "repository": "getsentry/sentry", - "source_ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b", - "source_files": [ - { - "fixturePath": "fixtures/sentry-uptime-robots-txt-safe-fetch/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py", - "sourcePath": "src/sentry/uptime/autodetect/tasks.py", - "ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b" - } - ] - } -} diff --git a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json b/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json deleted file mode 100644 index ad1463d52..000000000 --- a/packages/evals/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "uptime autodetect checks robots.txt for URLs pulled from customer event data with urllib.robotparser, which fetches through raw urllib instead of Sentry's SSRF-safe HTTP stack", - "files": [ - "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py" - ], - "should_find": [ - { - "finding": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)" - } - ], - "should_not_find": [ - "uptime monitor ranking logic", - "metrics tag cardinality" - ], - "supporting_files": [ - "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (pre-fix parent, getsentry/sentry#118358)", - "repository": "getsentry/sentry", - "source_ref": "86006042764d70705ca51df36e3f215effd3113d", - "source_files": [ - { - "fixturePath": "fixtures/sentry-uptime-robots-txt-ssrf/github/getsentry/sentry/src/sentry/uptime/autodetect/tasks.py", - "sourcePath": "src/sentry/uptime/autodetect/tasks.py", - "ref": "86006042764d70705ca51df36e3f215effd3113d" - } - ] - } -} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json deleted file mode 100644 index 551876863..000000000 --- a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "given": "EAP item serialization after the fix: recursive encoding stringifies values beyond a fixed depth", - "files": [ - "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py" - ], - "should_find": [ - { - "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", - "required": false - } - ], - "should_not_find": [ - "unbounded recursion in _encode_value", - "missing depth limit on nested event attributes" - ], - "supporting_files": [ - "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (fixed state)", - "repository": "getsentry/sentry", - "source_ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e", - "source_files": [ - { - "fixturePath": "fixtures/sentry-eap-item-encode-depth-limited/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py", - "sourcePath": "src/sentry/eventstream/item_helpers.py", - "ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e" - } - ] - } -} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json b/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json deleted file mode 100644 index 7b4574a47..000000000 --- a/packages/evals/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "occurrence-to-EAP item serialization recursively encodes nested event arrays and dicts with no depth limit", - "files": [ - "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py" - ], - "should_find": [ - { - "finding": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event" - } - ], - "should_not_find": [ - "float precision when encoding numbers", - "attribute key naming" - ], - "supporting_files": [ - "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (pre-fix parent, getsentry/sentry#108555)", - "repository": "getsentry/sentry", - "source_ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5", - "source_files": [ - { - "fixturePath": "fixtures/sentry-eap-item-encode-unbounded-recursion/github/getsentry/sentry/src/sentry/eventstream/item_helpers.py", - "sourcePath": "src/sentry/eventstream/item_helpers.py", - "ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5" - } - ] - } -} diff --git a/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json b/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json deleted file mode 100644 index 2476fd673..000000000 --- a/packages/evals/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "EAP search resolver recursively resolves user-supplied boolean and parenthesized query terms with no depth limit", - "files": [ - "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py" - ], - "should_find": [ - { - "finding": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request" - } - ], - "should_not_find": [ - "attribute type resolution", - "aggregate filter ordering" - ], - "supporting_files": [ - "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080 (pre-fix parent, getsentry/sentry#128180)", - "repository": "getsentry/sentry", - "source_ref": "975da7ea97f7c9f964d6238229338fc02896661e", - "source_files": [ - { - "fixturePath": "fixtures/sentry-eap-search-boolean-nesting-recursion/github/getsentry/sentry/src/sentry/search/eap/resolver.py", - "sourcePath": "src/sentry/search/eap/resolver.py", - "ref": "975da7ea97f7c9f964d6238229338fc02896661e" - } - ] - } -} diff --git a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json deleted file mode 100644 index 5edd3879b..000000000 --- a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "given": "message parameterization URL regex has an unbounded scheme quantifier that backtracks on long runs of scheme-valid characters", - "files": [ - "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py" - ], - "should_find": [ - { - "finding": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)" - } - ], - "should_not_find": [ - "IPv6 parameterization accuracy", - "hostname validation correctness" - ], - "supporting_files": [ - "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (pre-fix parent, getsentry/sentry#118512)", - "repository": "getsentry/sentry", - "source_ref": "f7c64397f494fe2da5c7775baa3943085998fa5a", - "source_files": [ - { - "fixturePath": "fixtures/sentry-grouping-url-regex-backtracking/github/getsentry/sentry/src/sentry/grouping/parameterization.py", - "sourcePath": "src/sentry/grouping/parameterization.py", - "ref": "f7c64397f494fe2da5c7775baa3943085998fa5a" - } - ] - } -} diff --git a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json b/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json deleted file mode 100644 index 842cf3578..000000000 --- a/packages/evals/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "given": "message parameterization after the fix: the URL scheme quantifier is bounded to 32 characters", - "files": [ - "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py" - ], - "should_find": [ - { - "finding": "no vulnerability in the reviewed change: the code shown already contains the fix, so zero findings or only unrelated low-severity notes are expected", - "required": false - } - ], - "should_not_find": [ - "catastrophic regex backtracking in the URL scheme pattern", - "unbounded scheme quantifier ReDoS" - ], - "supporting_files": [ - "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/LICENSE.md" - ], - "notes": { - "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (fixed state)", - "repository": "getsentry/sentry", - "source_ref": "a111b7473cc002875a38a7ffd3792ee1d395f362", - "source_files": [ - { - "fixturePath": "fixtures/sentry-grouping-url-regex-bounded-scheme/github/getsentry/sentry/src/sentry/grouping/parameterization.py", - "sourcePath": "src/sentry/grouping/parameterization.py", - "ref": "a111b7473cc002875a38a7ffd3792ee1d395f362" - } - ] - } -} diff --git a/packages/security-evals/.gitignore b/packages/security-evals/.gitignore new file mode 100644 index 000000000..ceddaa37f --- /dev/null +++ b/packages/security-evals/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md new file mode 100644 index 000000000..43024063c --- /dev/null +++ b/packages/security-evals/README.md @@ -0,0 +1,107 @@ +# Security Evals + +Optional suite that answers one question: does each security skill catch bugs +the others miss? It runs builtin `security-review` and the `wrdn-*` security +skills over the same real code and scores every lane against the same list of +known bugs. + +It never runs from `pnpm evals`, the `run-evals` label, or the PR Evals +workflow. It costs real money. Run it on purpose. + +## What it scans + +- **Corpus entries**: one per finding in + `packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json` + (86 entries, 6 SHAs, 79 files). +- **Cases**: `cases//*.json`, for bug classes the corpus barely + covers (authz production catches, DoS, data exfiltration, code execution). + `kind: "bug"` is a known bug at `ref`. `kind: "clean"` is the same code after + the fix; a hit there is a false positive. + +Every entry pins `repository`, `ref`, and `targets`. Skills scan the targets +inside a full worktree of that ref, so Read and Grep can trace callers, base +classes, and permission checks the way they do in production. + +## Running it + +You need a local getsentry/sentry clone. Blobless is enough: + +```bash +git clone --filter=blob:none https://github.com/getsentry/sentry.git ~/src/sentry +``` + +```bash +# Every lane over every entry +pnpm evals:security --sentry-repo ~/src/sentry + +# One candidate lane on its own cases, three passes; other lanes come from the baseline +pnpm evals:security --sentry-repo ~/src/sentry \ + --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3 +``` + +`pnpm evals:security --help` lists every flag. Each one maps to a +`WARDEN_SECURITY_EVALS_*` env var. Defaults match the production lane: pi +runtime, `openrouter/x-ai/grok-4.5` at high effort, and +`openrouter/openai/gpt-5.6-luna` at high effort for auxiliary work. `maxTurns` +defaults to 100, as in the corpus runbook, so a turn cap does not read as a +miss. + +Setup is the only network step: remote skills load from the pinned +warden-skills ref, and each SHA gets `git fetch origin ` if the clone +lacks it, then `git worktree add --detach` under `.cache/`. Worktrees are +reused across lanes, entries, and repeats. Without `--sentry-repo` the suite +fails before doing anything. + +In CI, run the **Security Evals** workflow by hand. It restores a cached +blobless clone, runs the suite, and uploads `results.json`. + +## How it scores + +Entries sharing a ref form one shard. Each pass builds one file-mode context +over the shard's targets and runs every selected skill on it. Per lane and +entry, only findings in the entry's target files reach the judge, which +applies the corpus rule: same bug, roughly the same location. Another bug in +the same file does not count. No findings in the targets is a miss with no +judge call. A failed chunk drops the entry from that repeat instead of +counting a miss. + +Model calls go through Roach like `packages/evals`. Worktree content is pinned, +so recordings replay; `roach.json` normalizes the worktree, checkout, and +skill cache paths. + +## Reading results + +`.cache/results.json` has sorted keys and no timestamps, so two runs diff +cleanly. The console and the job summary show: + +- per lane: recall, bugs found, bugs only that lane found, bugs it found that + security-review missed, clean cases flagged, cost, and tokens +- an entry x lane table of hits over repeats + +`missedBySecurityReview` is the number that decides whether a skill earns its +keep. Zero across a real fixture set is the case for dropping it. + +Single runs are noisy: two production-config security-review corpus runs agreed on 32 +of the 46 entries either one found. Use `--repeats 3` or more before deciding. + +## Baseline + +`baseline/results.json` holds lanes this run does not execute, so a +comparison only pays for the candidate lane. It currently has security-review +over the corpus, imported from the two published production-config benchmark +runs (warden 0.46.2 and 0.47.0, scored by the benchmark reviewer). It has no +case results. Pass `--baseline none` to skip it. + +Regenerate it from scored benchmark results: + +```bash +pnpm -C packages/security-evals import-benchmark \ + ../docs/src/data/benchmarking/results/.json ... +``` + +## Known gaps + +- `wrdn-code-execution` has one real bug case, plus corpus entries 060 and + 061. `wrdn-data-exfil` has two, plus corpus 052. The corpus has almost no + DoS or exfiltration entries. +- Cases are getsentry/sentry only. Other repositories need their own clone flag. diff --git a/packages/security-evals/baseline/results.json b/packages/security-evals/baseline/results.json new file mode 100644 index 000000000..86202631d --- /dev/null +++ b/packages/security-evals/baseline/results.json @@ -0,0 +1,1292 @@ +{ + "entries": { + "sentry-vuln-001": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "788ba30f1aa42b00c02d64ed4b8b2515ff8ab8da", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/accept_organization_invite.py" + ] + }, + "sentry-vuln-002": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/workflow_engine/endpoints/organization_alertrule_detector_index.py" + ] + }, + "sentry-vuln-003": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/internal_ea_features.py" + ] + }, + "sentry-vuln-004": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/oauth_userinfo.py" + ] + }, + "sentry-vuln-005": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/oauth_userinfo.py" + ] + }, + "sentry-vuln-006": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/organization_plugin_deprecation_info.py" + ] + }, + "sentry-vuln-007": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/release_thresholds/release_threshold_index.py" + ] + }, + "sentry-vuln-008": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ] + }, + "sentry-vuln-009": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "fbecd17d210c2eacfad4bf17872be62320b8a110", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/organization_events_root_cause_analysis.py" + ] + }, + "sentry-vuln-010": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "fbecd17d210c2eacfad4bf17872be62320b8a110", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ] + }, + "sentry-vuln-011": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/jira/webhooks/installed.py" + ] + }, + "sentry-vuln-012": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/msteams/webhook.py" + ] + }, + "sentry-vuln-013": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/perforce/integration.py" + ] + }, + "sentry-vuln-014": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/slack/webhooks/options_load.py" + ] + }, + "sentry-vuln-015": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/replays/endpoints/data_export_notifications.py" + ] + }, + "sentry-vuln-016": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/replays/endpoints/project_replay_details.py" + ] + }, + "sentry-vuln-017": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/preprod/api/endpoints/preprod_artifact_snapshot.py" + ] + }, + "sentry-vuln-018": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/organization_pinned_searches.py" + ] + }, + "sentry-vuln-019": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/auth/services/access/impl.py" + ] + }, + "sentry-vuln-020": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/users/models/user_merge_verification_code.py" + ] + }, + "sentry-vuln-021": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/codecov/endpoints/repository_token_regenerate/repository_token_regenerate.py" + ] + }, + "sentry-vuln-022": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/explore/endpoints/explore_saved_query_detail.py" + ] + }, + "sentry-vuln-023": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/flags/endpoints/secrets.py" + ] + }, + "sentry-vuln-024": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/project_rules.py" + ] + }, + "sentry-vuln-025": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/api/endpoints/external_team_details.py" + ] + }, + "sentry-vuln-026": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/messaging/linkage.py" + ] + }, + "sentry-vuln-027": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/slack/webhooks/action.py" + ] + }, + "sentry-vuln-028": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/accept_organization_invite.py" + ] + }, + "sentry-vuln-029": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/notifications/notification_action/group_type_notification_registry/handlers/metric_alert_registry_handler.py" + ] + }, + "sentry-vuln-030": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/preprod/api/endpoints/preprod_artifact_approve.py" + ] + }, + "sentry-vuln-031": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/preprod/api/endpoints/public/organization_preprod_size_analysis.py" + ] + }, + "sentry-vuln-032": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/projects/services/project_key/impl.py" + ] + }, + "sentry-vuln-033": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/replays/usecases/replay_counts.py" + ] + }, + "sentry-vuln-034": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/seer/autofix/autofix_agent.py" + ] + }, + "sentry-vuln-035": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/seer/endpoints/organization_autofix_automation_settings.py" + ] + }, + "sentry-vuln-036": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/seer/supergroups/endpoints/organization_supergroups_by_group.py" + ] + }, + "sentry-vuln-037": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/sentry_apps/services/cell/impl.py" + ] + }, + "sentry-vuln-038": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/sentry_apps/api/endpoints/installation_external_issue_actions.py" + ] + }, + "sentry-vuln-039": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/workflow_engine/endpoints/organization_open_periods.py" + ] + }, + "sentry-vuln-040": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/workflow_engine/endpoints/validators/utils.py" + ] + }, + "sentry-vuln-041": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/discover/endpoints/discover_saved_query_detail.py" + ] + }, + "sentry-vuln-042": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/integrations/slack/webhooks/action.py" + ] + }, + "sentry-vuln-043": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", + "source": "corpus", + "targets": [ + "src/sentry/notifications/platform/api/endpoints/internal_registered_templates.py" + ] + }, + "sentry-vuln-044": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "6993246259d0193a6a6aac49aa3763959b1126df", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/auth_index.py" + ] + }, + "sentry-vuln-045": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/seer/endpoints/organization_seer_rpc.py" + ] + }, + "sentry-vuln-046": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/preprod/api/endpoints/pull_request/organization_pullrequest_size_analysis_download.py" + ] + }, + "sentry-vuln-047": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/project_rule_details.py" + ] + }, + "sentry-vuln-048": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/preprod/api/endpoints/size_analysis/project_preprod_size_analysis_compare_download.py" + ] + }, + "sentry-vuln-049": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/api/endpoints/organization_pipeline.py" + ] + }, + "sentry-vuln-050": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/api/serializers/models/projectcodeowners.py" + ] + }, + "sentry-vuln-051": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/bitbucket_server/urls.py" + ] + }, + "sentry-vuln-052": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/models/data_forwarder.py" + ] + }, + "sentry-vuln-053": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/sentry_apps/api/endpoints/installation_external_issue_actions.py" + ] + }, + "sentry-vuln-054": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/relocation/api/endpoints/retry.py" + ] + }, + "sentry-vuln-055": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/seer/endpoints/organization_seer_rpc.py" + ] + }, + "sentry-vuln-056": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/api/serializers/models/apiapplication.py" + ] + }, + "sentry-vuln-057": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/tasks/post_process.py" + ] + }, + "sentry-vuln-058": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/workflow_engine/endpoints/organization_detector_anomaly_data.py" + ] + }, + "sentry-vuln-059": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/workflow_engine/endpoints/organization_alertrule_detector_index.py" + ] + }, + "sentry-vuln-060": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + ".github/workflows/getsentry-dispatch.yml" + ] + }, + "sentry-vuln-061": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "scripts/analyze-styled.ts" + ] + }, + "sentry-vuln-062": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/api/authentication.py" + ] + }, + "sentry-vuln-063": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/flags/providers.py" + ] + }, + "sentry-vuln-064": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/bitbucket/webhook.py" + ] + }, + "sentry-vuln-065": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/discord/requests/base.py" + ] + }, + "sentry-vuln-066": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/discord/requests/base.py" + ] + }, + "sentry-vuln-067": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/github_enterprise/webhook.py" + ] + }, + "sentry-vuln-068": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/github_enterprise/webhook.py" + ] + }, + "sentry-vuln-069": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/utils/atlassian_connect.py" + ] + }, + "sentry-vuln-070": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/jira_server/webhooks.py" + ] + }, + "sentry-vuln-071": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/integrations/utils/atlassian_connect.py" + ] + }, + "sentry-vuln-072": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/models/apiapplication.py" + ] + }, + "sentry-vuln-073": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/releases/endpoints/project_releases_token.py" + ] + }, + "sentry-vuln-074": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/seer/endpoints/seer_rpc.py" + ] + }, + "sentry-vuln-075": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/web/frontend/mailgun_inbound_webhook.py" + ] + }, + "sentry-vuln-076": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry/web/frontend/release_webhook.py" + ] + }, + "sentry-vuln-077": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry_plugins/heroku/plugin.py" + ] + }, + "sentry-vuln-078": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry_plugins/jira/client.py" + ] + }, + "sentry-vuln-079": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "src/sentry_plugins/jira/client.py" + ] + }, + "sentry-vuln-080": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/components/events/autofix/insights/autofixInsightSources.tsx" + ] + }, + "sentry-vuln-081": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/components/events/interfaces/utils.tsx" + ] + }, + "sentry-vuln-082": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 2, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/components/featureFlags/hooks/useFlagSeries.tsx" + ] + }, + "sentry-vuln-083": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/views/discover/utils.tsx" + ] + }, + "sentry-vuln-084": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/views/explore/releases/list/mobileBuildsChart.tsx" + ] + }, + "sentry-vuln-085": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 1, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/views/preprod/components/visualizations/appSizeTreemap.tsx" + ] + }, + "sentry-vuln-086": { + "kind": "bug", + "lanes": { + "security-review": { + "hits": 0, + "runs": 2 + } + }, + "ref": "7f41cc502faa1088108e896565409be72d92bc38", + "source": "corpus", + "targets": [ + "static/app/views/settings/organizationIntegrations/integrationDetailedView.tsx" + ] + } + }, + "lanes": { + "security-review": { + "config": { + "auxiliaryEffort": "high", + "auxiliaryModel": "openrouter/openai/gpt-5.6-luna", + "effort": "high", + "judgeModel": "agent-semantic-match-pass", + "maxTurns": 100, + "model": "openrouter/x-ai/grok-4.5", + "origin": "sentry-security-review-pi-openrouter-grok-4-5-high-luna-aux-high-traced-corpus-2026-08-21-001, sentry-security-review-pi-openrouter-grok-4-5-high-luna-aux-high-traced-corpus-2026-08-23-001", + "runtime": "pi" + }, + "failedRuns": 0, + "runs": 2, + "usage": { + "cacheReadInputTokens": 112996572, + "costUSD": 64.13501639, + "inputTokens": 137321818, + "judgeCostUSD": 0, + "outputTokens": 4385065 + } + } + }, + "summary": { + "security-review": { + "bugEntries": 86, + "cleanFlagged": { + "hits": 0, + "runs": 0 + }, + "found": 46, + "missedBySecurityReview": [], + "onlyThisLane": [ + "sentry-vuln-001", + "sentry-vuln-002", + "sentry-vuln-003", + "sentry-vuln-004", + "sentry-vuln-007", + "sentry-vuln-011", + "sentry-vuln-012", + "sentry-vuln-013", + "sentry-vuln-014", + "sentry-vuln-015", + "sentry-vuln-017", + "sentry-vuln-018", + "sentry-vuln-019", + "sentry-vuln-021", + "sentry-vuln-025", + "sentry-vuln-026", + "sentry-vuln-029", + "sentry-vuln-030", + "sentry-vuln-031", + "sentry-vuln-034", + "sentry-vuln-035", + "sentry-vuln-036", + "sentry-vuln-037", + "sentry-vuln-039", + "sentry-vuln-041", + "sentry-vuln-046", + "sentry-vuln-047", + "sentry-vuln-048", + "sentry-vuln-051", + "sentry-vuln-055", + "sentry-vuln-056", + "sentry-vuln-057", + "sentry-vuln-058", + "sentry-vuln-062", + "sentry-vuln-065", + "sentry-vuln-067", + "sentry-vuln-069", + "sentry-vuln-070", + "sentry-vuln-071", + "sentry-vuln-075", + "sentry-vuln-077", + "sentry-vuln-079", + "sentry-vuln-081", + "sentry-vuln-082", + "sentry-vuln-084", + "sentry-vuln-085" + ], + "recall": 0.45348837209302323 + } + }, + "version": 1 +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json new file mode 100644 index 000000000..67cde1977 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams", + "repository": "getsentry/sentry", + "ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2", + "targets": [ + "src/sentry/core/endpoints/scim/utils.py" + ], + "source": "https://github.com/getsentry/sentry/pull/130141" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json new file mode 100644 index 000000000..477e7d461 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "PUT replaces the workflow's project selection with the caller-supplied IDs after checking access only to those IDs, which deletes existing links to projects the caller cannot access and lets a member without global access convert an all_projects workflow to a narrower selection", + "repository": "getsentry/sentry", + "ref": "d74d48356f2c822f69463c41922823908683e084", + "targets": [ + "src/sentry/seer/endpoints/organization_seer_workflow_configs.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129734" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json new file mode 100644 index 000000000..7f99548dc --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "A service account with all_projects gets has_global_access, and _can_create_team_member allows has_global_access callers, so an org:read service account can add an organization member to any team even though all_projects is meant to grant project access only", + "repository": "getsentry/sentry", + "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034", + "targets": [ + "src/sentry/auth/access.py", + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129343" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json new file mode 100644 index 000000000..403a37169 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "GET organization projects returns every project in the organization to a service account without global access, because the non-human auth branch does not filter by request.access.accessible_project_ids the way the agent-token branch does", + "repository": "getsentry/sentry", + "ref": "595efb30334e9427e572792031929c0298d24703", + "targets": [ + "src/sentry/core/endpoints/organization_projects.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129154" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json new file mode 100644 index 000000000..7856e8855 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server", + "repository": "getsentry/sentry", + "ref": "101823f59fee6d71221feceb0cbcb28cc6024606", + "targets": [ + "src/sentry/api/endpoints/internal/feature_flags.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (pre-fix parent, getsentry/sentry#112980)" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json new file mode 100644 index 000000000..d1bbfdf9a --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server", + "repository": "getsentry/sentry", + "ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866", + "targets": [ + "src/sentry/api/endpoints/internal/feature_flags.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (fixed state)" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json new file mode 100644 index 000000000..849e9a964 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext", + "repository": "getsentry/sentry", + "ref": "36ae5236a01ac174b5d3a432ac350fd469f05441", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ], + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (fixed state)" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json new file mode 100644 index 000000000..779ffd320 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext", + "repository": "getsentry/sentry", + "ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ], + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (pre-fix parent, getsentry/sentry#114187)" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json new file mode 100644 index 000000000..cd82ec818 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)", + "repository": "getsentry/sentry", + "ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b", + "targets": [ + "src/sentry/uptime/autodetect/tasks.py" + ], + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (fixed state)" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json new file mode 100644 index 000000000..0d5b356f3 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)", + "repository": "getsentry/sentry", + "ref": "86006042764d70705ca51df36e3f215effd3113d", + "targets": [ + "src/sentry/uptime/autodetect/tasks.py" + ], + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (pre-fix parent, getsentry/sentry#118358)" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json new file mode 100644 index 000000000..43a000e6c --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event", + "repository": "getsentry/sentry", + "ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e", + "targets": [ + "src/sentry/eventstream/item_helpers.py" + ], + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (fixed state)" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json new file mode 100644 index 000000000..4ed0e4c6e --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event", + "repository": "getsentry/sentry", + "ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5", + "targets": [ + "src/sentry/eventstream/item_helpers.py" + ], + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (pre-fix parent, getsentry/sentry#108555)" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json new file mode 100644 index 000000000..dfc7daeaa --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request", + "repository": "getsentry/sentry", + "ref": "975da7ea97f7c9f964d6238229338fc02896661e", + "targets": [ + "src/sentry/search/eap/resolver.py" + ], + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080 (pre-fix parent, getsentry/sentry#128180)" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json new file mode 100644 index 000000000..fe24b56d3 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)", + "repository": "getsentry/sentry", + "ref": "f7c64397f494fe2da5c7775baa3943085998fa5a", + "targets": [ + "src/sentry/grouping/parameterization.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (pre-fix parent, getsentry/sentry#118512)" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json new file mode 100644 index 000000000..33221af46 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)", + "repository": "getsentry/sentry", + "ref": "a111b7473cc002875a38a7ffd3792ee1d395f362", + "targets": [ + "src/sentry/grouping/parameterization.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (fixed state)" +} diff --git a/packages/security-evals/package.json b/packages/security-evals/package.json new file mode 100644 index 000000000..1d70686b2 --- /dev/null +++ b/packages/security-evals/package.json @@ -0,0 +1,23 @@ +{ + "name": "warden-security-evals", + "private": true, + "type": "module", + "scripts": { + "evals:security": "tsx scripts/run.ts", + "import-benchmark": "tsx scripts/import-benchmark.ts", + "test": "vitest run --config vitest.unit.config.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@sentry/warden": "workspace:*", + "dotenv": "^17.4.2", + "zod": "^4.6.7" + }, + "devDependencies": { + "@sentry/roach": "^0.2.0", + "@types/node": "^25.9.1", + "tsx": "^4.23.12", + "typescript": "^6.0.3", + "vitest": "^4.1.11" + } +} diff --git a/packages/security-evals/roach.json b/packages/security-evals/roach.json new file mode 100644 index 000000000..ebcec0d91 --- /dev/null +++ b/packages/security-evals/roach.json @@ -0,0 +1,84 @@ +{ + "tenant": "getsentry/warden", + "allow": [ + "https://api.anthropic.com", + "https://openrouter.ai", + "https://api.openai.com", + "https://pi.dev" + ], + "rules": [ + { + "name": "model", + "match": { + "method": "POST", + "url": "https://api.anthropic.com/v1/" + }, + "values": { + "worktree-root": "(?:/[A-Za-z0-9_.-]+)+/security-evals-worktrees", + "git-commit": { + "builtin": "gitCommit" + }, + "uuid": { + "builtin": "uuid" + }, + "time": { + "builtin": "isoTime" + }, + "finding-id": "(?... +// Writes baseline/results.json from scored corpus benchmark results. +const cwd = process.env['INIT_CWD'] ?? process.cwd(); +const files = process.argv.slice(2).map((path) => JSON.parse(readFileSync(resolve(cwd, path), 'utf8')) as unknown); +if (files.length === 0) { + console.error('usage: import-benchmark ...'); + process.exit(2); +} +const results = importBenchmarkResults(loadCorpusEntries(), files); +writeFileSync(DEFAULT_BASELINE, serializeResults(results)); +console.log(`Wrote ${DEFAULT_BASELINE}`); diff --git a/packages/security-evals/scripts/prepare-sentry-clone.sh b/packages/security-evals/scripts/prepare-sentry-clone.sh new file mode 100755 index 000000000..0412a6c02 --- /dev/null +++ b/packages/security-evals/scripts/prepare-sentry-clone.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Ensure $SENTRY_CLONE holds a blobless getsentry/sentry clone. A restored +# cache is used as is: the suite fetches any pinned SHA the clone lacks. +set -eu + +: "${SENTRY_CLONE:?SENTRY_CLONE must name the clone directory}" + +if [ -d "$SENTRY_CLONE/.git" ]; then + printf 'Using cached clone at %s\n' "$SENTRY_CLONE" + exit 0 +fi + +timeout 1800 git clone --quiet --filter=blob:none --no-checkout \ + https://github.com/getsentry/sentry.git "$SENTRY_CLONE" diff --git a/packages/security-evals/scripts/run.ts b/packages/security-evals/scripts/run.ts new file mode 100644 index 000000000..eadc53dd4 --- /dev/null +++ b/packages/security-evals/scripts/run.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env tsx +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +// Each flag maps to WARDEN_SECURITY_EVALS_; see src/options.ts for defaults. +const FLAGS = { + 'sentry-repo': 'SENTRY_REPO', + skills: 'SKILLS', + entries: 'ENTRIES', + repeats: 'REPEATS', + runtime: 'RUNTIME', + model: 'MODEL', + effort: 'EFFORT', + 'aux-model': 'AUX_MODEL', + 'aux-effort': 'AUX_EFFORT', + 'judge-model': 'JUDGE_MODEL', + 'max-turns': 'MAX_TURNS', + concurrency: 'CONCURRENCY', + 'cache-dir': 'CACHE_DIR', + output: 'OUTPUT', + baseline: 'BASELINE', +} as const; + +const PATH_FLAGS = new Set(['sentry-repo', 'cache-dir', 'output', 'baseline']); + +const USAGE = `usage: pnpm evals:security --sentry-repo [options] + + --sentry-repo Local getsentry/sentry clone (required) + --skills Lanes to run (default: security-review and the five wrdn-* skills) + --entries Entry ids, or prefixes ending in * (default: all) + --repeats Passes per shard (default: 1) + --runtime, --model, --effort, --aux-model, --aux-effort + Model lane (default: pi, openrouter/x-ai/grok-4.5 high, + openrouter/openai/gpt-5.6-luna high) + --judge-model Scoring model (default: openrouter/anthropic/claude-sonnet-4.6) + --max-turns Agent turn cap (default: 100) + --concurrency Concurrent chunks per skill (default: 4) + --cache-dir Worktrees and results (default: packages/security-evals/.cache) + --output Results JSON (default: /results.json) + --baseline Lanes to compare against (default: baseline/results.json) + --remove-worktrees Delete each worktree after its shard (CI disk limits)`; + +const { values } = parseArgs({ + options: { + ...Object.fromEntries(Object.keys(FLAGS).map((flag) => [flag, { type: 'string' as const }])), + 'remove-worktrees': { type: 'boolean' }, + help: { type: 'boolean', short: 'h' }, + }, +}); + +if (values.help) { + console.log(USAGE); + process.exit(0); +} + +const env: NodeJS.ProcessEnv = { ...process.env }; +for (const [flag, name] of Object.entries(FLAGS)) { + const value = (values as Record)[flag]; + if (typeof value === 'string') { + // pnpm runs scripts from the package directory; INIT_CWD is where the caller ran it. + env[`WARDEN_SECURITY_EVALS_${name}`] = PATH_FLAGS.has(flag) && value !== 'none' + ? resolve(process.env['INIT_CWD'] ?? process.cwd(), value) + : value; + } +} +if (values['remove-worktrees']) { + env['WARDEN_SECURITY_EVALS_REMOVE_WORKTREES'] = '1'; +} + +const result = spawnSync('pnpm', ['exec', 'vitest', 'run', '--config', 'vitest.config.ts'], { + cwd: resolve(import.meta.dirname, '..'), + env, + stdio: 'inherit', +}); +process.exit(result.status ?? 1); diff --git a/packages/security-evals/src/baseline.test.ts b/packages/security-evals/src/baseline.test.ts new file mode 100644 index 000000000..d4ba956ff --- /dev/null +++ b/packages/security-evals/src/baseline.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { importBenchmarkResults } from './baseline.js'; +import type { SuiteEntry } from './entries.js'; + +const runOptions = { + runtime: 'pi', + model: 'openrouter/x-ai/grok-4.5', + effort: 'high', + auxiliaryModel: 'openrouter/openai/gpt-5.6-luna', + auxiliaryEffort: 'high', + maxTurns: 100, +}; + +function benchmark(runId: string, found: string[], overrides: Partial = {}) { + return { + runId, + corpusId: 'sentry-vulnerability-corpus', + skill: 'security-review', + runOptions: { ...runOptions, ...overrides }, + summary: { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 50, costUSD: 30 }, + scoring: { reviewer: 'agent-semantic-match-pass' }, + scores: [ + ...found.map((id) => ({ matchedCorpusIds: [id], verdict: 'known-found' })), + { matchedCorpusIds: [], verdict: 'not-known' }, + ], + }; +} + +function entry(id: string, source: SuiteEntry['source'] = 'corpus'): SuiteEntry { + return { id, source, kind: 'bug', repository: 'getsentry/sentry', ref: 'a', targets: ['x.py'], summary: id }; +} + +describe('importBenchmarkResults', () => { + it('turns each scored benchmark run into one security-review repeat', () => { + const results = importBenchmarkResults( + [entry('sentry-vuln-001'), entry('sentry-vuln-002'), entry('wrdn-authz/case', 'case')], + [benchmark('run-1', ['sentry-vuln-001']), benchmark('run-2', ['sentry-vuln-001', 'sentry-vuln-002'])], + ); + + expect(Object.keys(results.entries)).toEqual(['sentry-vuln-001', 'sentry-vuln-002']); + expect(results.entries['sentry-vuln-001']!.lanes['security-review']).toEqual({ hits: 2, runs: 2 }); + expect(results.entries['sentry-vuln-002']!.lanes['security-review']).toEqual({ hits: 1, runs: 2 }); + expect(results.lanes['security-review']).toMatchObject({ + runs: 2, + config: { model: 'openrouter/x-ai/grok-4.5', origin: 'run-1, run-2' }, + usage: { costUSD: 60 }, + }); + }); + + it('refuses runs from different model lanes', () => { + expect(() => importBenchmarkResults([entry('sentry-vuln-001')], [ + benchmark('run-1', []), + benchmark('run-2', [], { effort: 'low' }), + ])).toThrow('share one runtime'); + }); +}); diff --git a/packages/security-evals/src/baseline.ts b/packages/security-evals/src/baseline.ts new file mode 100644 index 000000000..46bdd6564 --- /dev/null +++ b/packages/security-evals/src/baseline.ts @@ -0,0 +1,72 @@ +import { z } from 'zod'; +import type { SuiteEntry } from './entries.js'; +import { createResults, recordShardRun, summarize, type LaneConfig, type SuiteResults } from './results.js'; + +const BenchmarkResultSchema = z.object({ + runId: z.string(), + corpusId: z.literal('sentry-vulnerability-corpus'), + skill: z.literal('security-review'), + runOptions: z.object({ + runtime: z.string(), + model: z.string(), + effort: z.string(), + auxiliaryModel: z.string(), + auxiliaryEffort: z.string(), + maxTurns: z.number(), + }), + summary: z.object({ + inputTokens: z.number(), + outputTokens: z.number(), + cacheReadInputTokens: z.number(), + costUSD: z.number(), + }), + scoring: z.object({ reviewer: z.string() }), + scores: z.array(z.object({ matchedCorpusIds: z.array(z.string()), verdict: z.string() })), +}); + +/** + * Turn scored corpus benchmark results into a security-review baseline lane, + * one repeat per result file. Every file must share one model lane. + */ +export function importBenchmarkResults(entries: SuiteEntry[], files: unknown[]): SuiteResults { + const runs = files.map((file) => BenchmarkResultSchema.parse(file)); + const [first] = runs; + const lanes = new Set(runs.map((run) => JSON.stringify(run.runOptions))); + if (!first || lanes.size !== 1) { + throw new Error('Benchmark results must share one runtime, model, effort, and auxiliary lane'); + } + + const corpus = entries.filter((entry) => entry.source === 'corpus'); + const results = createResults(corpus); + const { runOptions } = first; + const config: LaneConfig = { + runtime: runOptions.runtime, + model: runOptions.model, + effort: runOptions.effort, + auxiliaryModel: runOptions.auxiliaryModel, + auxiliaryEffort: runOptions.auxiliaryEffort, + maxTurns: runOptions.maxTurns, + judgeModel: first.scoring.reviewer, + origin: runs.map((run) => run.runId).join(', '), + }; + + for (const run of runs) { + const found = new Set(run.scores + .filter((score) => score.verdict === 'known-found') + .flatMap((score) => score.matchedCorpusIds)); + recordShardRun(results, { + skill: 'security-review', + config, + usage: { + inputTokens: run.summary.inputTokens, + outputTokens: run.summary.outputTokens, + cacheReadInputTokens: run.summary.cacheReadInputTokens, + costUSD: run.summary.costUSD, + judgeCostUSD: 0, + }, + hits: new Map(corpus.map((entry) => [entry.id, found.has(entry.id)])), + }); + } + results.summary = summarize(results); + return results; +} diff --git a/packages/security-evals/src/entries.test.ts b/packages/security-evals/src/entries.test.ts new file mode 100644 index 000000000..aa18ea637 --- /dev/null +++ b/packages/security-evals/src/entries.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; +import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries, type SuiteEntry } from './entries.js'; + +function entry(id: string, ref: string, targets: string[]): SuiteEntry { + return { id, source: 'corpus', kind: 'bug', repository: 'getsentry/sentry', ref, targets, summary: id }; +} + +describe('loadCorpusEntries', () => { + it('loads one entry per corpus finding at its pinned sha', () => { + const entries = loadCorpusEntries(); + expect(entries).toHaveLength(86); + expect(new Set(entries.map((e) => e.ref)).size).toBe(6); + expect(entries[0]).toMatchObject({ id: 'sentry-vuln-001', kind: 'bug', lines: '265-275' }); + }); +}); + +describe('loadCaseEntries', () => { + it('names cases by owning skill and keeps clean cases separate', () => { + const cases = loadCaseEntries(); + expect(cases.map((c) => c.id)).toContain('wrdn-dos-review/sentry-grouping-url-regex-backtracking'); + expect(cases.filter((c) => c.kind === 'clean').length).toBeGreaterThan(0); + expect(cases.every((c) => /^[0-9a-f]{40}$/.test(c.ref) && c.targets.length > 0)).toBe(true); + }); + + it('does not duplicate corpus entries', () => { + const corpus = new Set(loadCorpusEntries().map((e) => `${e.ref}:${e.targets[0]}`)); + expect(loadCaseEntries().filter((c) => corpus.has(`${c.ref}:${c.targets[0]}`))).toEqual([]); + }); +}); + +describe('filterEntries', () => { + const entries = [entry('sentry-vuln-001', 'a', ['x.py']), entry('wrdn-authz/scim', 'b', ['y.py'])]; + + it('matches exact ids and trailing-star prefixes', () => { + expect(filterEntries(entries, ['wrdn-authz/*']).map((e) => e.id)).toEqual(['wrdn-authz/scim']); + expect(filterEntries(entries, ['sentry-vuln-001']).map((e) => e.id)).toEqual(['sentry-vuln-001']); + expect(filterEntries(entries, [])).toBe(entries); + }); + + it('fails when nothing matches', () => { + expect(() => filterEntries(entries, ['nope'])).toThrow('No entries match nope'); + }); +}); + +describe('groupShards', () => { + it('makes one shard per ref with deduped targets', () => { + const shards = groupShards([ + entry('one', 'b', ['x.py']), + entry('two', 'a', ['y.py']), + entry('three', 'b', ['x.py', 'z.py']), + ]); + expect(shards.map((s) => [s.ref, s.targets, s.entries.length])).toEqual([ + ['a', ['y.py'], 1], + ['b', ['x.py', 'z.py'], 2], + ]); + }); +}); diff --git a/packages/security-evals/src/entries.ts b/packages/security-evals/src/entries.ts new file mode 100644 index 000000000..fd1fc3c63 --- /dev/null +++ b/packages/security-evals/src/entries.ts @@ -0,0 +1,121 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { z } from 'zod'; + +/** Benchmark corpus that seeds most entries. */ +const CORPUS_PATH = join( + import.meta.dirname, + '..', '..', 'docs', 'src', 'data', 'benchmarking', 'sentry-vulnerability-corpus.json', +); + +/** Hand-picked cases for bug classes the corpus does not cover, one directory per owning skill. */ +const CASES_DIR = join(import.meta.dirname, '..', 'cases'); + +/** + * One scored expectation. A `bug` entry is a hit when a skill reports it. A + * `clean` entry describes a bug the code at `ref` already fixed, so a hit there + * is a false positive. + */ +export interface SuiteEntry { + id: string; + source: 'corpus' | 'case'; + kind: 'bug' | 'clean'; + repository: string; + ref: string; + /** Files the skills scan for this entry. The rest of the worktree is context. */ + targets: string[]; + /** Line range of the bug in the first target, when known. */ + lines?: string; + summary: string; +} + +/** Entries that share a repository and ref run as one Warden pass over one worktree. */ +export interface SuiteShard { + repository: string; + ref: string; + targets: string[]; + entries: SuiteEntry[]; +} + +const CorpusSchema = z.object({ + findings: z.array(z.object({ + id: z.string(), + repository: z.string(), + sha: z.string(), + summary: z.string(), + code: z.object({ path: z.string(), lines: z.string().optional() }), + })), +}); + +const CaseSchema = z.object({ + kind: z.enum(['bug', 'clean']), + summary: z.string(), + repository: z.string(), + ref: z.string().regex(/^[0-9a-f]{40}$/), + targets: z.array(z.string()).min(1), + source: z.string(), +}); + +/** Load one entry per corpus finding. */ +export function loadCorpusEntries(path = CORPUS_PATH): SuiteEntry[] { + const corpus = CorpusSchema.parse(JSON.parse(readFileSync(path, 'utf8'))); + return corpus.findings.map((finding) => ({ + id: finding.id, + source: 'corpus', + kind: 'bug', + repository: finding.repository, + ref: finding.sha, + targets: [finding.code.path], + lines: finding.code.lines, + summary: finding.summary, + })); +} + +/** Load hand-picked cases. Ids read as `/`. */ +export function loadCaseEntries(dir = CASES_DIR): SuiteEntry[] { + return readdirSync(dir, { withFileTypes: true }) + .filter((owner) => owner.isDirectory()) + .flatMap((owner) => readdirSync(join(dir, owner.name)) + .filter((file) => file.endsWith('.json')) + .map((file) => { + const data = CaseSchema.parse(JSON.parse(readFileSync(join(dir, owner.name, file), 'utf8'))); + return { + id: `${owner.name}/${basename(file, '.json')}`, + source: 'case' as const, + kind: data.kind, + repository: data.repository, + ref: data.ref, + targets: data.targets, + summary: data.summary, + }; + })) + .sort((a, b) => a.id.localeCompare(b.id)); +} + +/** + * Keep entries matching any filter. A filter is an exact id, or a prefix + * ending in `*` (`wrdn-dos-review/*`, `sentry-vuln-0*`). No filters keeps all. + */ +export function filterEntries(entries: SuiteEntry[], filters: string[]): SuiteEntry[] { + if (filters.length === 0) return entries; + const selected = entries.filter((entry) => filters.some((filter) => filter.endsWith('*') + ? entry.id.startsWith(filter.slice(0, -1)) + : entry.id === filter)); + if (selected.length === 0) { + throw new Error(`No entries match ${filters.join(', ')}`); + } + return selected; +} + +/** Group entries into one shard per repository and ref. */ +export function groupShards(entries: SuiteEntry[]): SuiteShard[] { + const shards = new Map(); + for (const entry of entries) { + const key = `${entry.repository}@${entry.ref}`; + const shard = shards.get(key) ?? { repository: entry.repository, ref: entry.ref, targets: [], entries: [] }; + shard.entries.push(entry); + shard.targets = [...new Set([...shard.targets, ...entry.targets])].sort(); + shards.set(key, shard); + } + return [...shards.values()].sort((a, b) => a.ref.localeCompare(b.ref)); +} diff --git a/packages/security-evals/src/options.test.ts b/packages/security-evals/src/options.test.ts new file mode 100644 index 000000000..996bcdea4 --- /dev/null +++ b/packages/security-evals/src/options.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { MATRIX_SKILLS, resolveSuiteOptions } from './options.js'; + +describe('resolveSuiteOptions', () => { + it('defaults to the production lane over every matrix skill', () => { + const options = resolveSuiteOptions({}); + expect(options).toMatchObject({ + skills: [...MATRIX_SKILLS], + entries: [], + repeats: 1, + runtime: 'pi', + model: 'openrouter/x-ai/grok-4.5', + effort: 'high', + auxiliaryModel: 'openrouter/openai/gpt-5.6-luna', + auxiliaryEffort: 'high', + maxTurns: 100, + removeWorktrees: false, + }); + expect(options.baseline).toMatch(/baseline\/results\.json$/); + }); + + it('reads filters and overrides from env', () => { + const options = resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_SKILLS: 'wrdn-pii, security-review', + WARDEN_SECURITY_EVALS_ENTRIES: 'wrdn-pii/*', + WARDEN_SECURITY_EVALS_REPEATS: '3', + WARDEN_SECURITY_EVALS_BASELINE: 'none', + }); + expect(options.skills).toEqual(['wrdn-pii', 'security-review']); + expect(options.entries).toEqual(['wrdn-pii/*']); + expect(options.repeats).toBe(3); + expect(options.baseline).toBeUndefined(); + }); + + it('rejects unknown skills and bad counts', () => { + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_SKILLS: 'wrdn-nope' })).toThrow('Unknown skills'); + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_REPEATS: '0' })).toThrow('positive integer'); + }); +}); diff --git a/packages/security-evals/src/options.ts b/packages/security-evals/src/options.ts new file mode 100644 index 000000000..7e2bf9658 --- /dev/null +++ b/packages/security-evals/src/options.ts @@ -0,0 +1,97 @@ +import { join } from 'node:path'; +import { EffortSchema, RuntimeNameSchema, type Effort, type RuntimeName } from '@sentry/warden'; + +/** Skills the matrix compares: builtin security-review plus the remote security skills production runs. */ +export const MATRIX_SKILLS = [ + 'security-review', + 'wrdn-authz', + 'wrdn-pii', + 'wrdn-data-exfil', + 'wrdn-code-execution', + 'wrdn-dos-review', +] as const; + +/** + * Pinned so a warden-skills edit cannot silently move results. + * Bump on purpose and rerun the affected lanes. + */ +export const WARDEN_SKILLS_REMOTE = 'getsentry/warden-skills@daf01f970a28450627a8d2d61d9915a742a735e2'; + +const PACKAGE_DIR = join(import.meta.dirname, '..'); + +/** Committed results used for lanes a run does not execute. */ +export const DEFAULT_BASELINE = join(PACKAGE_DIR, 'baseline', 'results.json'); + +export interface SuiteOptions { + sentryRepo?: string; + skills: string[]; + entries: string[]; + repeats: number; + runtime: RuntimeName; + model: string; + effort: Effort; + auxiliaryModel: string; + auxiliaryEffort: Effort; + judgeModel: string; + maxTurns: number; + /** Concurrent chunk analyses per skill. */ + concurrency: number; + cacheDir: string; + output: string; + /** Results file whose lanes fill in skills this run skips. Undefined disables it. */ + baseline?: string; + /** Delete each worktree after its shard finishes, for disk-limited CI runners. */ + removeWorktrees: boolean; +} + +const PREFIX = 'WARDEN_SECURITY_EVALS_'; + +function list(value: string | undefined): string[] { + return (value ?? '').split(',').map((item) => item.trim()).filter(Boolean); +} + +function positiveInt(name: string, value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${PREFIX}${name} must be a positive integer, got ${value}`); + } + return parsed; +} + +/** + * Read suite options from `WARDEN_SECURITY_EVALS_*`. `scripts/run.ts` maps CLI + * flags onto the same variables. Defaults match the production lane and the + * corpus runbook (maxTurns 100, so a turn limit does not read as a miss). + */ +export function resolveSuiteOptions(env: NodeJS.ProcessEnv = process.env): SuiteOptions { + const read = (name: string): string | undefined => env[`${PREFIX}${name}`]?.trim() || undefined; + + const skills = list(read('SKILLS')); + const unknown = skills.filter((skill) => !(MATRIX_SKILLS as readonly string[]).includes(skill)); + if (unknown.length > 0) { + throw new Error(`Unknown skills: ${unknown.join(', ')}. Choose from ${MATRIX_SKILLS.join(', ')}`); + } + + const cacheDir = read('CACHE_DIR') ?? join(PACKAGE_DIR, '.cache'); + const baseline = read('BASELINE') ?? DEFAULT_BASELINE; + + return { + sentryRepo: read('SENTRY_REPO'), + skills: skills.length > 0 ? skills : [...MATRIX_SKILLS], + entries: list(read('ENTRIES')), + repeats: positiveInt('REPEATS', read('REPEATS'), 1), + runtime: RuntimeNameSchema.parse(read('RUNTIME') ?? 'pi'), + model: read('MODEL') ?? 'openrouter/x-ai/grok-4.5', + effort: EffortSchema.parse(read('EFFORT') ?? 'high'), + auxiliaryModel: read('AUX_MODEL') ?? 'openrouter/openai/gpt-5.6-luna', + auxiliaryEffort: EffortSchema.parse(read('AUX_EFFORT') ?? 'high'), + judgeModel: read('JUDGE_MODEL') ?? 'openrouter/anthropic/claude-sonnet-4.6', + maxTurns: positiveInt('MAX_TURNS', read('MAX_TURNS'), 100), + concurrency: positiveInt('CONCURRENCY', read('CONCURRENCY'), 4), + cacheDir, + output: read('OUTPUT') ?? join(cacheDir, 'results.json'), + baseline: baseline === 'none' ? undefined : baseline, + removeWorktrees: read('REMOVE_WORKTREES') === '1', + }; +} diff --git a/packages/security-evals/src/results.test.ts b/packages/security-evals/src/results.test.ts new file mode 100644 index 000000000..7dd4e09c5 --- /dev/null +++ b/packages/security-evals/src/results.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest'; +import type { SuiteEntry } from './entries.js'; +import { + createResults, + mergeBaseline, + recordShardRun, + renderMarkdown, + serializeResults, + summarize, + type LaneConfig, + type LaneUsage, +} from './results.js'; + +const config: LaneConfig = { + runtime: 'pi', + model: 'm', + effort: 'high', + auxiliaryModel: 'a', + auxiliaryEffort: 'high', + maxTurns: 100, + judgeModel: 'j', +}; +const usage = (costUSD: number): LaneUsage => ({ + inputTokens: 10, outputTokens: 1, cacheReadInputTokens: 5, costUSD, judgeCostUSD: 0.01, +}); + +function entry(id: string, kind: SuiteEntry['kind'] = 'bug'): SuiteEntry { + return { id, source: 'corpus', kind, repository: 'getsentry/sentry', ref: 'a', targets: ['x.py'], summary: id }; +} + +function record(results: ReturnType, skill: string, hits: Record): void { + recordShardRun(results, { skill, config, usage: usage(1), hits: new Map(Object.entries(hits)) }); +} + +describe('summarize', () => { + it('reports recall, unique catches, misses by security-review, and clean flags', () => { + const results = createResults([entry('authz'), entry('dos'), entry('both'), entry('fixed', 'clean')]); + record(results, 'security-review', { authz: false, dos: false, both: true, fixed: false }); + record(results, 'security-review', { authz: false, dos: false, both: true, fixed: false }); + record(results, 'wrdn-authz', { authz: true, dos: false, both: true, fixed: true }); + record(results, 'wrdn-dos-review', { authz: true, dos: true, both: false, fixed: false }); + + const summary = summarize(results); + + expect(summary['security-review']).toMatchObject({ found: 1, recall: 1 / 3, onlyThisLane: [] }); + expect(summary['wrdn-authz']).toMatchObject({ + found: 2, + onlyThisLane: [], + missedBySecurityReview: ['authz'], + cleanFlagged: { hits: 1, runs: 1 }, + }); + expect(summary['wrdn-dos-review']).toMatchObject({ + onlyThisLane: ['dos'], + missedBySecurityReview: ['authz', 'dos'], + }); + expect(results.lanes['security-review']).toMatchObject({ runs: 2, usage: { costUSD: 2, judgeCostUSD: 0.02 } }); + }); + + it('leaves entries out of a lane when its pass failed', () => { + const results = createResults([entry('authz')]); + recordShardRun(results, { skill: 'wrdn-authz', config, usage: usage(3) }); + expect(results.lanes['wrdn-authz']).toMatchObject({ runs: 0, failedRuns: 1, usage: { costUSD: 3 } }); + expect(summarize(results)['wrdn-authz']).toMatchObject({ bugEntries: 0, recall: null }); + }); +}); + +describe('mergeBaseline', () => { + it('adds lanes the run skipped, limited to the selected entries', () => { + const baseline = createResults([entry('authz'), entry('dos')]); + record(baseline, 'security-review', { authz: true, dos: false }); + record(baseline, 'wrdn-authz', { authz: false, dos: false }); + + const results = createResults([entry('authz')]); + record(results, 'wrdn-authz', { authz: true }); + mergeBaseline(results, baseline, ['wrdn-authz']); + + expect(Object.keys(results.entries)).toEqual(['authz']); + expect(results.entries['authz']!.lanes).toEqual({ + 'security-review': { hits: 1, runs: 1 }, + 'wrdn-authz': { hits: 1, runs: 1 }, + }); + }); +}); + +describe('output', () => { + it('renders the lane table and the entry x lane table', () => { + const results = createResults([entry('authz')]); + record(results, 'security-review', { authz: false }); + record(results, 'wrdn-authz', { authz: true }); + results.summary = summarize(results); + + const markdown = renderMarkdown(results); + expect(markdown).toContain('| wrdn-authz | 1.00 | 1/1 | 1 | 1 | 0/0 | 1.00 | 10 | 1 |'); + expect(markdown).toContain('| authz | bug | 0/1 | 1/1 |'); + }); + + it('serializes with sorted keys', () => { + const results = createResults([entry('b'), entry('a')]); + expect(Object.keys(JSON.parse(serializeResults(results)).entries)).toEqual(['a', 'b']); + }); +}); diff --git a/packages/security-evals/src/results.ts b/packages/security-evals/src/results.ts new file mode 100644 index 000000000..229236607 --- /dev/null +++ b/packages/security-evals/src/results.ts @@ -0,0 +1,213 @@ +import type { SuiteEntry } from './entries.js'; + +/** How a lane ran. Baseline lanes keep the config they were recorded with. */ +export interface LaneConfig { + runtime: string; + model: string; + effort: string; + auxiliaryModel: string; + auxiliaryEffort: string; + maxTurns: number; + judgeModel: string; + /** Pinned remote ref for wrdn-* skills. */ + skillsRemote?: string; + /** Where the lane came from when it was not run by this suite. */ + origin?: string; +} + +export interface LaneUsage { + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + costUSD: number; + judgeCostUSD: number; +} + +export interface LaneResult { + config: LaneConfig; + usage: LaneUsage; + /** Passes completed: one per shard and repeat, or one per imported benchmark run. */ + runs: number; + /** Shard passes that errored and were not scored. */ + failedRuns: number; +} + +/** Hits over completed repeats for one entry in one lane. */ +export interface Tally { + hits: number; + runs: number; +} + +export interface EntryResult { + source: SuiteEntry['source']; + kind: SuiteEntry['kind']; + ref: string; + targets: string[]; + lanes: Record; +} + +export interface LaneSummary { + bugEntries: number; + /** Mean hit rate over bug entries this lane ran. */ + recall: number | null; + found: number; + /** Bug entries this lane found and every other lane that ran them missed. */ + onlyThisLane: string[]; + /** Bug entries this lane found and security-review missed. The "do we need it" list. */ + missedBySecurityReview: string[]; + cleanFlagged: Tally; +} + +export interface SuiteResults { + version: 1; + lanes: Record; + entries: Record; + summary: Record; +} + +/** One skill's pass over one shard, scored per entry. */ +export interface ShardRun { + skill: string; + config: LaneConfig; + usage: LaneUsage; + /** Undefined when the pass errored; the entries then get no repeat. */ + hits?: Map; +} + +const SECURITY_REVIEW = 'security-review'; + +/** Start an empty result set for the selected entries. */ +export function createResults(entries: SuiteEntry[]): SuiteResults { + return { + version: 1, + lanes: {}, + entries: Object.fromEntries(entries.map((entry) => [entry.id, { + source: entry.source, + kind: entry.kind, + ref: entry.ref, + targets: entry.targets, + lanes: {}, + }])), + summary: {}, + }; +} + +/** Fold one shard pass into the results. */ +export function recordShardRun(results: SuiteResults, run: ShardRun): void { + const lane = results.lanes[run.skill] ?? { + config: run.config, + usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, costUSD: 0, judgeCostUSD: 0 }, + runs: 0, + failedRuns: 0, + }; + for (const key of Object.keys(lane.usage) as (keyof LaneUsage)[]) { + lane.usage[key] += run.usage[key]; + } + if (!run.hits) { + lane.failedRuns += 1; + } else { + lane.runs += 1; + for (const [entryId, hit] of run.hits) { + const entry = results.entries[entryId]; + if (!entry) continue; + const tally = entry.lanes[run.skill] ?? { hits: 0, runs: 0 }; + tally.runs += 1; + tally.hits += hit ? 1 : 0; + entry.lanes[run.skill] = tally; + } + } + results.lanes[run.skill] = lane; +} + +/** + * Copy lanes this run skipped from a baseline, limited to the selected + * entries, so a candidate lane compares without rerunning the rest. + */ +export function mergeBaseline(results: SuiteResults, baseline: SuiteResults, ranSkills: string[]): void { + for (const [skill, lane] of Object.entries(baseline.lanes)) { + if (ranSkills.includes(skill)) continue; + results.lanes[skill] = lane; + for (const [entryId, entry] of Object.entries(results.entries)) { + const tally = baseline.entries[entryId]?.lanes[skill]; + if (tally) entry.lanes[skill] = tally; + } + } +} + +function hitRate(tally: Tally | undefined): number | undefined { + return tally && tally.runs > 0 ? tally.hits / tally.runs : undefined; +} + +/** Compute per-lane recall, unique catches, and clean-case false positives. */ +export function summarize(results: SuiteResults): Record { + const entries = Object.entries(results.entries); + const summary: Record = {}; + + for (const skill of Object.keys(results.lanes).sort()) { + // Entries this lane ran, with its hit rate on each. + const ran = entries.flatMap(([id, entry]) => { + const rate = hitRate(entry.lanes[skill]); + return rate === undefined ? [] : [{ id, entry, rate }]; + }); + const bugs = ran.filter(({ entry }) => entry.kind === 'bug'); + const clean = ran.filter(({ entry }) => entry.kind === 'clean'); + const found = bugs.filter(({ rate }) => rate > 0); + const othersMissed = (entry: EntryResult) => Object.entries(entry.lanes) + .every(([other, tally]) => other === skill || tally.hits === 0); + + summary[skill] = { + bugEntries: bugs.length, + recall: bugs.length > 0 ? bugs.reduce((sum, { rate }) => sum + rate, 0) / bugs.length : null, + found: found.length, + onlyThisLane: found.filter(({ entry }) => othersMissed(entry)).map(({ id }) => id), + missedBySecurityReview: skill === SECURITY_REVIEW + ? [] + : found.filter(({ entry }) => entry.lanes[SECURITY_REVIEW]?.hits === 0).map(({ id }) => id), + cleanFlagged: { hits: clean.filter(({ rate }) => rate > 0).length, runs: clean.length }, + }; + } + return summary; +} + +function cell(tally: Tally | undefined): string { + return tally ? `${tally.hits}/${tally.runs}` : '-'; +} + +/** Render the summary and the entry x lane hit table as Markdown. */ +export function renderMarkdown(results: SuiteResults): string { + const skills = Object.keys(results.lanes).sort(); + const lines = [ + '## Security evals', + '', + '| lane | recall | found | only this lane | missed by security-review | clean flagged | cost USD | input tokens | output tokens |', + '|---|---|---|---|---|---|---|---|---|', + ]; + for (const skill of skills) { + const lane = results.lanes[skill]; + const s = results.summary[skill]; + if (!lane || !s) continue; + const recall = s.recall === null ? '-' : s.recall.toFixed(2); + lines.push( + `| ${skill}${lane.config.origin ? ' (baseline)' : ''} | ${recall} | ${s.found}/${s.bugEntries} ` + + `| ${s.onlyThisLane.length} | ${skill === SECURITY_REVIEW ? '-' : s.missedBySecurityReview.length} ` + + `| ${cell(s.cleanFlagged)} | ${lane.usage.costUSD.toFixed(2)} | ${lane.usage.inputTokens} | ${lane.usage.outputTokens} |`, + ); + } + + lines.push('', `| entry | kind | ${skills.join(' | ')} |`, `|---|---|${skills.map(() => '---|').join('')}`); + for (const [id, entry] of Object.entries(results.entries).sort(([a], [b]) => a.localeCompare(b))) { + lines.push(`| ${id} | ${entry.kind} | ${skills.map((skill) => cell(entry.lanes[skill])).join(' | ')} |`); + } + return lines.join('\n'); +} + +/** Serialize with sorted keys so two result files diff cleanly. */ +export function serializeResults(results: SuiteResults): string { + const sortKeys = (_key: string, value: unknown): unknown => { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); + } + return value; + }; + return `${JSON.stringify(results, sortKeys, 2)}\n`; +} diff --git a/packages/security-evals/src/score.test.ts b/packages/security-evals/src/score.test.ts new file mode 100644 index 000000000..c98a1a639 --- /dev/null +++ b/packages/security-evals/src/score.test.ts @@ -0,0 +1,78 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { getRuntime, type Finding } from '@sentry/warden'; +import type * as WardenPublic from '@sentry/warden'; +import type { SuiteEntry } from './entries.js'; +import { buildScorePrompt, scoreEntry } from './score.js'; + +vi.mock('@sentry/warden', async (importOriginal) => ({ + ...(await importOriginal()), + getRuntime: vi.fn(), +})); + +const runAuxiliary = vi.fn(); +const usage = { inputTokens: 1, outputTokens: 1, costUSD: 0.002 }; + +const entry: SuiteEntry = { + id: 'sentry-vuln-001', + source: 'corpus', + kind: 'bug', + repository: 'getsentry/sentry', + ref: 'a', + targets: ['src/sentry/api/accept.py'], + lines: '265-275', + summary: 'Invite acceptance deletes arbitrary members.', +}; + +function finding(path: string, title: string): Finding { + return { + id: 'ABC-DEF', + severity: 'high', + title, + description: `${title} details`, + location: { path, startLine: 270, endLine: 272 }, + } as Finding; +} + +describe('scoreEntry', () => { + beforeEach(() => { + runAuxiliary.mockReset(); + vi.mocked(getRuntime).mockReturnValue({ runAuxiliary } as unknown as ReturnType); + }); + + it('skips the judge when no finding is in a target file', async () => { + const verdict = await scoreEntry(entry, [finding('src/other.py', 'Other bug')], { model: 'judge' }); + expect(verdict.hit).toBe(false); + expect(runAuxiliary).not.toHaveBeenCalled(); + }); + + it('asks the judge about target-file findings only', async () => { + runAuxiliary.mockResolvedValue({ + success: true, + data: { matched: true, reasoning: 'same bug' }, + usage, + }); + + const verdict = await scoreEntry(entry, [ + finding('src/other.py', 'Other bug'), + finding('src/sentry/api/accept.py', 'Member deletion without token check'), + ], { model: 'judge' }); + + expect(verdict).toEqual({ hit: true, reasoning: 'same bug', usage }); + const prompt = runAuxiliary.mock.calls[0]![0].prompt as string; + expect(prompt).toContain('Location: src/sentry/api/accept.py lines 265-275'); + expect(prompt).toContain('Member deletion without token check'); + expect(prompt).not.toContain('Other bug'); + }); + + it('throws instead of scoring a miss when the judge fails', async () => { + runAuxiliary.mockResolvedValue({ success: false, error: 'timeout', usage }); + await expect(scoreEntry(entry, [finding('src/sentry/api/accept.py', 'x')], { model: 'judge' })) + .rejects.toThrow('Judge failed for sentry-vuln-001: timeout'); + }); +}); + +describe('buildScorePrompt', () => { + it('states the corpus match rule', () => { + expect(buildScorePrompt(entry, [])).toContain('different bug in the same file does not match'); + }); +}); diff --git a/packages/security-evals/src/score.ts b/packages/security-evals/src/score.ts new file mode 100644 index 000000000..f3358e5f8 --- /dev/null +++ b/packages/security-evals/src/score.ts @@ -0,0 +1,79 @@ +import { z } from 'zod'; +import { getRuntime, type Finding, type UsageStats } from '@sentry/warden'; +import type { SuiteEntry } from './entries.js'; + +const JUDGE_MAX_TOKENS = 2048; +const JUDGE_TIMEOUT_MS = 60_000; + +const VerdictSchema = z.object({ + matched: z.boolean(), + reasoning: z.string(), +}); + +export interface EntryVerdict { + hit: boolean; + reasoning: string; + usage?: UsageStats; +} + +/** Findings a skill reported in any of the entry's target files. */ +export function candidateFindings(entry: SuiteEntry, findings: Finding[]): Finding[] { + return findings.filter((finding) => finding.location && entry.targets.includes(finding.location.path)); +} + +/** Build the corpus semantic-match prompt for one entry. */ +export function buildScorePrompt(entry: SuiteEntry, findings: Finding[]): string { + const location = entry.lines ? `${entry.targets[0]} lines ${entry.lines}` : entry.targets.join(', '); + const block = findings.map((finding, index) => { + const loc = finding.location; + const lines = loc?.endLine ? `${loc.startLine}-${loc.endLine}` : `${loc?.startLine ?? '?'}`; + return `[${index}] ${loc?.path}:${lines}\n title: ${finding.title}\n description: ${finding.description}`; + }).join('\n\n'); + + return `You score a security code reviewer against a known vulnerability. + +## Known vulnerability +Location: ${location} +${entry.summary} + +## Reported findings +${block} + +## Rule +A finding matches when it would lead a reviewer to the same bug in roughly the +same code location. Wording and exact line numbers do not need to match. A +finding about a different bug in the same file does not match, even when it is +real. A broad finding that only names the file or the bug class without this +specific flaw does not match. + +Respond with ONLY a JSON object: +{"matched": true, "reasoning": "one sentence"}`; +} + +/** Decide whether a skill's findings name the entry's bug. No candidates means no hit and no model call. */ +export async function scoreEntry( + entry: SuiteEntry, + findings: Finding[], + options: { model: string; apiKey?: string }, +): Promise { + const candidates = candidateFindings(entry, findings); + if (candidates.length === 0) { + return { hit: false, reasoning: 'no findings in target files' }; + } + + const result = await getRuntime('pi').runAuxiliary({ + task: 'eval_judge', + agentName: 'security-eval-judge', + apiKey: options.apiKey || undefined, + prompt: buildScorePrompt(entry, candidates), + schema: VerdictSchema, + model: options.model, + maxTokens: JUDGE_MAX_TOKENS, + timeout: JUDGE_TIMEOUT_MS, + maxRetries: 2, + }); + if (!result.success) { + throw new Error(`Judge failed for ${entry.id}: ${result.error}`); + } + return { hit: result.data.matched, reasoning: result.data.reasoning, usage: result.usage }; +} diff --git a/packages/security-evals/src/security.eval.ts b/packages/security-evals/src/security.eval.ts new file mode 100644 index 000000000..13de7ac27 --- /dev/null +++ b/packages/security-evals/src/security.eval.ts @@ -0,0 +1,82 @@ +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { afterAll, describe, it } from 'vitest'; +import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries } from './entries.js'; +import { resolveSuiteOptions } from './options.js'; +import { + createResults, + mergeBaseline, + recordShardRun, + renderMarkdown, + serializeResults, + summarize, + type SuiteResults, +} from './results.js'; +import { loadSkills, runShardPass } from './shard.js'; +import { ensureWorktree, removeWorktree, resolveSentryRepo } from './worktrees.js'; + +const options = resolveSuiteOptions(); +const sentryRepo = resolveSentryRepo(options.sentryRepo); +const entries = filterEntries([...loadCorpusEntries(), ...loadCaseEntries()], options.entries); +const shards = groupShards(entries); +const foreign = shards.filter((shard) => shard.repository !== 'getsentry/sentry'); +if (foreign.length > 0) { + throw new Error(`Only getsentry/sentry entries are supported, got ${foreign.map((s) => s.repository).join(', ')}`); +} + +// The directory name is the Roach `worktree-root` value; keep the two in sync. +const worktreesDir = join(options.cacheDir, 'security-evals-worktrees'); +const apiKey = process.env['WARDEN_ANTHROPIC_API_KEY'] || process.env['ANTHROPIC_API_KEY'] || ''; +const results = createResults(entries); +let skills: ReturnType | undefined; + +// A pass runs every selected skill over a whole shard, which can take an hour on large shards. +const PASS_TIMEOUT_MS = 3 * 60 * 60 * 1000; + +describe('security evals', () => { + for (const shard of shards) { + for (let repeat = 1; repeat <= options.repeats; repeat++) { + const name = `${shard.ref.slice(0, 8)} pass ${repeat}/${options.repeats} ` + + `(${shard.targets.length} files, ${shard.entries.length} entries)`; + it(name, { timeout: PASS_TIMEOUT_MS }, async () => { + skills ??= loadSkills(options.skills); + const worktree = ensureWorktree(sentryRepo, worktreesDir, shard.ref); + try { + const runs = await runShardPass({ + shard, + worktree, + skills: await skills, + options, + apiKey, + log: (message) => console.log(` ${message}`), + }); + for (const run of runs) recordShardRun(results, run); + const failed = runs.filter((run) => !run.hits).map((run) => run.skill); + if (failed.length > 0) { + throw new Error(`Skill passes failed: ${failed.join(', ')}`); + } + } finally { + if (options.removeWorktrees && repeat === options.repeats) { + removeWorktree(sentryRepo, worktree); + } + } + }); + } + } +}); + +afterAll(() => { + if (Object.keys(results.lanes).length === 0) return; + if (options.baseline && existsSync(options.baseline)) { + const baseline = JSON.parse(readFileSync(options.baseline, 'utf8')) as SuiteResults; + mergeBaseline(results, baseline, options.skills); + } + results.summary = summarize(results); + + mkdirSync(dirname(options.output), { recursive: true }); + writeFileSync(options.output, serializeResults(results)); + const markdown = renderMarkdown(results); + console.log(`\n${markdown}\n\nResults: ${options.output}`); + const stepSummary = process.env['GITHUB_STEP_SUMMARY']; + if (stepSummary) appendFileSync(stepSummary, `${markdown}\n`); +}); diff --git a/packages/security-evals/src/setup.ts b/packages/security-evals/src/setup.ts new file mode 100644 index 000000000..c9a08e9ee --- /dev/null +++ b/packages/security-evals/src/setup.ts @@ -0,0 +1,14 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { config as dotenvConfig } from 'dotenv'; +import { bridgeWardenProviderApiKeyEnv } from '../../warden/src/utils/index.js'; + +// Same env files as packages/evals, later files win. +const root = join(import.meta.dirname, '..', '..', '..'); +for (const file of ['.env', '.env.local', '.env.test']) { + const path = join(root, file); + if (existsSync(path)) { + dotenvConfig({ path, override: true }); + } +} +bridgeWardenProviderApiKeyEnv(); diff --git a/packages/security-evals/src/shard.ts b/packages/security-evals/src/shard.ts new file mode 100644 index 000000000..62d5a57f4 --- /dev/null +++ b/packages/security-evals/src/shard.ts @@ -0,0 +1,116 @@ +import { join } from 'node:path'; +import { resolveSkillAsync, runSkill, type SkillReport, type UsageStats } from '@sentry/warden'; +// Not on the public surface; the CLI's file mode builds its context the same way. +import { buildFileEventContext } from '../../warden/src/cli/context.js'; +import type { SuiteShard } from './entries.js'; +import type { SuiteOptions } from './options.js'; +import { WARDEN_SKILLS_REMOTE } from './options.js'; +import type { LaneConfig, LaneUsage, ShardRun } from './results.js'; +import { scoreEntry } from './score.js'; + +type SkillDefinition = Awaited>; + +const BUILTIN_SECURITY_REVIEW = join(import.meta.dirname, '..', '..', 'warden', 'src', 'builtin-skills', 'security-review'); + +/** Corpus runbook scan limits, so explicit targets are never dropped by PR-sized budgets. */ +const SCAN = { maxFiles: 200, maxChangedLines: 50_000 }; + +/** + * Load each skill once. security-review comes from this checkout by path, so a + * same-named skill in the scanned repo cannot shadow it. wrdn-* skills come + * from the pinned warden-skills ref. + */ +export async function loadSkills(names: string[]): Promise> { + const skills = new Map(); + // Sequential: remote skills share one checkout of the pinned ref. + for (const name of names) { + skills.set(name, name === 'security-review' + ? await resolveSkillAsync(BUILTIN_SECURITY_REVIEW) + : await resolveSkillAsync(name, undefined, { remote: WARDEN_SKILLS_REMOTE })); + } + return skills; +} + +/** Lane config recorded with each result. */ +function laneConfig(skill: string, options: SuiteOptions): LaneConfig { + return { + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + maxTurns: options.maxTurns, + judgeModel: options.judgeModel, + ...(skill === 'security-review' ? {} : { skillsRemote: WARDEN_SKILLS_REMOTE }), + }; +} + +function reportUsage(report: SkillReport): LaneUsage { + const parts: UsageStats[] = [report.usage, ...Object.values(report.auxiliaryUsage ?? {})] + .filter((usage): usage is UsageStats => usage !== undefined); + return { + inputTokens: parts.reduce((sum, usage) => sum + usage.inputTokens, 0), + outputTokens: parts.reduce((sum, usage) => sum + usage.outputTokens, 0), + cacheReadInputTokens: parts.reduce((sum, usage) => sum + (usage.cacheReadInputTokens ?? 0), 0), + costUSD: parts.reduce((sum, usage) => sum + usage.costUSD, 0), + judgeCostUSD: 0, + }; +} + +/** + * One Warden pass over a shard's worktree: build the file context once, run + * every selected skill against it, then score each entry per skill. Entries in + * a file with a failed chunk get no repeat instead of a false miss. + */ +export async function runShardPass(args: { + shard: SuiteShard; + worktree: string; + skills: Map; + options: SuiteOptions; + apiKey: string; + log: (message: string) => void; +}): Promise { + const { shard, worktree, skills, options, apiKey, log } = args; + const context = await buildFileEventContext({ patterns: shard.targets, cwd: worktree, scan: SCAN }); + const scanned = new Set(context.pullRequest?.files.map((file) => file.filename)); + const missing = shard.targets.filter((target) => !scanned.has(target)); + if (missing.length > 0) { + throw new Error(`Shard ${shard.ref} is missing targets: ${missing.join(', ')}`); + } + + return Promise.all([...skills].map(async ([name, skill]): Promise => { + const config = laneConfig(name, options); + const report = await runSkill(skill, context, { + apiKey: apiKey || undefined, + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + synthesisModel: options.auxiliaryModel, + maxTurns: options.maxTurns, + concurrency: options.concurrency, + verifyFindings: true, + }); + const usage = reportUsage(report); + if (report.error) { + log(`${name} @ ${shard.ref.slice(0, 8)} failed: ${report.error.message}`); + return { skill: name, config, usage }; + } + + const failedFiles = new Set(report.hunkFailures?.map((failure) => failure.filename)); + const hits = new Map(); + for (const entry of shard.entries) { + if (entry.targets.some((target) => failedFiles.has(target))) { + log(`${name} @ ${shard.ref.slice(0, 8)}: ${entry.id} skipped, a target chunk failed`); + continue; + } + const verdict = await scoreEntry(entry, report.findings, { model: options.judgeModel, apiKey }); + usage.judgeCostUSD += verdict.usage?.costUSD ?? 0; + hits.set(entry.id, verdict.hit); + } + log(`${name} @ ${shard.ref.slice(0, 8)}: ${report.findings.length} findings, ` + + `${[...hits.values()].filter(Boolean).length}/${hits.size} entries hit`); + return { skill: name, config, usage, hits }; + })); +} diff --git a/packages/security-evals/src/worktrees.test.ts b/packages/security-evals/src/worktrees.test.ts new file mode 100644 index 000000000..c3d975ed7 --- /dev/null +++ b/packages/security-evals/src/worktrees.test.ts @@ -0,0 +1,86 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ensureWorktree, removeWorktree, resolveSentryRepo } from './worktrees.js'; + +function configuredEmail(): string { + try { + return execFileSync('git', ['config', '--get', 'user.email'], { encoding: 'utf8' }).trim(); + } catch { + return ''; + } +} + +// CI runners have no git identity; keep a configured one so commits stay attributable locally. +const IDENTITY = configuredEmail() ? [] : ['-c', 'user.name=Warden Test', '-c', 'user.email=test@warden.dev']; + +function git(args: string[], cwd: string): string { + return execFileSync('git', [...IDENTITY, '-c', 'commit.gpgsign=false', ...args], { cwd, encoding: 'utf8' }).trim(); +} + +function commit(repo: string, content: string): string { + writeFileSync(join(repo, 'app.py'), content); + git(['add', 'app.py'], repo); + git(['commit', '-q', '-m', content], repo); + return git(['rev-parse', 'HEAD'], repo); +} + +describe('ensureWorktree', () => { + let root: string; + let upstream: string; + let clone: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'security-evals-')); + upstream = join(root, 'upstream'); + clone = join(root, 'clone'); + git(['init', '-q', '--initial-branch=main', upstream], root); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it('creates a detached worktree once and reuses it', () => { + const ref = commit(upstream, 'v1'); + git(['clone', '-q', upstream, clone], root); + const worktrees = join(root, 'worktrees'); + + const path = ensureWorktree(clone, worktrees, ref); + expect(path).toBe(join(worktrees, ref)); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v1'); + expect(ensureWorktree(clone, worktrees, ref)).toBe(path); + + removeWorktree(clone, path); + expect(existsSync(path)).toBe(false); + }); + + it('fetches a ref the clone does not have yet', () => { + commit(upstream, 'v1'); + git(['clone', '-q', upstream, clone], root); + const later = commit(upstream, 'v2'); + + const path = ensureWorktree(clone, join(root, 'worktrees'), later); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v2'); + }); + + it('refuses a worktree that moved off its ref', () => { + const first = commit(upstream, 'v1'); + const second = commit(upstream, 'v2'); + git(['clone', '-q', upstream, clone], root); + const worktrees = join(root, 'worktrees'); + const path = ensureWorktree(clone, worktrees, first); + git(['checkout', '-q', '--detach', second], path); + + expect(() => ensureWorktree(clone, worktrees, first)).toThrow(`expected ${first}`); + }); +}); + +describe('resolveSentryRepo', () => { + it('fails fast without a clone instead of cloning', () => { + expect(() => resolveSentryRepo(undefined)).toThrow('WARDEN_SECURITY_EVALS_SENTRY_REPO'); + expect(() => resolveSentryRepo(tmpdir())).toThrow('is not a git checkout'); + }); +}); diff --git a/packages/security-evals/src/worktrees.ts b/packages/security-evals/src/worktrees.ts new file mode 100644 index 000000000..677b32705 --- /dev/null +++ b/packages/security-evals/src/worktrees.ts @@ -0,0 +1,68 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** Env var that points the suite at a local getsentry/sentry clone. */ +const SENTRY_REPO_ENV = 'WARDEN_SECURITY_EVALS_SENTRY_REPO'; + +function git(args: string[], cwd: string): string { + return execFileSync('git', args, { + cwd, + encoding: 'utf8', + // A partial clone would otherwise fetch a missing object on a presence check. + env: { ...process.env, GIT_NO_LAZY_FETCH: '1' }, + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function hasCommit(repoPath: string, ref: string): boolean { + try { + git(['cat-file', '-e', `${ref}^{commit}`], repoPath); + return true; + } catch { + return false; + } +} + +/** Resolve the clone the suite builds worktrees from. Never clones on its own. */ +export function resolveSentryRepo(path: string | undefined): string { + if (!path) { + throw new Error( + `${SENTRY_REPO_ENV} (or --sentry-repo) must point at a local getsentry/sentry clone. ` + + 'A blobless clone is enough: git clone --filter=blob:none https://github.com/getsentry/sentry.git', + ); + } + const repoPath = resolve(path); + try { + return git(['rev-parse', '--show-toplevel'], repoPath); + } catch { + throw new Error(`${repoPath} is not a git checkout. Point ${SENTRY_REPO_ENV} at a getsentry/sentry clone.`); + } +} + +/** + * Return a detached worktree of `ref`, creating it once and reusing it after. + * Fetches `ref` from origin only when the clone does not already have it. + */ +export function ensureWorktree(repoPath: string, worktreesDir: string, ref: string): string { + const path = join(worktreesDir, ref); + if (existsSync(path)) { + const head = git(['rev-parse', 'HEAD'], path); + if (head !== ref) { + throw new Error(`Worktree ${path} is at ${head}, expected ${ref}. Remove it and rerun.`); + } + return path; + } + + if (!hasCommit(repoPath, ref)) { + git(['fetch', '--quiet', 'origin', ref], repoPath); + } + mkdirSync(worktreesDir, { recursive: true }); + git(['worktree', 'add', '--quiet', '--detach', path, ref], repoPath); + return path; +} + +/** Remove a worktree created by `ensureWorktree`. CI uses this to cap disk use. */ +export function removeWorktree(repoPath: string, path: string): void { + git(['worktree', 'remove', '--force', path], repoPath); +} diff --git a/packages/security-evals/tsconfig.json b/packages/security-evals/tsconfig.json new file mode 100644 index 000000000..d1eb8b1d4 --- /dev/null +++ b/packages/security-evals/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "jsx": "react-jsx", + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "paths": { + "@sentry/warden": ["../warden/src/index.ts"] + }, + "types": ["node", "vitest"] + }, + "include": ["src/**/*", "scripts/**/*"] +} diff --git a/packages/security-evals/vitest.config.ts b/packages/security-evals/vitest.config.ts new file mode 100644 index 000000000..bdd25284e --- /dev/null +++ b/packages/security-evals/vitest.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +// Run through `pnpm evals:security`, which maps CLI flags onto WARDEN_SECURITY_EVALS_* env vars. +export default defineConfig({ + resolve: { + alias: { + '@sentry/warden': fileURLToPath(new URL('../warden/src/index.ts', import.meta.url)), + '@sentry/warden-service-api': fileURLToPath(new URL('../warden-service-api/src/index.ts', import.meta.url)), + }, + }, + test: { + include: ['src/**/*.eval.ts'], + exclude: ['**/node_modules/**', '**/dist/**'], + // Roach records model calls and replays them while the worktree content stays pinned. + globalSetup: ['@sentry/roach/vitest'], + setupFiles: ['./src/setup.ts', '@sentry/roach/vitest/setup'], + // Passes share one results object and one worktree per shard. + fileParallelism: false, + }, +}); diff --git a/packages/security-evals/vitest.unit.config.ts b/packages/security-evals/vitest.unit.config.ts new file mode 100644 index 000000000..a971045fe --- /dev/null +++ b/packages/security-evals/vitest.unit.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +export default defineConfig({ + resolve: { + alias: { + '@sentry/warden': fileURLToPath(new URL('../warden/src/index.ts', import.meta.url)), + '@sentry/warden-service-api': fileURLToPath(new URL('../warden-service-api/src/index.ts', import.meta.url)), + }, + }, + test: { + include: ['src/**/*.test.ts'], + exclude: ['**/node_modules/**', '**/dist/**'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09437490c..b29f3593e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -212,6 +212,34 @@ importers: specifier: ^4.1.11 version: 4.1.11(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.11)(happy-dom@21.0.4)(vite@8.3.5(@types/node@25.9.1)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.12)(yaml@2.9.0)) + packages/security-evals: + dependencies: + '@sentry/warden': + specifier: workspace:* + version: link:../warden + dotenv: + specifier: ^17.4.2 + version: 17.4.2 + zod: + specifier: ^4.6.7 + version: 4.6.7 + devDependencies: + '@sentry/roach': + specifier: ^0.2.0 + version: 0.2.0(rollup@4.60.4)(vitest@4.1.11) + '@types/node': + specifier: ^25.9.1 + version: 25.9.1 + tsx: + specifier: ^4.23.12 + version: 4.23.12 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.11 + version: 4.1.11(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.11)(happy-dom@21.0.4)(vite@8.3.5(@types/node@25.9.1)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.12)(yaml@2.9.0)) + packages/warden: dependencies: '@anthropic-ai/claude-agent-sdk': From ec590f956f0dab97a2fbdefd15502ea451ed62f8 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:58:19 +0000 Subject: [PATCH 04/11] fix(evals): Load wrdn skills from a local warden-skills worktree The first live run lost the whole wrdn-authz lane: every case cloned getsentry/warden-skills on its own and the clone failed, so the lane recorded nothing. Resolve the pinned ref once at setup instead. - --skills-repo (WARDEN_SECURITY_EVALS_SKILLS_REPO) points at a local clone; the suite makes one worktree at the pinned SHA and loads wrdn-* skills from it by path - a missing clone or a skill that fails to load stops collection once, before any pass, so nothing is scored as a miss - the Security Evals workflow checks out warden-skills at the pin; a unit test keeps the workflow and options.ts pins in sync - git calls drop Roach's proxy variables, and lazy fetch is only disabled for the presence check, so worktrees of blobless clones can fetch their blobs --- .github/workflows/security-evals.yml | 11 ++++ packages/security-evals/README.md | 33 +++++++----- packages/security-evals/roach.json | 6 +-- packages/security-evals/scripts/run.ts | 6 ++- packages/security-evals/src/options.test.ts | 14 ++++- packages/security-evals/src/options.ts | 10 ++-- packages/security-evals/src/results.ts | 4 +- packages/security-evals/src/security.eval.ts | 25 ++++++--- packages/security-evals/src/shard.test.ts | 52 +++++++++++++++++++ packages/security-evals/src/shard.ts | 35 +++++++++---- packages/security-evals/src/worktrees.test.ts | 18 +++++-- packages/security-evals/src/worktrees.ts | 50 ++++++++++++------ 12 files changed, 202 insertions(+), 62 deletions(-) create mode 100644 packages/security-evals/src/shard.test.ts diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml index aaab2d48c..857e9ebdc 100644 --- a/.github/workflows/security-evals.yml +++ b/.github/workflows/security-evals.yml @@ -39,6 +39,9 @@ jobs: timeout-minutes: 360 env: SENTRY_CLONE: ${{ github.workspace }}/../sentry + # Must match WARDEN_SKILLS_REF in packages/security-evals/src/options.ts. + WARDEN_SKILLS_REF: daf01f970a28450627a8d2d61d9915a742a735e2 + SKILLS_CHECKOUT: packages/security-evals/.cache/warden-skills steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -50,6 +53,13 @@ jobs: - run: pnpm install --frozen-lockfile + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: getsentry/warden-skills + ref: ${{ env.WARDEN_SKILLS_REF }} + path: ${{ env.SKILLS_CHECKOUT }} + persist-credentials: false + # Keyed on the files that pin the scanned SHAs. Older clones still restore; # the suite fetches whatever SHA they lack. - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -66,6 +76,7 @@ jobs: WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} WARDEN_SECURITY_EVALS_SENTRY_REPO: ${{ env.SENTRY_CLONE }} + WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }} WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} WARDEN_SECURITY_EVALS_REPEATS: ${{ inputs.repeats }} diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index 43024063c..83d007250 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -24,18 +24,20 @@ classes, and permission checks the way they do in production. ## Running it -You need a local getsentry/sentry clone. Blobless is enough: +You need local clones of getsentry/sentry and getsentry/warden-skills. Blobless +is enough: ```bash git clone --filter=blob:none https://github.com/getsentry/sentry.git ~/src/sentry +git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills ``` ```bash # Every lane over every entry -pnpm evals:security --sentry-repo ~/src/sentry +pnpm evals:security --sentry-repo ~/src/sentry --skills-repo ~/src/warden-skills # One candidate lane on its own cases, three passes; other lanes come from the baseline -pnpm evals:security --sentry-repo ~/src/sentry \ +pnpm evals:security --sentry-repo ~/src/sentry --skills-repo ~/src/warden-skills \ --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3 ``` @@ -44,16 +46,21 @@ pnpm evals:security --sentry-repo ~/src/sentry \ runtime, `openrouter/x-ai/grok-4.5` at high effort, and `openrouter/openai/gpt-5.6-luna` at high effort for auxiliary work. `maxTurns` defaults to 100, as in the corpus runbook, so a turn cap does not read as a -miss. - -Setup is the only network step: remote skills load from the pinned -warden-skills ref, and each SHA gets `git fetch origin ` if the clone -lacks it, then `git worktree add --detach` under `.cache/`. Worktrees are -reused across lanes, entries, and repeats. Without `--sentry-repo` the suite -fails before doing anything. - -In CI, run the **Security Evals** workflow by hand. It restores a cached -blobless clone, runs the suite, and uploads `results.json`. +miss. `--skills-repo` is only needed when a `wrdn-*` lane is selected. + +Each pinned ref gets `git fetch origin ` only if its clone lacks it, then +one `git worktree add --detach` under `.cache/`. The warden-skills worktree +(ref in `src/options.ts`) and every skill load happen once, before any pass. A +missing clone or a skill that will not load fails the run there, before any +entry is scored. Each sentry SHA gets its worktree before its first pass, and +it is reused across lanes, entries, and repeats. Past that one fetch, model +APIs are the only network a pass touches. + +In CI, run the **Security Evals** workflow by hand. It checks out +warden-skills at the pinned ref, restores a cached blobless sentry clone, +runs the suite, and uploads `results.json`. Bumping the pin means editing +`WARDEN_SKILLS_REF` in both `src/options.ts` and the workflow; a unit test +fails if they drift. ## How it scores diff --git a/packages/security-evals/roach.json b/packages/security-evals/roach.json index ebcec0d91..57f7331d4 100644 --- a/packages/security-evals/roach.json +++ b/packages/security-evals/roach.json @@ -26,7 +26,7 @@ }, "finding-id": "(?; see src/options.ts for defaults. const FLAGS = { 'sentry-repo': 'SENTRY_REPO', + 'skills-repo': 'SKILLS_REPO', skills: 'SKILLS', entries: 'ENTRIES', repeats: 'REPEATS', @@ -22,11 +23,12 @@ const FLAGS = { baseline: 'BASELINE', } as const; -const PATH_FLAGS = new Set(['sentry-repo', 'cache-dir', 'output', 'baseline']); +const PATH_FLAGS = new Set(['sentry-repo', 'skills-repo', 'cache-dir', 'output', 'baseline']); -const USAGE = `usage: pnpm evals:security --sentry-repo [options] +const USAGE = `usage: pnpm evals:security --sentry-repo [--skills-repo ] [options] --sentry-repo Local getsentry/sentry clone (required) + --skills-repo Local getsentry/warden-skills clone (required for wrdn-* lanes) --skills Lanes to run (default: security-review and the five wrdn-* skills) --entries Entry ids, or prefixes ending in * (default: all) --repeats Passes per shard (default: 1) diff --git a/packages/security-evals/src/options.test.ts b/packages/security-evals/src/options.test.ts index 996bcdea4..603d60460 100644 --- a/packages/security-evals/src/options.test.ts +++ b/packages/security-evals/src/options.test.ts @@ -1,5 +1,7 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { MATRIX_SKILLS, resolveSuiteOptions } from './options.js'; +import { MATRIX_SKILLS, WARDEN_SKILLS_REF, resolveSuiteOptions } from './options.js'; describe('resolveSuiteOptions', () => { it('defaults to the production lane over every matrix skill', () => { @@ -25,13 +27,23 @@ describe('resolveSuiteOptions', () => { WARDEN_SECURITY_EVALS_ENTRIES: 'wrdn-pii/*', WARDEN_SECURITY_EVALS_REPEATS: '3', WARDEN_SECURITY_EVALS_BASELINE: 'none', + WARDEN_SECURITY_EVALS_SKILLS_REPO: '/src/warden-skills', }); + expect(options.skillsRepo).toBe('/src/warden-skills'); expect(options.skills).toEqual(['wrdn-pii', 'security-review']); expect(options.entries).toEqual(['wrdn-pii/*']); expect(options.repeats).toBe(3); expect(options.baseline).toBeUndefined(); }); + it('pins the same warden-skills ref the CI workflow checks out', () => { + const workflow = readFileSync( + join(import.meta.dirname, '..', '..', '..', '.github', 'workflows', 'security-evals.yml'), + 'utf8', + ); + expect(workflow).toContain(`WARDEN_SKILLS_REF: ${WARDEN_SKILLS_REF}`); + }); + it('rejects unknown skills and bad counts', () => { expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_SKILLS: 'wrdn-nope' })).toThrow('Unknown skills'); expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_REPEATS: '0' })).toThrow('positive integer'); diff --git a/packages/security-evals/src/options.ts b/packages/security-evals/src/options.ts index 7e2bf9658..248fce670 100644 --- a/packages/security-evals/src/options.ts +++ b/packages/security-evals/src/options.ts @@ -12,10 +12,11 @@ export const MATRIX_SKILLS = [ ] as const; /** - * Pinned so a warden-skills edit cannot silently move results. - * Bump on purpose and rerun the affected lanes. + * getsentry/warden-skills commit the wrdn-* lanes load from. Pinned so a + * warden-skills edit cannot silently move results. Bump on purpose, update + * .github/workflows/security-evals.yml to match, and rerun the affected lanes. */ -export const WARDEN_SKILLS_REMOTE = 'getsentry/warden-skills@daf01f970a28450627a8d2d61d9915a742a735e2'; +export const WARDEN_SKILLS_REF = 'daf01f970a28450627a8d2d61d9915a742a735e2'; const PACKAGE_DIR = join(import.meta.dirname, '..'); @@ -24,6 +25,8 @@ export const DEFAULT_BASELINE = join(PACKAGE_DIR, 'baseline', 'results.json'); export interface SuiteOptions { sentryRepo?: string; + /** Local getsentry/warden-skills checkout. Required when a wrdn-* lane is selected. */ + skillsRepo?: string; skills: string[]; entries: string[]; repeats: number; @@ -78,6 +81,7 @@ export function resolveSuiteOptions(env: NodeJS.ProcessEnv = process.env): Suite return { sentryRepo: read('SENTRY_REPO'), + skillsRepo: read('SKILLS_REPO'), skills: skills.length > 0 ? skills : [...MATRIX_SKILLS], entries: list(read('ENTRIES')), repeats: positiveInt('REPEATS', read('REPEATS'), 1), diff --git a/packages/security-evals/src/results.ts b/packages/security-evals/src/results.ts index 229236607..bc5f747d1 100644 --- a/packages/security-evals/src/results.ts +++ b/packages/security-evals/src/results.ts @@ -9,8 +9,8 @@ export interface LaneConfig { auxiliaryEffort: string; maxTurns: number; judgeModel: string; - /** Pinned remote ref for wrdn-* skills. */ - skillsRemote?: string; + /** Pinned warden-skills ref for wrdn-* lanes. */ + skillsRef?: string; /** Where the lane came from when it was not run by this suite. */ origin?: string; } diff --git a/packages/security-evals/src/security.eval.ts b/packages/security-evals/src/security.eval.ts index 13de7ac27..43c12bb6f 100644 --- a/packages/security-evals/src/security.eval.ts +++ b/packages/security-evals/src/security.eval.ts @@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } fr import { dirname, join } from 'node:path'; import { afterAll, describe, it } from 'vitest'; import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries } from './entries.js'; -import { resolveSuiteOptions } from './options.js'; +import { WARDEN_SKILLS_REF, resolveSuiteOptions } from './options.js'; import { createResults, mergeBaseline, @@ -13,10 +13,10 @@ import { type SuiteResults, } from './results.js'; import { loadSkills, runShardPass } from './shard.js'; -import { ensureWorktree, removeWorktree, resolveSentryRepo } from './worktrees.js'; +import { SENTRY_CHECKOUT, SKILLS_CHECKOUT, ensureWorktree, removeWorktree, resolveCheckout } from './worktrees.js'; const options = resolveSuiteOptions(); -const sentryRepo = resolveSentryRepo(options.sentryRepo); +const sentryRepo = resolveCheckout(SENTRY_CHECKOUT, options.sentryRepo); const entries = filterEntries([...loadCorpusEntries(), ...loadCaseEntries()], options.entries); const shards = groupShards(entries); const foreign = shards.filter((shard) => shard.repository !== 'getsentry/sentry'); @@ -24,11 +24,23 @@ if (foreign.length > 0) { throw new Error(`Only getsentry/sentry entries are supported, got ${foreign.map((s) => s.repository).join(', ')}`); } -// The directory name is the Roach `worktree-root` value; keep the two in sync. +// Both worktree directory names are Roach values in roach.json; keep them in sync. const worktreesDir = join(options.cacheDir, 'security-evals-worktrees'); +const skillsWorktreesDir = join(options.cacheDir, 'skills-worktrees'); const apiKey = process.env['WARDEN_ANTHROPIC_API_KEY'] || process.env['ANTHROPIC_API_KEY'] || ''; const results = createResults(entries); -let skills: ReturnType | undefined; + +// Setup, once per run: a skills problem fails collection here, so no pass runs and +// no entry is scored as a miss. No network unless the clone lacks the pinned ref. +const needsSkillsRepo = options.skills.some((skill) => skill !== 'security-review'); +const skillsWorktree = needsSkillsRepo + ? ensureWorktree( + resolveCheckout(SKILLS_CHECKOUT, options.skillsRepo), + skillsWorktreesDir, + WARDEN_SKILLS_REF, + ) + : undefined; +const skills = await loadSkills(options.skills, skillsWorktree); // A pass runs every selected skill over a whole shard, which can take an hour on large shards. const PASS_TIMEOUT_MS = 3 * 60 * 60 * 1000; @@ -39,13 +51,12 @@ describe('security evals', () => { const name = `${shard.ref.slice(0, 8)} pass ${repeat}/${options.repeats} ` + `(${shard.targets.length} files, ${shard.entries.length} entries)`; it(name, { timeout: PASS_TIMEOUT_MS }, async () => { - skills ??= loadSkills(options.skills); const worktree = ensureWorktree(sentryRepo, worktreesDir, shard.ref); try { const runs = await runShardPass({ shard, worktree, - skills: await skills, + skills, options, apiKey, log: (message) => console.log(` ${message}`), diff --git a/packages/security-evals/src/shard.test.ts b/packages/security-evals/src/shard.test.ts new file mode 100644 index 000000000..6dcb0b761 --- /dev/null +++ b/packages/security-evals/src/shard.test.ts @@ -0,0 +1,52 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { loadSkills } from './shard.js'; + +function writeSkill(root: string, dir: string, name: string): void { + mkdirSync(join(root, 'skills', dir), { recursive: true }); + writeFileSync( + join(root, 'skills', dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: Finds ${name} bugs.\n---\n\nLook for ${name} bugs.\n`, + ); +} + +describe('loadSkills', () => { + let worktree: string; + + beforeEach(() => { + worktree = mkdtempSync(join(tmpdir(), 'warden-skills-')); + }); + + afterEach(() => { + rmSync(worktree, { recursive: true, force: true }); + }); + + it('loads builtin security-review and wrdn skills from the skills worktree', async () => { + writeSkill(worktree, 'wrdn-authz', 'wrdn-authz'); + + const skills = await loadSkills(['security-review', 'wrdn-authz'], worktree); + + expect([...skills.keys()]).toEqual(['security-review', 'wrdn-authz']); + expect(skills.get('security-review')?.rootDir).toMatch(/builtin-skills\/security-review$/); + expect(skills.get('wrdn-authz')?.rootDir).toBe(join(worktree, 'skills', 'wrdn-authz')); + }); + + it('needs no skills checkout for security-review alone', async () => { + const skills = await loadSkills(['security-review'], undefined); + expect([...skills.keys()]).toEqual(['security-review']); + }); + + it('fails once with the skill and path when a wrdn skill cannot load', async () => { + await expect(loadSkills(['wrdn-authz'], undefined)).rejects.toThrow('wrdn-authz needs a getsentry/warden-skills checkout'); + await expect(loadSkills(['wrdn-pii'], worktree)).rejects.toThrow( + `Could not load wrdn-pii from ${join(worktree, 'skills', 'wrdn-pii')}`, + ); + }); + + it('refuses a directory whose skill has another name', async () => { + writeSkill(worktree, 'wrdn-pii', 'wrdn-other'); + await expect(loadSkills(['wrdn-pii'], worktree)).rejects.toThrow('loaded wrdn-other'); + }); +}); diff --git a/packages/security-evals/src/shard.ts b/packages/security-evals/src/shard.ts index 62d5a57f4..7e237f712 100644 --- a/packages/security-evals/src/shard.ts +++ b/packages/security-evals/src/shard.ts @@ -3,8 +3,7 @@ import { resolveSkillAsync, runSkill, type SkillReport, type UsageStats } from ' // Not on the public surface; the CLI's file mode builds its context the same way. import { buildFileEventContext } from '../../warden/src/cli/context.js'; import type { SuiteShard } from './entries.js'; -import type { SuiteOptions } from './options.js'; -import { WARDEN_SKILLS_REMOTE } from './options.js'; +import { WARDEN_SKILLS_REF, type SuiteOptions } from './options.js'; import type { LaneConfig, LaneUsage, ShardRun } from './results.js'; import { scoreEntry } from './score.js'; @@ -16,17 +15,31 @@ const BUILTIN_SECURITY_REVIEW = join(import.meta.dirname, '..', '..', 'warden', const SCAN = { maxFiles: 200, maxChangedLines: 50_000 }; /** - * Load each skill once. security-review comes from this checkout by path, so a - * same-named skill in the scanned repo cannot shadow it. wrdn-* skills come - * from the pinned warden-skills ref. + * Load each skill once, from disk only. security-review comes from this + * checkout by path, so a same-named skill in the scanned repo cannot shadow + * it. wrdn-* skills come from a worktree of warden-skills at the pinned ref. + * Any failure throws once, before a pass runs, instead of reading as misses. */ -export async function loadSkills(names: string[]): Promise> { +export async function loadSkills(names: string[], skillsWorktree: string | undefined): Promise> { const skills = new Map(); - // Sequential: remote skills share one checkout of the pinned ref. for (const name of names) { - skills.set(name, name === 'security-review' - ? await resolveSkillAsync(BUILTIN_SECURITY_REVIEW) - : await resolveSkillAsync(name, undefined, { remote: WARDEN_SKILLS_REMOTE })); + let dir = BUILTIN_SECURITY_REVIEW; + if (name !== 'security-review') { + if (!skillsWorktree) { + throw new Error(`${name} needs a getsentry/warden-skills checkout; set --skills-repo`); + } + dir = join(skillsWorktree, 'skills', name); + } + try { + const skill = await resolveSkillAsync(dir); + if (skill.name !== name) { + throw new Error(`loaded ${skill.name}`); + } + skills.set(name, skill); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Could not load ${name} from ${dir}: ${message}`); + } } return skills; } @@ -41,7 +54,7 @@ function laneConfig(skill: string, options: SuiteOptions): LaneConfig { auxiliaryEffort: options.auxiliaryEffort, maxTurns: options.maxTurns, judgeModel: options.judgeModel, - ...(skill === 'security-review' ? {} : { skillsRemote: WARDEN_SKILLS_REMOTE }), + ...(skill === 'security-review' ? {} : { skillsRef: `getsentry/warden-skills@${WARDEN_SKILLS_REF}` }), }; } diff --git a/packages/security-evals/src/worktrees.test.ts b/packages/security-evals/src/worktrees.test.ts index c3d975ed7..1a872fe79 100644 --- a/packages/security-evals/src/worktrees.test.ts +++ b/packages/security-evals/src/worktrees.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'no import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { ensureWorktree, removeWorktree, resolveSentryRepo } from './worktrees.js'; +import { SENTRY_CHECKOUT, SKILLS_CHECKOUT, ensureWorktree, removeWorktree, resolveCheckout } from './worktrees.js'; function configuredEmail(): string { try { @@ -66,6 +66,15 @@ describe('ensureWorktree', () => { expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v2'); }); + it('checks out a blobless clone, fetching blobs on demand', () => { + const ref = commit(upstream, 'v1'); + git(['config', 'uploadpack.allowFilter', 'true'], upstream); + git(['clone', '-q', '--filter=blob:none', '--no-checkout', `file://${upstream}`, clone], root); + + const path = ensureWorktree(clone, join(root, 'worktrees'), ref); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v1'); + }); + it('refuses a worktree that moved off its ref', () => { const first = commit(upstream, 'v1'); const second = commit(upstream, 'v2'); @@ -78,9 +87,10 @@ describe('ensureWorktree', () => { }); }); -describe('resolveSentryRepo', () => { +describe('resolveCheckout', () => { it('fails fast without a clone instead of cloning', () => { - expect(() => resolveSentryRepo(undefined)).toThrow('WARDEN_SECURITY_EVALS_SENTRY_REPO'); - expect(() => resolveSentryRepo(tmpdir())).toThrow('is not a git checkout'); + expect(() => resolveCheckout(SENTRY_CHECKOUT, undefined)).toThrow('WARDEN_SECURITY_EVALS_SENTRY_REPO (or --sentry-repo)'); + expect(() => resolveCheckout(SKILLS_CHECKOUT, undefined)).toThrow('getsentry/warden-skills clone'); + expect(() => resolveCheckout(SENTRY_CHECKOUT, tmpdir())).toThrow('is not a git checkout'); }); }); diff --git a/packages/security-evals/src/worktrees.ts b/packages/security-evals/src/worktrees.ts index 677b32705..2c08ba168 100644 --- a/packages/security-evals/src/worktrees.ts +++ b/packages/security-evals/src/worktrees.ts @@ -2,41 +2,59 @@ import { execFileSync } from 'node:child_process'; import { existsSync, mkdirSync } from 'node:fs'; import { join, resolve } from 'node:path'; -/** Env var that points the suite at a local getsentry/sentry clone. */ -const SENTRY_REPO_ENV = 'WARDEN_SECURITY_EVALS_SENTRY_REPO'; +/** A repository the suite reads from a local clone, never clones itself. */ +export interface CheckoutSpec { + repository: string; + /** Env var that names the clone; the CLI flag maps onto it. */ + env: string; + flag: string; +} + +export const SENTRY_CHECKOUT: CheckoutSpec = { + repository: 'getsentry/sentry', + env: 'WARDEN_SECURITY_EVALS_SENTRY_REPO', + flag: '--sentry-repo', +}; + +export const SKILLS_CHECKOUT: CheckoutSpec = { + repository: 'getsentry/warden-skills', + env: 'WARDEN_SECURITY_EVALS_SKILLS_REPO', + flag: '--skills-repo', +}; + +// Roach hands its proxy variables to test workers and refuses origins other +// than the model APIs, so git goes direct. +const PROXY_VARS = new Set(['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY', 'all_proxy']); -function git(args: string[], cwd: string): string { - return execFileSync('git', args, { - cwd, - encoding: 'utf8', - // A partial clone would otherwise fetch a missing object on a presence check. - env: { ...process.env, GIT_NO_LAZY_FETCH: '1' }, - stdio: ['ignore', 'pipe', 'pipe'], - }).trim(); +function git(args: string[], cwd: string, extraEnv: NodeJS.ProcessEnv = {}): string { + const env = Object.fromEntries(Object.entries({ ...process.env, ...extraEnv }) + .filter(([name]) => !PROXY_VARS.has(name))); + return execFileSync('git', args, { cwd, encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'pipe'] }).trim(); } function hasCommit(repoPath: string, ref: string): boolean { try { - git(['cat-file', '-e', `${ref}^{commit}`], repoPath); + // Without this, a blobless clone would fetch a missing commit during the check. + git(['cat-file', '-e', `${ref}^{commit}`], repoPath, { GIT_NO_LAZY_FETCH: '1' }); return true; } catch { return false; } } -/** Resolve the clone the suite builds worktrees from. Never clones on its own. */ -export function resolveSentryRepo(path: string | undefined): string { +/** Resolve a local clone the suite builds worktrees from. Never clones on its own. */ +export function resolveCheckout(spec: CheckoutSpec, path: string | undefined): string { if (!path) { throw new Error( - `${SENTRY_REPO_ENV} (or --sentry-repo) must point at a local getsentry/sentry clone. ` - + 'A blobless clone is enough: git clone --filter=blob:none https://github.com/getsentry/sentry.git', + `${spec.env} (or ${spec.flag}) must point at a local ${spec.repository} clone. ` + + `A blobless clone is enough: git clone --filter=blob:none https://github.com/${spec.repository}.git`, ); } const repoPath = resolve(path); try { return git(['rev-parse', '--show-toplevel'], repoPath); } catch { - throw new Error(`${repoPath} is not a git checkout. Point ${SENTRY_REPO_ENV} at a getsentry/sentry clone.`); + throw new Error(`${repoPath} is not a git checkout. Point ${spec.env} at a ${spec.repository} clone.`); } } From a5c06770e7f91ed61f1fe4c5e9ccf19e8f243ff4 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:21:01 +0000 Subject: [PATCH 05/11] feat(security-evals): Make cases cheap to add and verdicts hard to fake Add `pnpm evals:security:add --skill `, which writes a bug case at the fix's parent and its clean twin at the fix, with targets inferred from the non-test .py/.ts files the fix modified. Add a suite validator to the unit tests: schema, unique ids, no corpus overlap, known skill directory, no TODO summary, a clean twin or noClean reason, and baseline entries that still exist at the same ref. Each wrdn-* lane now gets one verdict (earns-keep, no-unique-catches, insufficient-evidence) from VERDICT_RULES. Failed passes, chunks, and judge calls count as errors, never misses, and block the verdict. Baseline lanes record provenance (model, effort, warden version, skill ref) and are dropped with a warning when it differs from the run. The old imported benchmark lane could never match, so the baseline starts empty; baseline:update folds in a clean run. Replace --sentry-repo with repeatable --repo owner/name=, keeping the old flag as an alias. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .github/workflows/security-evals.yml | 2 +- package.json | 1 + packages/security-evals/README.md | 112 +- packages/security-evals/baseline/results.json | 1294 +---------------- .../sentry-scim-restriction-kept.json | 10 + ...kflow-put-drops-inaccessible-projects.json | 3 +- ...rvice-account-all-projects-team-write.json | 3 +- ...-service-account-org-projects-listing.json | 3 +- ...ternal-feature-flags-config-injection.json | 2 +- ...ernal-feature-flags-restricted-values.json | 2 +- .../sentry-system-options-logs-redacted.json | 2 +- ...try-system-options-logs-secret-values.json | 2 +- .../sentry-uptime-robots-txt-safe-fetch.json | 2 +- .../sentry-uptime-robots-txt-ssrf.json | 2 +- .../sentry-eap-item-encode-depth-limited.json | 2 +- ...y-eap-item-encode-unbounded-recursion.json | 2 +- ...ry-eap-search-boolean-nesting-bounded.json | 10 + ...-eap-search-boolean-nesting-recursion.json | 2 +- ...entry-grouping-url-regex-backtracking.json | 2 +- ...try-grouping-url-regex-bounded-scheme.json | 2 +- packages/security-evals/package.json | 3 +- packages/security-evals/scripts/add-case.ts | 46 + .../scripts/import-benchmark.ts | 19 - packages/security-evals/scripts/run.ts | 24 +- .../security-evals/scripts/update-baseline.ts | 19 + packages/security-evals/src/baseline.test.ts | 162 ++- packages/security-evals/src/baseline.ts | 190 ++- packages/security-evals/src/entries.test.ts | 20 +- packages/security-evals/src/entries.ts | 71 +- packages/security-evals/src/options.test.ts | 21 +- packages/security-evals/src/options.ts | 63 +- packages/security-evals/src/results.test.ts | 113 +- packages/security-evals/src/results.ts | 126 +- packages/security-evals/src/scaffold.test.ts | 138 ++ packages/security-evals/src/scaffold.ts | 183 +++ packages/security-evals/src/security.eval.ts | 38 +- packages/security-evals/src/shard.test.ts | 91 +- packages/security-evals/src/shard.ts | 92 +- packages/security-evals/src/testing.ts | 53 + packages/security-evals/src/validate.test.ts | 118 ++ packages/security-evals/src/validate.ts | 80 + packages/security-evals/src/verdict.test.ts | 107 ++ packages/security-evals/src/verdict.ts | 93 ++ packages/security-evals/src/worktrees.test.ts | 7 +- packages/security-evals/src/worktrees.ts | 9 +- 45 files changed, 1629 insertions(+), 1717 deletions(-) create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json create mode 100644 packages/security-evals/scripts/add-case.ts delete mode 100644 packages/security-evals/scripts/import-benchmark.ts create mode 100644 packages/security-evals/scripts/update-baseline.ts create mode 100644 packages/security-evals/src/scaffold.test.ts create mode 100644 packages/security-evals/src/scaffold.ts create mode 100644 packages/security-evals/src/testing.ts create mode 100644 packages/security-evals/src/validate.test.ts create mode 100644 packages/security-evals/src/validate.ts create mode 100644 packages/security-evals/src/verdict.test.ts create mode 100644 packages/security-evals/src/verdict.ts diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml index 857e9ebdc..6ed0ece3b 100644 --- a/.github/workflows/security-evals.yml +++ b/.github/workflows/security-evals.yml @@ -75,7 +75,7 @@ jobs: env: WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} - WARDEN_SECURITY_EVALS_SENTRY_REPO: ${{ env.SENTRY_CLONE }} + WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }} WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }} WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} diff --git a/package.json b/package.json index 531b3d887..1ce5348a2 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "evals": "pnpm -C packages/evals evals", "evals:scaffold": "pnpm -C packages/evals scaffold", "evals:security": "pnpm -C packages/security-evals evals:security", + "evals:security:add": "pnpm -C packages/security-evals add-case", "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck && pnpm -C packages/security-evals typecheck", "generate:jsonl-schema": "pnpm --filter @sentry/warden generate:jsonl-schema", "docs": "pnpm --filter warden-docs dev", diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index 83d007250..925acd6ac 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -24,8 +24,8 @@ classes, and permission checks the way they do in production. ## Running it -You need local clones of getsentry/sentry and getsentry/warden-skills. Blobless -is enough: +You need a local clone of every repository the selected entries pin (today +only getsentry/sentry) and of getsentry/warden-skills. Blobless is enough: ```bash git clone --filter=blob:none https://github.com/getsentry/sentry.git ~/src/sentry @@ -34,10 +34,10 @@ git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills ```bash # Every lane over every entry -pnpm evals:security --sentry-repo ~/src/sentry --skills-repo ~/src/warden-skills +pnpm evals:security --repo getsentry/sentry=~/src/sentry --skills-repo ~/src/warden-skills # One candidate lane on its own cases, three passes; other lanes come from the baseline -pnpm evals:security --sentry-repo ~/src/sentry --skills-repo ~/src/warden-skills \ +pnpm evals:security --repo getsentry/sentry=~/src/sentry --skills-repo ~/src/warden-skills \ --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3 ``` @@ -46,7 +46,9 @@ pnpm evals:security --sentry-repo ~/src/sentry --skills-repo ~/src/warden-skills runtime, `openrouter/x-ai/grok-4.5` at high effort, and `openrouter/openai/gpt-5.6-luna` at high effort for auxiliary work. `maxTurns` defaults to 100, as in the corpus runbook, so a turn cap does not read as a -miss. `--skills-repo` is only needed when a `wrdn-*` lane is selected. +miss. `--skills-repo` is only needed when a `wrdn-*` lane is selected. Repeat +`--repo owner/name=` per repository; `--sentry-repo ` is shorthand +for the getsentry/sentry one. Each pinned ref gets `git fetch origin ` only if its clone lacks it, then one `git worktree add --detach` under `.cache/`. The warden-skills worktree @@ -69,8 +71,11 @@ over the shard's targets and runs every selected skill on it. Per lane and entry, only findings in the entry's target files reach the judge, which applies the corpus rule: same bug, roughly the same location. Another bug in the same file does not count. No findings in the targets is a miss with no -judge call. A failed chunk drops the entry from that repeat instead of -counting a miss. +judge call. + +A failed skill pass, failed chunk, or failed judge call records the entry as +an **error** for that lane, never a miss. Any error fails the run and blocks +that skill's verdict. Model calls go through Roach like `packages/evals`. Worktree content is pinned, so recordings replay; `roach.json` normalizes the worktree, checkout, and @@ -81,34 +86,89 @@ skill cache paths. `.cache/results.json` has sorted keys and no timestamps, so two runs diff cleanly. The console and the job summary show: -- per lane: recall, bugs found, bugs only that lane found, bugs it found that - security-review missed, clean cases flagged, cost, and tokens -- an entry x lane table of hits over repeats +- one verdict per `wrdn-*` lane, first +- per lane: recall, bugs found, bugs only that lane found, clean cases + flagged, errors, cost, and tokens +- an entry x lane table of hits over repeats, with error counts + +Single runs are noisy: two production-config security-review corpus runs +agreed on 32 of the 46 entries either one found. That is why the verdict +wants repeats. + +## Proving a skill + +Every `wrdn-*` lane gets exactly one verdict against security-review. The +thresholds live in `VERDICT_RULES` in `src/verdict.ts`. + +| verdict | when | +| --- | --- | +| `earns-keep` | At least one bug the skill caught in 2 of 3 repeats (or better) that security-review caught in none, both lanes at 3+ repeats. | +| `no-unique-catches` | No such bug, nothing unsettled, and at least 5 bug cases under `cases//` with 3+ repeats in both lanes. | +| `insufficient-evidence` | Anything else: an errored entry in either lane, no security-review lane, fewer than 5 owned bug cases, fewer than 3 repeats, or a bug the skill caught that security-review caught only some of the time. | + +Corpus entries can produce an `earns-keep`, but only owned cases count toward +the 5. `no-unique-catches` across a real case set is the evidence for +dropping a skill. A bare zero from one pass is not. + +## Adding a case -`missedBySecurityReview` is the number that decides whether a skill earns its -keep. Zero across a real fixture set is the case for dropping it. +A good case is a real bug that was fixed, squarely in the owning skill's +domain, and traceable inside the repository at that ref. Synthetic code and +bugs that need another service to explain do not belong here. -Single runs are noisy: two production-config security-review corpus runs agreed on 32 -of the 46 entries either one found. Use `--repeats 3` or more before deciding. +```bash +pnpm evals:security:add https://github.com/getsentry/sentry/pull/ --skill wrdn-dos-review +``` + +It writes `cases//.json` (the bug, at the fix commit's parent) +and `-fixed.json` (the clean twin, at the fix). Targets are the non-test +`.py` and `.ts` files the fix modified; it prints anything it skipped. A +commit URL works too. The name comes from the PR title; `--name ` picks +your own. `GITHUB_TOKEN` avoids rate limits. + +Then: + +1. Write `summary` in both files: one sentence naming the bug and where it + lives. The judge matches findings against it. +2. `pnpm -C packages/security-evals test`. The suite validator checks the + schema, unique ids, no overlap with the corpus, a known skill directory, + no TODO summary, a clean twin or a `noClean` reason, and that the baseline + only names entries that exist. It runs in CI with no model and no clone. +3. Run it for real: + `pnpm evals:security --repo getsentry/sentry= --skills-repo --skills security-review, --entries /,/-fixed --repeats 3`. + A case no lane ever finds is still a valid case; say so in the PR. + +Drop the clean twin only when the fix is not pinnable, and say why in +`noClean` on the bug case. ## Baseline -`baseline/results.json` holds lanes this run does not execute, so a -comparison only pays for the candidate lane. It currently has security-review -over the corpus, imported from the two published production-config benchmark -runs (warden 0.46.2 and 0.47.0, scored by the benchmark reviewer). It has no -case results. Pass `--baseline none` to skip it. +`baseline/results.json` holds lanes a run does not execute, so a comparison +only pays for the candidate lane. Each lane records its provenance: runtime, +model, effort, auxiliary model and effort, max turns, judge model, warden +version, and skill ref (`builtin` or `getsentry/warden-skills@`). A lane +whose provenance differs from the current run is dropped with a warning, and +an entry whose ref changed is skipped. A stale baseline never gets compared +against a new skill version. Pass `--baseline none` to skip it. -Regenerate it from scored benchmark results: +It is empty right now. Fill it after a clean run: ```bash -pnpm -C packages/security-evals import-benchmark \ - ../docs/src/data/benchmarking/results/.json ... +pnpm evals:security ... --skills security-review --repeats 3 --output /tmp/sr.json +pnpm -C packages/security-evals baseline:update /tmp/sr.json ``` +`baseline:update` replaces the tallies for entries the run covered, resets a +lane whose provenance changed, and refuses results with errors. Update it when +you bump the warden version, the model, or the warden-skills pin, and after +adding cases you want compared for free. + ## Known gaps -- `wrdn-code-execution` has one real bug case, plus corpus entries 060 and - 061. `wrdn-data-exfil` has two, plus corpus 052. The corpus has almost no - DoS or exfiltration entries. -- Cases are getsentry/sentry only. Other repositories need their own clone flag. +- Owned bug cases today: wrdn-authz 4, wrdn-dos-review 3, wrdn-data-exfil 2, + wrdn-code-execution 1, wrdn-pii 0. Every skill is below 5, so every lane + reads `insufficient-evidence` unless it earns its keep outright. +- The corpus has almost no DoS or exfiltration entries. Cases have to carry + those skills. +- Next: harvest candidate cases from production Warden PR comments that led + to a fix. Not built. diff --git a/packages/security-evals/baseline/results.json b/packages/security-evals/baseline/results.json index 86202631d..9341710a0 100644 --- a/packages/security-evals/baseline/results.json +++ b/packages/security-evals/baseline/results.json @@ -1,1292 +1,6 @@ { - "entries": { - "sentry-vuln-001": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "788ba30f1aa42b00c02d64ed4b8b2515ff8ab8da", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/accept_organization_invite.py" - ] - }, - "sentry-vuln-002": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/workflow_engine/endpoints/organization_alertrule_detector_index.py" - ] - }, - "sentry-vuln-003": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/internal_ea_features.py" - ] - }, - "sentry-vuln-004": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/oauth_userinfo.py" - ] - }, - "sentry-vuln-005": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/oauth_userinfo.py" - ] - }, - "sentry-vuln-006": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/organization_plugin_deprecation_info.py" - ] - }, - "sentry-vuln-007": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/release_thresholds/release_threshold_index.py" - ] - }, - "sentry-vuln-008": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "2d929588e20b931d2631ade31405f6ddbd34603a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/system_options.py" - ] - }, - "sentry-vuln-009": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "fbecd17d210c2eacfad4bf17872be62320b8a110", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/organization_events_root_cause_analysis.py" - ] - }, - "sentry-vuln-010": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "fbecd17d210c2eacfad4bf17872be62320b8a110", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/system_options.py" - ] - }, - "sentry-vuln-011": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/jira/webhooks/installed.py" - ] - }, - "sentry-vuln-012": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/msteams/webhook.py" - ] - }, - "sentry-vuln-013": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/perforce/integration.py" - ] - }, - "sentry-vuln-014": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/slack/webhooks/options_load.py" - ] - }, - "sentry-vuln-015": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/replays/endpoints/data_export_notifications.py" - ] - }, - "sentry-vuln-016": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/replays/endpoints/project_replay_details.py" - ] - }, - "sentry-vuln-017": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/preprod/api/endpoints/preprod_artifact_snapshot.py" - ] - }, - "sentry-vuln-018": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/organization_pinned_searches.py" - ] - }, - "sentry-vuln-019": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/auth/services/access/impl.py" - ] - }, - "sentry-vuln-020": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/users/models/user_merge_verification_code.py" - ] - }, - "sentry-vuln-021": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/codecov/endpoints/repository_token_regenerate/repository_token_regenerate.py" - ] - }, - "sentry-vuln-022": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/explore/endpoints/explore_saved_query_detail.py" - ] - }, - "sentry-vuln-023": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/flags/endpoints/secrets.py" - ] - }, - "sentry-vuln-024": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/project_rules.py" - ] - }, - "sentry-vuln-025": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/api/endpoints/external_team_details.py" - ] - }, - "sentry-vuln-026": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/messaging/linkage.py" - ] - }, - "sentry-vuln-027": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/slack/webhooks/action.py" - ] - }, - "sentry-vuln-028": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/accept_organization_invite.py" - ] - }, - "sentry-vuln-029": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/notifications/notification_action/group_type_notification_registry/handlers/metric_alert_registry_handler.py" - ] - }, - "sentry-vuln-030": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/preprod/api/endpoints/preprod_artifact_approve.py" - ] - }, - "sentry-vuln-031": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/preprod/api/endpoints/public/organization_preprod_size_analysis.py" - ] - }, - "sentry-vuln-032": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/projects/services/project_key/impl.py" - ] - }, - "sentry-vuln-033": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/replays/usecases/replay_counts.py" - ] - }, - "sentry-vuln-034": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/seer/autofix/autofix_agent.py" - ] - }, - "sentry-vuln-035": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/seer/endpoints/organization_autofix_automation_settings.py" - ] - }, - "sentry-vuln-036": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/seer/supergroups/endpoints/organization_supergroups_by_group.py" - ] - }, - "sentry-vuln-037": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/sentry_apps/services/cell/impl.py" - ] - }, - "sentry-vuln-038": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/sentry_apps/api/endpoints/installation_external_issue_actions.py" - ] - }, - "sentry-vuln-039": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/workflow_engine/endpoints/organization_open_periods.py" - ] - }, - "sentry-vuln-040": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/workflow_engine/endpoints/validators/utils.py" - ] - }, - "sentry-vuln-041": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/discover/endpoints/discover_saved_query_detail.py" - ] - }, - "sentry-vuln-042": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/integrations/slack/webhooks/action.py" - ] - }, - "sentry-vuln-043": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "9fc06de579fbbee810565304efcc982e5dea1d1a", - "source": "corpus", - "targets": [ - "src/sentry/notifications/platform/api/endpoints/internal_registered_templates.py" - ] - }, - "sentry-vuln-044": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "6993246259d0193a6a6aac49aa3763959b1126df", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/auth_index.py" - ] - }, - "sentry-vuln-045": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/seer/endpoints/organization_seer_rpc.py" - ] - }, - "sentry-vuln-046": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/preprod/api/endpoints/pull_request/organization_pullrequest_size_analysis_download.py" - ] - }, - "sentry-vuln-047": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/project_rule_details.py" - ] - }, - "sentry-vuln-048": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/preprod/api/endpoints/size_analysis/project_preprod_size_analysis_compare_download.py" - ] - }, - "sentry-vuln-049": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/api/endpoints/organization_pipeline.py" - ] - }, - "sentry-vuln-050": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/api/serializers/models/projectcodeowners.py" - ] - }, - "sentry-vuln-051": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/bitbucket_server/urls.py" - ] - }, - "sentry-vuln-052": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/models/data_forwarder.py" - ] - }, - "sentry-vuln-053": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/sentry_apps/api/endpoints/installation_external_issue_actions.py" - ] - }, - "sentry-vuln-054": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/relocation/api/endpoints/retry.py" - ] - }, - "sentry-vuln-055": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/seer/endpoints/organization_seer_rpc.py" - ] - }, - "sentry-vuln-056": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/api/serializers/models/apiapplication.py" - ] - }, - "sentry-vuln-057": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/tasks/post_process.py" - ] - }, - "sentry-vuln-058": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/workflow_engine/endpoints/organization_detector_anomaly_data.py" - ] - }, - "sentry-vuln-059": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/workflow_engine/endpoints/organization_alertrule_detector_index.py" - ] - }, - "sentry-vuln-060": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - ".github/workflows/getsentry-dispatch.yml" - ] - }, - "sentry-vuln-061": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "scripts/analyze-styled.ts" - ] - }, - "sentry-vuln-062": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/api/authentication.py" - ] - }, - "sentry-vuln-063": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/flags/providers.py" - ] - }, - "sentry-vuln-064": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/bitbucket/webhook.py" - ] - }, - "sentry-vuln-065": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/discord/requests/base.py" - ] - }, - "sentry-vuln-066": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/discord/requests/base.py" - ] - }, - "sentry-vuln-067": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/github_enterprise/webhook.py" - ] - }, - "sentry-vuln-068": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/github_enterprise/webhook.py" - ] - }, - "sentry-vuln-069": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/utils/atlassian_connect.py" - ] - }, - "sentry-vuln-070": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/jira_server/webhooks.py" - ] - }, - "sentry-vuln-071": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/integrations/utils/atlassian_connect.py" - ] - }, - "sentry-vuln-072": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/models/apiapplication.py" - ] - }, - "sentry-vuln-073": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/releases/endpoints/project_releases_token.py" - ] - }, - "sentry-vuln-074": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/seer/endpoints/seer_rpc.py" - ] - }, - "sentry-vuln-075": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/web/frontend/mailgun_inbound_webhook.py" - ] - }, - "sentry-vuln-076": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry/web/frontend/release_webhook.py" - ] - }, - "sentry-vuln-077": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry_plugins/heroku/plugin.py" - ] - }, - "sentry-vuln-078": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry_plugins/jira/client.py" - ] - }, - "sentry-vuln-079": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "src/sentry_plugins/jira/client.py" - ] - }, - "sentry-vuln-080": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/components/events/autofix/insights/autofixInsightSources.tsx" - ] - }, - "sentry-vuln-081": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/components/events/interfaces/utils.tsx" - ] - }, - "sentry-vuln-082": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 2, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/components/featureFlags/hooks/useFlagSeries.tsx" - ] - }, - "sentry-vuln-083": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/views/discover/utils.tsx" - ] - }, - "sentry-vuln-084": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/views/explore/releases/list/mobileBuildsChart.tsx" - ] - }, - "sentry-vuln-085": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 1, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/views/preprod/components/visualizations/appSizeTreemap.tsx" - ] - }, - "sentry-vuln-086": { - "kind": "bug", - "lanes": { - "security-review": { - "hits": 0, - "runs": 2 - } - }, - "ref": "7f41cc502faa1088108e896565409be72d92bc38", - "source": "corpus", - "targets": [ - "static/app/views/settings/organizationIntegrations/integrationDetailedView.tsx" - ] - } - }, - "lanes": { - "security-review": { - "config": { - "auxiliaryEffort": "high", - "auxiliaryModel": "openrouter/openai/gpt-5.6-luna", - "effort": "high", - "judgeModel": "agent-semantic-match-pass", - "maxTurns": 100, - "model": "openrouter/x-ai/grok-4.5", - "origin": "sentry-security-review-pi-openrouter-grok-4-5-high-luna-aux-high-traced-corpus-2026-08-21-001, sentry-security-review-pi-openrouter-grok-4-5-high-luna-aux-high-traced-corpus-2026-08-23-001", - "runtime": "pi" - }, - "failedRuns": 0, - "runs": 2, - "usage": { - "cacheReadInputTokens": 112996572, - "costUSD": 64.13501639, - "inputTokens": 137321818, - "judgeCostUSD": 0, - "outputTokens": 4385065 - } - } - }, - "summary": { - "security-review": { - "bugEntries": 86, - "cleanFlagged": { - "hits": 0, - "runs": 0 - }, - "found": 46, - "missedBySecurityReview": [], - "onlyThisLane": [ - "sentry-vuln-001", - "sentry-vuln-002", - "sentry-vuln-003", - "sentry-vuln-004", - "sentry-vuln-007", - "sentry-vuln-011", - "sentry-vuln-012", - "sentry-vuln-013", - "sentry-vuln-014", - "sentry-vuln-015", - "sentry-vuln-017", - "sentry-vuln-018", - "sentry-vuln-019", - "sentry-vuln-021", - "sentry-vuln-025", - "sentry-vuln-026", - "sentry-vuln-029", - "sentry-vuln-030", - "sentry-vuln-031", - "sentry-vuln-034", - "sentry-vuln-035", - "sentry-vuln-036", - "sentry-vuln-037", - "sentry-vuln-039", - "sentry-vuln-041", - "sentry-vuln-046", - "sentry-vuln-047", - "sentry-vuln-048", - "sentry-vuln-051", - "sentry-vuln-055", - "sentry-vuln-056", - "sentry-vuln-057", - "sentry-vuln-058", - "sentry-vuln-062", - "sentry-vuln-065", - "sentry-vuln-067", - "sentry-vuln-069", - "sentry-vuln-070", - "sentry-vuln-071", - "sentry-vuln-075", - "sentry-vuln-077", - "sentry-vuln-079", - "sentry-vuln-081", - "sentry-vuln-082", - "sentry-vuln-084", - "sentry-vuln-085" - ], - "recall": 0.45348837209302323 - } - }, - "version": 1 + "entries": {}, + "lanes": {}, + "summary": {}, + "version": 2 } diff --git a/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json new file mode 100644 index 000000000..59ea22355 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams", + "repository": "getsentry/sentry", + "ref": "c347ba8739618e24da2e590ecfa3d7fc989866ca", + "targets": [ + "src/sentry/core/endpoints/scim/utils.py" + ], + "source": "https://github.com/getsentry/sentry/pull/130141" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json index 477e7d461..cea0944ef 100644 --- a/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json +++ b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json @@ -6,5 +6,6 @@ "targets": [ "src/sentry/seer/endpoints/organization_seer_workflow_configs.py" ], - "source": "https://github.com/getsentry/sentry/pull/129734" + "source": "https://github.com/getsentry/sentry/pull/129734", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." } diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json index 7f99548dc..504ea6525 100644 --- a/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json @@ -7,5 +7,6 @@ "src/sentry/auth/access.py", "src/sentry/core/endpoints/organization_member_team_details.py" ], - "source": "https://github.com/getsentry/sentry/pull/129343" + "source": "https://github.com/getsentry/sentry/pull/129343", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." } diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json index 403a37169..31cbe0768 100644 --- a/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json @@ -6,5 +6,6 @@ "targets": [ "src/sentry/core/endpoints/organization_projects.py" ], - "source": "https://github.com/getsentry/sentry/pull/129154" + "source": "https://github.com/getsentry/sentry/pull/129154", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." } diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json index 7856e8855..a20a8a13d 100644 --- a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/api/endpoints/internal/feature_flags.py" ], - "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (pre-fix parent, getsentry/sentry#112980)" + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866" } diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json index d1bbfdf9a..7db3ff210 100644 --- a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/api/endpoints/internal/feature_flags.py" ], - "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866 (fixed state)" + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866" } diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json index 849e9a964..3f82b787e 100644 --- a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/api/endpoints/system_options.py" ], - "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (fixed state)" + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441" } diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json index 779ffd320..675a21612 100644 --- a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/api/endpoints/system_options.py" ], - "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441 (pre-fix parent, getsentry/sentry#114187)" + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441" } diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json index cd82ec818..2b3a9a104 100644 --- a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/uptime/autodetect/tasks.py" ], - "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (fixed state)" + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b" } diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json index 0d5b356f3..dedd5e604 100644 --- a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/uptime/autodetect/tasks.py" ], - "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b (pre-fix parent, getsentry/sentry#118358)" + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b" } diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json index 43a000e6c..811792d98 100644 --- a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/eventstream/item_helpers.py" ], - "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (fixed state)" + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e" } diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json index 4ed0e4c6e..7b9f9d2eb 100644 --- a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/eventstream/item_helpers.py" ], - "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e (pre-fix parent, getsentry/sentry#108555)" + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e" } diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json new file mode 100644 index 000000000..55da31606 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request", + "repository": "getsentry/sentry", + "ref": "93b67135ae9cee140be76f43a8d4ba6bc58bf080", + "targets": [ + "src/sentry/search/eap/resolver.py" + ], + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json index dfc7daeaa..55e884761 100644 --- a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/search/eap/resolver.py" ], - "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080 (pre-fix parent, getsentry/sentry#128180)" + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080" } diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json index fe24b56d3..3702ee7c2 100644 --- a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/grouping/parameterization.py" ], - "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (pre-fix parent, getsentry/sentry#118512)" + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362" } diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json index 33221af46..a31085f73 100644 --- a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json @@ -6,5 +6,5 @@ "targets": [ "src/sentry/grouping/parameterization.py" ], - "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362 (fixed state)" + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362" } diff --git a/packages/security-evals/package.json b/packages/security-evals/package.json index 1d70686b2..b55a6ee2d 100644 --- a/packages/security-evals/package.json +++ b/packages/security-evals/package.json @@ -4,7 +4,8 @@ "type": "module", "scripts": { "evals:security": "tsx scripts/run.ts", - "import-benchmark": "tsx scripts/import-benchmark.ts", + "add-case": "tsx scripts/add-case.ts", + "baseline:update": "tsx scripts/update-baseline.ts", "test": "vitest run --config vitest.unit.config.ts", "typecheck": "tsc --noEmit" }, diff --git a/packages/security-evals/scripts/add-case.ts b/packages/security-evals/scripts/add-case.ts new file mode 100644 index 000000000..f179336bc --- /dev/null +++ b/packages/security-evals/scripts/add-case.ts @@ -0,0 +1,46 @@ +#!/usr/bin/env tsx +import { relative } from 'node:path'; +import { parseArgs } from 'node:util'; +import { createGitHubReader, scaffoldCase } from '../src/scaffold.js'; + +const USAGE = `usage: pnpm evals:security:add --skill [--name ] [--force] + +Writes cases//.json (bug, at the fix's parent) and +cases//-fixed.json (clean, at the fix). Targets are the non-test +.py/.ts files the fix modified.`; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + skill: { type: 'string' }, + name: { type: 'string' }, + force: { type: 'boolean' }, + help: { type: 'boolean', short: 'h' }, + }, +}); +const [url] = positionals; +if (values.help || !url || !values.skill || positionals.length > 1) { + console.log(USAGE); + process.exit(values.help ? 0 : 2); +} + +const result = await scaffoldCase({ + url, + skill: values.skill, + name: values.name, + force: values.force, + github: createGitHubReader(), +}); +const cwd = process.env['INIT_CWD'] ?? process.cwd(); +console.log(`Bug: ${relative(cwd, result.bugPath)} @ ${result.bugRef}`); +console.log(`Clean: ${relative(cwd, result.cleanPath)} @ ${result.fixRef}`); +console.log(`Targets:\n${result.targets.map((target) => ` ${target}`).join('\n')}`); +if (result.skipped.length > 0) { + console.log(`Skipped:\n${result.skipped.map((file) => ` ${file.path} (${file.reason})`).join('\n')}`); +} +console.log(` +Next: + 1. Replace the TODO summary in both files with the same sentence: the bug, where it lives, what an attacker gets. + 2. pnpm -C packages/security-evals test (free; checks the pair, ids, and summary) + 3. pnpm evals:security --repo ${result.repository}= --skills-repo \\ + --skills security-review,${values.skill} --entries ${result.bugId},${result.cleanId} --repeats 3`); diff --git a/packages/security-evals/scripts/import-benchmark.ts b/packages/security-evals/scripts/import-benchmark.ts deleted file mode 100644 index 271cd9095..000000000 --- a/packages/security-evals/scripts/import-benchmark.ts +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env tsx -import { readFileSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { importBenchmarkResults } from '../src/baseline.js'; -import { loadCorpusEntries } from '../src/entries.js'; -import { DEFAULT_BASELINE } from '../src/options.js'; -import { serializeResults } from '../src/results.js'; - -// usage: pnpm -C packages/security-evals import-benchmark ... -// Writes baseline/results.json from scored corpus benchmark results. -const cwd = process.env['INIT_CWD'] ?? process.cwd(); -const files = process.argv.slice(2).map((path) => JSON.parse(readFileSync(resolve(cwd, path), 'utf8')) as unknown); -if (files.length === 0) { - console.error('usage: import-benchmark ...'); - process.exit(2); -} -const results = importBenchmarkResults(loadCorpusEntries(), files); -writeFileSync(DEFAULT_BASELINE, serializeResults(results)); -console.log(`Wrote ${DEFAULT_BASELINE}`); diff --git a/packages/security-evals/scripts/run.ts b/packages/security-evals/scripts/run.ts index 0eaec5e4a..e6bb533b4 100644 --- a/packages/security-evals/scripts/run.ts +++ b/packages/security-evals/scripts/run.ts @@ -25,9 +25,12 @@ const FLAGS = { const PATH_FLAGS = new Set(['sentry-repo', 'skills-repo', 'cache-dir', 'output', 'baseline']); -const USAGE = `usage: pnpm evals:security --sentry-repo [--skills-repo ] [options] +const USAGE = `usage: pnpm evals:security --repo getsentry/sentry= [--skills-repo ] [options] - --sentry-repo Local getsentry/sentry clone (required) + --repo + Local clone of a repository the selected entries scan. + Repeat it per repository (required) + --sentry-repo Same as --repo getsentry/sentry= --skills-repo Local getsentry/warden-skills clone (required for wrdn-* lanes) --skills Lanes to run (default: security-review and the five wrdn-* skills) --entries Entry ids, or prefixes ending in * (default: all) @@ -46,6 +49,7 @@ const USAGE = `usage: pnpm evals:security --sentry-repo [--skills-repo

[flag, { type: 'string' as const }])), + repo: { type: 'string', multiple: true }, 'remove-worktrees': { type: 'boolean' }, help: { type: 'boolean', short: 'h' }, }, @@ -56,16 +60,22 @@ if (values.help) { process.exit(0); } +// pnpm runs scripts from the package directory; INIT_CWD is where the caller ran it. +const callerPath = (path: string): string => resolve(process.env['INIT_CWD'] ?? process.cwd(), path); + const env: NodeJS.ProcessEnv = { ...process.env }; for (const [flag, name] of Object.entries(FLAGS)) { - const value = (values as Record)[flag]; + const value = (values as Record)[flag]; if (typeof value === 'string') { - // pnpm runs scripts from the package directory; INIT_CWD is where the caller ran it. - env[`WARDEN_SECURITY_EVALS_${name}`] = PATH_FLAGS.has(flag) && value !== 'none' - ? resolve(process.env['INIT_CWD'] ?? process.cwd(), value) - : value; + env[`WARDEN_SECURITY_EVALS_${name}`] = PATH_FLAGS.has(flag) && value !== 'none' ? callerPath(value) : value; } } +if (values.repo) { + env['WARDEN_SECURITY_EVALS_REPOS'] = values.repo.map((pair) => { + const split = pair.indexOf('='); + return split < 0 ? pair : `${pair.slice(0, split)}=${callerPath(pair.slice(split + 1))}`; + }).join(','); +} if (values['remove-worktrees']) { env['WARDEN_SECURITY_EVALS_REMOVE_WORKTREES'] = '1'; } diff --git a/packages/security-evals/scripts/update-baseline.ts b/packages/security-evals/scripts/update-baseline.ts new file mode 100644 index 000000000..2c8c3de0f --- /dev/null +++ b/packages/security-evals/scripts/update-baseline.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env tsx +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { loadResultsFile, updateBaseline } from '../src/baseline.js'; +import { DEFAULT_BASELINE } from '../src/options.js'; +import { serializeResults } from '../src/results.js'; + +// usage: pnpm -C packages/security-evals baseline:update +// Folds the lanes a run executed into baseline/results.json. +const [path] = process.argv.slice(2); +if (!path) { + console.error('usage: baseline:update '); + process.exit(2); +} +const run = loadResultsFile(resolve(process.env['INIT_CWD'] ?? process.cwd(), path)); +const baseline = updateBaseline(loadResultsFile(DEFAULT_BASELINE), run); +writeFileSync(DEFAULT_BASELINE, serializeResults(baseline)); +const lanes = Object.entries(run.lanes).filter(([, lane]) => !lane.fromBaseline).map(([skill]) => skill); +console.log(`Updated ${lanes.join(', ')} in ${DEFAULT_BASELINE}`); diff --git a/packages/security-evals/src/baseline.test.ts b/packages/security-evals/src/baseline.test.ts index d4ba956ff..bd0f05586 100644 --- a/packages/security-evals/src/baseline.test.ts +++ b/packages/security-evals/src/baseline.test.ts @@ -1,56 +1,118 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { importBenchmarkResults } from './baseline.js'; -import type { SuiteEntry } from './entries.js'; - -const runOptions = { - runtime: 'pi', - model: 'openrouter/x-ai/grok-4.5', - effort: 'high', - auxiliaryModel: 'openrouter/openai/gpt-5.6-luna', - auxiliaryEffort: 'high', - maxTurns: 100, -}; - -function benchmark(runId: string, found: string[], overrides: Partial = {}) { - return { - runId, - corpusId: 'sentry-vulnerability-corpus', - skill: 'security-review', - runOptions: { ...runOptions, ...overrides }, - summary: { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 50, costUSD: 30 }, - scoring: { reviewer: 'agent-semantic-match-pass' }, - scores: [ - ...found.map((id) => ({ matchedCorpusIds: [id], verdict: 'known-found' })), - { matchedCorpusIds: [], verdict: 'not-known' }, - ], - }; -} - -function entry(id: string, source: SuiteEntry['source'] = 'corpus'): SuiteEntry { - return { id, source, kind: 'bug', repository: 'getsentry/sentry', ref: 'a', targets: ['x.py'], summary: id }; -} - -describe('importBenchmarkResults', () => { - it('turns each scored benchmark run into one security-review repeat', () => { - const results = importBenchmarkResults( - [entry('sentry-vuln-001'), entry('sentry-vuln-002'), entry('wrdn-authz/case', 'case')], - [benchmark('run-1', ['sentry-vuln-001']), benchmark('run-2', ['sentry-vuln-001', 'sentry-vuln-002'])], - ); - - expect(Object.keys(results.entries)).toEqual(['sentry-vuln-001', 'sentry-vuln-002']); - expect(results.entries['sentry-vuln-001']!.lanes['security-review']).toEqual({ hits: 2, runs: 2 }); - expect(results.entries['sentry-vuln-002']!.lanes['security-review']).toEqual({ hits: 1, runs: 2 }); - expect(results.lanes['security-review']).toMatchObject({ - runs: 2, - config: { model: 'openrouter/x-ai/grok-4.5', origin: 'run-1, run-2' }, - usage: { costUSD: 60 }, +import { loadResultsFile, mergeBaseline, updateBaseline } from './baseline.js'; +import { DEFAULT_BASELINE } from './options.js'; +import { createResults, recordShardRun, serializeResults } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; + +const current = testConfig(); + +describe('mergeBaseline', () => { + it('adds matching lanes the run skipped, limited to the selected entries', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 1, dos: 0 }, 1); + + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'wrdn-authz', { authz: 1 }, 1); + const warnings = mergeBaseline(results, baseline, ['wrdn-authz'], () => current); + + expect(warnings).toEqual([]); + expect(results.lanes['security-review']?.fromBaseline).toBe(true); + expect(results.entries['authz']!.lanes).toEqual({ + 'security-review': { hits: 1, runs: 1, errors: 0 }, + 'wrdn-authz': { hits: 1, runs: 1, errors: 0 }, }); }); - it('refuses runs from different model lanes', () => { - expect(() => importBenchmarkResults([entry('sentry-vuln-001')], [ - benchmark('run-1', []), - benchmark('run-2', [], { effort: 'low' }), - ])).toThrow('share one runtime'); + it('drops a lane recorded with another model, warden version, or skill ref, with a warning', () => { + const baseline = createResults([testEntry('authz')]); + recordRepeats(baseline, 'security-review', { authz: 1 }, 1, testConfig({ wardenVersion: '0.9.0' })); + recordRepeats(baseline, 'wrdn-pii', { authz: 1 }, 1, testConfig({ skillRef: 'getsentry/warden-skills@old' })); + + const results = createResults([testEntry('authz')]); + const warnings = mergeBaseline(results, baseline, [], () => current); + + expect(warnings).toEqual([ + 'Dropped baseline lane security-review: wardenVersion 0.9.0 != 1.0.0', + 'Dropped baseline lane wrdn-pii: skillRef getsentry/warden-skills@old != builtin', + ]); + expect(results.lanes).toEqual({}); + expect(results.entries['authz']!.lanes).toEqual({}); + }); + + it('ignores tallies recorded at a different ref', () => { + const baseline = createResults([testEntry('authz', { ref: 'b'.repeat(40) })]); + recordRepeats(baseline, 'security-review', { authz: 1 }, 1); + + const results = createResults([testEntry('authz')]); + mergeBaseline(results, baseline, [], () => current); + expect(results.entries['authz']!.lanes).toEqual({}); + }); +}); + +describe('updateBaseline', () => { + it('replaces tallies for entries the run covered and keeps the rest', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 0, dos: 1 }, 3); + + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 3 }, 3); + updateBaseline(baseline, run); + + expect(baseline.entries['authz']!.lanes['security-review']).toEqual({ hits: 3, runs: 3, errors: 0 }); + expect(baseline.entries['dos']!.lanes['security-review']).toEqual({ hits: 1, runs: 3, errors: 0 }); + expect(baseline.lanes['security-review']).toMatchObject({ runs: 6 }); + }); + + it('starts a lane over when its config changed', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 0, dos: 1 }, 3, testConfig({ model: 'old' })); + + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 3 }, 3); + updateBaseline(baseline, run); + + expect(baseline.lanes['security-review']).toMatchObject({ runs: 3, config: { model: 'm' } }); + expect(baseline.entries['dos']!.lanes).toEqual({}); + }); + + it('skips lanes the run copied from the baseline', () => { + const baseline = createResults([testEntry('authz')]); + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 1 }, 1); + run.lanes['security-review']!.fromBaseline = true; + expect(updateBaseline(baseline, run).lanes).toEqual({}); + }); + + it('refuses results with errored entries', () => { + const run = createResults([testEntry('authz')]); + recordShardRun(run, { + skill: 'wrdn-authz', config: current, usage: testUsage(), error: 'boom', hits: new Map(), errors: ['authz'], + }); + expect(() => updateBaseline(createResults([]), run)).toThrow('1 errored entry runs'); + }); +}); + +describe('loadResultsFile', () => { + it('round-trips a results file, keeping which lanes came from the baseline', () => { + const dir = mkdtempSync(join(tmpdir(), 'security-evals-results-')); + try { + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'security-review', { authz: 1 }, 1); + results.lanes['security-review']!.fromBaseline = true; + writeFileSync(join(dir, 'results.json'), serializeResults(results)); + + const loaded = loadResultsFile(join(dir, 'results.json')); + expect(loaded.lanes['security-review']?.fromBaseline).toBe(true); + expect(loaded.entries['authz']!.lanes['security-review']).toEqual({ hits: 1, runs: 1, errors: 0 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('loads the committed baseline', () => { + expect(() => loadResultsFile(DEFAULT_BASELINE)).not.toThrow(); }); }); diff --git a/packages/security-evals/src/baseline.ts b/packages/security-evals/src/baseline.ts index 46bdd6564..4778260b5 100644 --- a/packages/security-evals/src/baseline.ts +++ b/packages/security-evals/src/baseline.ts @@ -1,72 +1,140 @@ +import { readFileSync } from 'node:fs'; import { z } from 'zod'; -import type { SuiteEntry } from './entries.js'; -import { createResults, recordShardRun, summarize, type LaneConfig, type SuiteResults } from './results.js'; +import type { LaneConfig } from './options.js'; +import { summarize, type LaneResult, type LaneUsage, type SuiteResults } from './results.js'; -const BenchmarkResultSchema = z.object({ - runId: z.string(), - corpusId: z.literal('sentry-vulnerability-corpus'), - skill: z.literal('security-review'), - runOptions: z.object({ - runtime: z.string(), - model: z.string(), - effort: z.string(), - auxiliaryModel: z.string(), - auxiliaryEffort: z.string(), - maxTurns: z.number(), - }), - summary: z.object({ - inputTokens: z.number(), - outputTokens: z.number(), - cacheReadInputTokens: z.number(), - costUSD: z.number(), - }), - scoring: z.object({ reviewer: z.string() }), - scores: z.array(z.object({ matchedCorpusIds: z.array(z.string()), verdict: z.string() })), +const TallySchema = z.object({ hits: z.number().int(), runs: z.number().int(), errors: z.number().int() }); + +const LaneConfigSchema = z.object({ + runtime: z.string(), + model: z.string(), + effort: z.string(), + auxiliaryModel: z.string(), + auxiliaryEffort: z.string(), + maxTurns: z.number(), + judgeModel: z.string(), + wardenVersion: z.string(), + skillRef: z.string(), +}); + +const UsageSchema = z.object({ + inputTokens: z.number(), + outputTokens: z.number(), + cacheReadInputTokens: z.number(), + costUSD: z.number(), + judgeCostUSD: z.number(), +}); + +/** A results file. Summaries are recomputed on load, so they are not checked. */ +const ResultsFileSchema = z.object({ + version: z.literal(2), + lanes: z.record(z.string(), z.object({ + config: LaneConfigSchema, + usage: UsageSchema, + runs: z.number().int(), + failedRuns: z.number().int(), + fromBaseline: z.literal(true).optional(), + })), + entries: z.record(z.string(), z.object({ + source: z.enum(['corpus', 'case']), + kind: z.enum(['bug', 'clean']), + repository: z.string(), + ref: z.string(), + targets: z.array(z.string()), + lanes: z.record(z.string(), TallySchema), + })), }); +/** Read and check a results file written by the suite. */ +export function loadResultsFile(path: string): SuiteResults { + const parsed = ResultsFileSchema.safeParse(JSON.parse(readFileSync(path, 'utf8'))); + if (!parsed.success) { + throw new Error(`${path}: ${z.prettifyError(parsed.error)}`); + } + const results: SuiteResults = { ...parsed.data, summary: {} }; + results.summary = summarize(results); + return results; +} + +/** Fields where two lane configs differ. Empty means one can stand in for the other. */ +function configMismatch(recorded: LaneConfig, expected: LaneConfig): string[] { + return (Object.keys(expected) as (keyof LaneConfig)[]) + .filter((key) => recorded[key] !== expected[key]) + .map((key) => `${key} ${String(recorded[key])} != ${String(expected[key])}`); +} + /** - * Turn scored corpus benchmark results into a security-review baseline lane, - * one repeat per result file. Every file must share one model lane. + * Copy lanes this run skipped from a baseline, limited to the selected + * entries, so a candidate lane compares without rerunning the rest. A lane + * recorded with a different config is dropped, never compared. Returns one + * warning per dropped lane. */ -export function importBenchmarkResults(entries: SuiteEntry[], files: unknown[]): SuiteResults { - const runs = files.map((file) => BenchmarkResultSchema.parse(file)); - const [first] = runs; - const lanes = new Set(runs.map((run) => JSON.stringify(run.runOptions))); - if (!first || lanes.size !== 1) { - throw new Error('Benchmark results must share one runtime, model, effort, and auxiliary lane'); +export function mergeBaseline( + results: SuiteResults, + baseline: SuiteResults, + ranSkills: string[], + expectedConfig: (skill: string) => LaneConfig, +): string[] { + const warnings: string[] = []; + for (const [skill, lane] of Object.entries(baseline.lanes)) { + if (ranSkills.includes(skill)) continue; + const mismatch = configMismatch(lane.config, expectedConfig(skill)); + if (mismatch.length > 0) { + warnings.push(`Dropped baseline lane ${skill}: ${mismatch.join('; ')}`); + continue; + } + results.lanes[skill] = { ...lane, fromBaseline: true }; + for (const [entryId, entry] of Object.entries(results.entries)) { + const recorded = baseline.entries[entryId]; + const tally = recorded?.lanes[skill]; + if (tally && recorded.ref === entry.ref) entry.lanes[skill] = tally; + } } + return warnings; +} - const corpus = entries.filter((entry) => entry.source === 'corpus'); - const results = createResults(corpus); - const { runOptions } = first; - const config: LaneConfig = { - runtime: runOptions.runtime, - model: runOptions.model, - effort: runOptions.effort, - auxiliaryModel: runOptions.auxiliaryModel, - auxiliaryEffort: runOptions.auxiliaryEffort, - maxTurns: runOptions.maxTurns, - judgeModel: first.scoring.reviewer, - origin: runs.map((run) => run.runId).join(', '), - }; +/** + * Fold the lanes a run executed into the baseline. Entry tallies are replaced + * per entry, so a filtered run only touches what it ran. A lane whose config + * changed is replaced outright. Lane usage and run counts add up across the + * runs that fed it. Refuses results with errored entries. + */ +export function updateBaseline(baseline: SuiteResults, run: SuiteResults): SuiteResults { + const ran = Object.entries(run.lanes).filter(([, lane]) => !lane.fromBaseline); + for (const [skill] of ran) { + const errors = Object.values(run.entries).reduce((sum, entry) => sum + (entry.lanes[skill]?.errors ?? 0), 0); + if (errors > 0) { + throw new Error(`${skill} has ${errors} errored entry runs; rerun them before updating the baseline`); + } + } + + for (const [skill, lane] of ran) { + let previous: LaneResult | undefined = baseline.lanes[skill]; + if (previous && configMismatch(previous.config, lane.config).length > 0) { + for (const entry of Object.values(baseline.entries)) { + entry.lanes = Object.fromEntries(Object.entries(entry.lanes).filter(([name]) => name !== skill)); + } + previous = undefined; + } + const usage = { ...lane.usage }; + for (const key of Object.keys(usage) as (keyof LaneUsage)[]) usage[key] += previous?.usage[key] ?? 0; + baseline.lanes[skill] = { + config: lane.config, + usage, + runs: lane.runs + (previous?.runs ?? 0), + failedRuns: lane.failedRuns + (previous?.failedRuns ?? 0), + }; - for (const run of runs) { - const found = new Set(run.scores - .filter((score) => score.verdict === 'known-found') - .flatMap((score) => score.matchedCorpusIds)); - recordShardRun(results, { - skill: 'security-review', - config, - usage: { - inputTokens: run.summary.inputTokens, - outputTokens: run.summary.outputTokens, - cacheReadInputTokens: run.summary.cacheReadInputTokens, - costUSD: run.summary.costUSD, - judgeCostUSD: 0, - }, - hits: new Map(corpus.map((entry) => [entry.id, found.has(entry.id)])), - }); + for (const [entryId, entry] of Object.entries(run.entries)) { + const tally = entry.lanes[skill]; + if (!tally) continue; + const recorded = baseline.entries[entryId]; + // An entry whose ref moved starts over: its old tallies describe other code. + const target = recorded && recorded.ref === entry.ref ? recorded : { ...entry, lanes: {} }; + target.lanes[skill] = tally; + baseline.entries[entryId] = target; + } } - results.summary = summarize(results); - return results; + baseline.summary = summarize(baseline); + return baseline; } diff --git a/packages/security-evals/src/entries.test.ts b/packages/security-evals/src/entries.test.ts index aa18ea637..db5337dd7 100644 --- a/packages/security-evals/src/entries.test.ts +++ b/packages/security-evals/src/entries.test.ts @@ -1,5 +1,8 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries, type SuiteEntry } from './entries.js'; +import { filterEntries, groupShards, loadCaseEntries, loadCaseFiles, loadCorpusEntries, type SuiteEntry } from './entries.js'; function entry(id: string, ref: string, targets: string[]): SuiteEntry { return { id, source: 'corpus', kind: 'bug', repository: 'getsentry/sentry', ref, targets, summary: id }; @@ -22,9 +25,18 @@ describe('loadCaseEntries', () => { expect(cases.every((c) => /^[0-9a-f]{40}$/.test(c.ref) && c.targets.length > 0)).toBe(true); }); - it('does not duplicate corpus entries', () => { - const corpus = new Set(loadCorpusEntries().map((e) => `${e.ref}:${e.targets[0]}`)); - expect(loadCaseEntries().filter((c) => corpus.has(`${c.ref}:${c.targets[0]}`))).toEqual([]); + it('rejects a malformed case with its path', () => { + const dir = mkdtempSync(join(tmpdir(), 'security-evals-cases-')); + try { + mkdirSync(join(dir, 'wrdn-pii')); + writeFileSync(join(dir, 'wrdn-pii', 'leak.json'), JSON.stringify({ + kind: 'clean', summary: 's', repository: 'getsentry/sentry', ref: 'a'.repeat(40), + targets: ['x.py'], source: 'https://github.com/getsentry/sentry/pull/1', noClean: 'why', + })); + expect(() => loadCaseFiles(dir)).toThrow(/wrdn-pii\/leak\.json:[\s\S]*noClean only applies to bug cases/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); }); diff --git a/packages/security-evals/src/entries.ts b/packages/security-evals/src/entries.ts index fd1fc3c63..311cccd50 100644 --- a/packages/security-evals/src/entries.ts +++ b/packages/security-evals/src/entries.ts @@ -9,7 +9,10 @@ const CORPUS_PATH = join( ); /** Hand-picked cases for bug classes the corpus does not cover, one directory per owning skill. */ -const CASES_DIR = join(import.meta.dirname, '..', 'cases'); +export const CASES_DIR = join(import.meta.dirname, '..', 'cases'); + +/** Summary the scaffolder writes. The validator rejects any case that still has it. */ +export const SUMMARY_TODO = 'TODO: one sentence naming the bug, where it lives, and what an attacker gets'; /** * One scored expectation. A `bug` entry is a hit when a skill reports it. A @@ -47,15 +50,38 @@ const CorpusSchema = z.object({ })), }); -const CaseSchema = z.object({ +const RelativePath = z.string().min(1).refine( + (path) => !path.startsWith('/') && !path.split('/').includes('..'), + 'must be a repository-relative path', +); + +/** + * One file under `cases//`. A bug case pairs with a clean case + * that shares its source, targets, and summary at the fixed ref, unless + * `noClean` says why it cannot. + */ +export const CaseSchema = z.object({ kind: z.enum(['bug', 'clean']), - summary: z.string(), - repository: z.string(), + summary: z.string().min(1), + repository: z.string().regex(/^[\w.-]+\/[\w.-]+$/), ref: z.string().regex(/^[0-9a-f]{40}$/), - targets: z.array(z.string()).min(1), - source: z.string(), + targets: z.array(RelativePath).min(1), + source: z.url(), + noClean: z.string().min(1).optional(), +}).strict().refine((data) => data.kind === 'bug' || data.noClean === undefined, { + message: 'noClean only applies to bug cases', + path: ['noClean'], }); +export type CaseData = z.infer; + +/** A parsed case file and the skill directory that owns it. */ +export interface CaseFile { + id: string; + owner: string; + data: CaseData; +} + /** Load one entry per corpus finding. */ export function loadCorpusEntries(path = CORPUS_PATH): SuiteEntry[] { const corpus = CorpusSchema.parse(JSON.parse(readFileSync(path, 'utf8'))); @@ -71,27 +97,36 @@ export function loadCorpusEntries(path = CORPUS_PATH): SuiteEntry[] { })); } -/** Load hand-picked cases. Ids read as `/`. */ -export function loadCaseEntries(dir = CASES_DIR): SuiteEntry[] { +/** Parse every case file. Ids read as `/`. A bad file throws with its path. */ +export function loadCaseFiles(dir = CASES_DIR): CaseFile[] { return readdirSync(dir, { withFileTypes: true }) .filter((owner) => owner.isDirectory()) .flatMap((owner) => readdirSync(join(dir, owner.name)) .filter((file) => file.endsWith('.json')) .map((file) => { - const data = CaseSchema.parse(JSON.parse(readFileSync(join(dir, owner.name, file), 'utf8'))); - return { - id: `${owner.name}/${basename(file, '.json')}`, - source: 'case' as const, - kind: data.kind, - repository: data.repository, - ref: data.ref, - targets: data.targets, - summary: data.summary, - }; + const path = join(dir, owner.name, file); + const parsed = CaseSchema.safeParse(JSON.parse(readFileSync(path, 'utf8'))); + if (!parsed.success) { + throw new Error(`${path}: ${z.prettifyError(parsed.error)}`); + } + return { id: `${owner.name}/${basename(file, '.json')}`, owner: owner.name, data: parsed.data }; })) .sort((a, b) => a.id.localeCompare(b.id)); } +/** Load hand-picked cases as suite entries. */ +export function loadCaseEntries(dir = CASES_DIR): SuiteEntry[] { + return loadCaseFiles(dir).map(({ id, data }) => ({ + id, + source: 'case', + kind: data.kind, + repository: data.repository, + ref: data.ref, + targets: data.targets, + summary: data.summary, + })); +} + /** * Keep entries matching any filter. A filter is an exact id, or a prefix * ending in `*` (`wrdn-dos-review/*`, `sentry-vuln-0*`). No filters keeps all. diff --git a/packages/security-evals/src/options.test.ts b/packages/security-evals/src/options.test.ts index 603d60460..e7f33c857 100644 --- a/packages/security-evals/src/options.test.ts +++ b/packages/security-evals/src/options.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { MATRIX_SKILLS, WARDEN_SKILLS_REF, resolveSuiteOptions } from './options.js'; +import { MATRIX_SKILLS, WARDEN_SKILLS_REF, laneConfig, resolveSuiteOptions } from './options.js'; describe('resolveSuiteOptions', () => { it('defaults to the production lane over every matrix skill', () => { @@ -36,6 +36,25 @@ describe('resolveSuiteOptions', () => { expect(options.baseline).toBeUndefined(); }); + it('maps --repo pairs and the --sentry-repo alias onto one clone per repository', () => { + expect(resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_REPOS: 'getsentry/getsentry=/src/getsentry', + WARDEN_SECURITY_EVALS_SENTRY_REPO: '/src/sentry', + }).repos).toEqual({ 'getsentry/getsentry': '/src/getsentry', 'getsentry/sentry': '/src/sentry' }); + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_REPOS: '/src/sentry' })).toThrow('owner/name='); + expect(() => resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_REPOS: 'getsentry/sentry=/a', + WARDEN_SECURITY_EVALS_SENTRY_REPO: '/b', + })).toThrow('set by both'); + }); + + it('records provenance a baseline lane must match', () => { + const options = resolveSuiteOptions({}); + expect(laneConfig('security-review', options)).toMatchObject({ skillRef: 'builtin', model: 'openrouter/x-ai/grok-4.5' }); + expect(laneConfig('wrdn-pii', options).skillRef).toBe(`getsentry/warden-skills@${WARDEN_SKILLS_REF}`); + expect(laneConfig('wrdn-pii', options).wardenVersion).toMatch(/^\d+\.\d+\.\d+/); + }); + it('pins the same warden-skills ref the CI workflow checks out', () => { const workflow = readFileSync( join(import.meta.dirname, '..', '..', '..', '.github', 'workflows', 'security-evals.yml'), diff --git a/packages/security-evals/src/options.ts b/packages/security-evals/src/options.ts index 248fce670..cf687b7a1 100644 --- a/packages/security-evals/src/options.ts +++ b/packages/security-evals/src/options.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { EffortSchema, RuntimeNameSchema, type Effort, type RuntimeName } from '@sentry/warden'; @@ -24,7 +25,8 @@ const PACKAGE_DIR = join(import.meta.dirname, '..'); export const DEFAULT_BASELINE = join(PACKAGE_DIR, 'baseline', 'results.json'); export interface SuiteOptions { - sentryRepo?: string; + /** Local clone per repository, keyed `owner/name`. Every entry's repository needs one. */ + repos: Record; /** Local getsentry/warden-skills checkout. Required when a wrdn-* lane is selected. */ skillsRepo?: string; skills: string[]; @@ -53,6 +55,27 @@ function list(value: string | undefined): string[] { return (value ?? '').split(',').map((item) => item.trim()).filter(Boolean); } +/** Parse `owner/name=` pairs. `--sentry-repo` is an alias for the getsentry/sentry pair. */ +function repoPaths(value: string | undefined, sentryRepo: string | undefined): Record { + const repos: Record = {}; + for (const pair of list(value)) { + const split = pair.indexOf('='); + const repository = pair.slice(0, split); + const path = pair.slice(split + 1); + if (split < 0 || !/^[\w.-]+\/[\w.-]+$/.test(repository) || !path) { + throw new Error(`${PREFIX}REPOS entries look like owner/name=, got ${pair}`); + } + repos[repository] = path; + } + if (sentryRepo) { + if (repos['getsentry/sentry'] && repos['getsentry/sentry'] !== sentryRepo) { + throw new Error('getsentry/sentry is set by both --repo and --sentry-repo; pass one'); + } + repos['getsentry/sentry'] = sentryRepo; + } + return repos; +} + function positiveInt(name: string, value: string | undefined, fallback: number): number { if (!value) return fallback; const parsed = Number(value); @@ -80,7 +103,7 @@ export function resolveSuiteOptions(env: NodeJS.ProcessEnv = process.env): Suite const baseline = read('BASELINE') ?? DEFAULT_BASELINE; return { - sentryRepo: read('SENTRY_REPO'), + repos: repoPaths(read('REPOS'), read('SENTRY_REPO')), skillsRepo: read('SKILLS_REPO'), skills: skills.length > 0 ? skills : [...MATRIX_SKILLS], entries: list(read('ENTRIES')), @@ -99,3 +122,39 @@ export function resolveSuiteOptions(env: NodeJS.ProcessEnv = process.env): Suite removeWorktrees: read('REMOVE_WORKTREES') === '1', }; } + +const WARDEN_VERSION = (JSON.parse( + readFileSync(join(PACKAGE_DIR, '..', 'warden', 'package.json'), 'utf8'), +) as { version: string }).version; + +/** + * Everything that changes what a lane finds or how it is scored. A baseline + * lane only stands in for a lane whose config matches field for field. + */ +export interface LaneConfig { + runtime: string; + model: string; + effort: string; + auxiliaryModel: string; + auxiliaryEffort: string; + maxTurns: number; + judgeModel: string; + wardenVersion: string; + /** `builtin` for security-review, `getsentry/warden-skills@` for wrdn-* skills. */ + skillRef: string; +} + +/** The config a lane runs with under these options. */ +export function laneConfig(skill: string, options: SuiteOptions): LaneConfig { + return { + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + maxTurns: options.maxTurns, + judgeModel: options.judgeModel, + wardenVersion: WARDEN_VERSION, + skillRef: skill === 'security-review' ? 'builtin' : `getsentry/warden-skills@${WARDEN_SKILLS_REF}`, + }; +} diff --git a/packages/security-evals/src/results.test.ts b/packages/security-evals/src/results.test.ts index 7dd4e09c5..c63cc0fae 100644 --- a/packages/security-evals/src/results.test.ts +++ b/packages/security-evals/src/results.test.ts @@ -1,101 +1,60 @@ import { describe, expect, it } from 'vitest'; -import type { SuiteEntry } from './entries.js'; -import { - createResults, - mergeBaseline, - recordShardRun, - renderMarkdown, - serializeResults, - summarize, - type LaneConfig, - type LaneUsage, -} from './results.js'; - -const config: LaneConfig = { - runtime: 'pi', - model: 'm', - effort: 'high', - auxiliaryModel: 'a', - auxiliaryEffort: 'high', - maxTurns: 100, - judgeModel: 'j', -}; -const usage = (costUSD: number): LaneUsage => ({ - inputTokens: 10, outputTokens: 1, cacheReadInputTokens: 5, costUSD, judgeCostUSD: 0.01, -}); - -function entry(id: string, kind: SuiteEntry['kind'] = 'bug'): SuiteEntry { - return { id, source: 'corpus', kind, repository: 'getsentry/sentry', ref: 'a', targets: ['x.py'], summary: id }; -} - -function record(results: ReturnType, skill: string, hits: Record): void { - recordShardRun(results, { skill, config, usage: usage(1), hits: new Map(Object.entries(hits)) }); -} +import { createResults, recordShardRun, renderMarkdown, serializeResults, summarize } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; describe('summarize', () => { - it('reports recall, unique catches, misses by security-review, and clean flags', () => { - const results = createResults([entry('authz'), entry('dos'), entry('both'), entry('fixed', 'clean')]); - record(results, 'security-review', { authz: false, dos: false, both: true, fixed: false }); - record(results, 'security-review', { authz: false, dos: false, both: true, fixed: false }); - record(results, 'wrdn-authz', { authz: true, dos: false, both: true, fixed: true }); - record(results, 'wrdn-dos-review', { authz: true, dos: true, both: false, fixed: false }); + it('reports recall, unique catches, and clean flags', () => { + const results = createResults([ + testEntry('authz'), testEntry('dos'), testEntry('both'), testEntry('fixed', { kind: 'clean' }), + ]); + recordRepeats(results, 'security-review', { authz: 0, dos: 0, both: 2, fixed: 0 }, 2); + recordRepeats(results, 'wrdn-authz', { authz: 1, dos: 0, both: 1, fixed: 1 }, 1); + recordRepeats(results, 'wrdn-dos-review', { authz: 1, dos: 1, both: 0, fixed: 0 }, 1); const summary = summarize(results); - expect(summary['security-review']).toMatchObject({ found: 1, recall: 1 / 3, onlyThisLane: [] }); - expect(summary['wrdn-authz']).toMatchObject({ - found: 2, - onlyThisLane: [], - missedBySecurityReview: ['authz'], - cleanFlagged: { hits: 1, runs: 1 }, - }); - expect(summary['wrdn-dos-review']).toMatchObject({ - onlyThisLane: ['dos'], - missedBySecurityReview: ['authz', 'dos'], - }); + expect(summary['security-review']).toMatchObject({ found: 1, recall: 1 / 3, onlyThisLane: [], errors: 0 }); + expect(summary['wrdn-authz']).toMatchObject({ found: 2, onlyThisLane: [], cleanFlagged: { hits: 1, runs: 1 } }); + expect(summary['wrdn-dos-review']).toMatchObject({ onlyThisLane: ['dos'] }); expect(results.lanes['security-review']).toMatchObject({ runs: 2, usage: { costUSD: 2, judgeCostUSD: 0.02 } }); }); - it('leaves entries out of a lane when its pass failed', () => { - const results = createResults([entry('authz')]); - recordShardRun(results, { skill: 'wrdn-authz', config, usage: usage(3) }); - expect(results.lanes['wrdn-authz']).toMatchObject({ runs: 0, failedRuns: 1, usage: { costUSD: 3 } }); - expect(summarize(results)['wrdn-authz']).toMatchObject({ bugEntries: 0, recall: null }); - }); -}); - -describe('mergeBaseline', () => { - it('adds lanes the run skipped, limited to the selected entries', () => { - const baseline = createResults([entry('authz'), entry('dos')]); - record(baseline, 'security-review', { authz: true, dos: false }); - record(baseline, 'wrdn-authz', { authz: false, dos: false }); - - const results = createResults([entry('authz')]); - record(results, 'wrdn-authz', { authz: true }); - mergeBaseline(results, baseline, ['wrdn-authz']); - - expect(Object.keys(results.entries)).toEqual(['authz']); - expect(results.entries['authz']!.lanes).toEqual({ - 'security-review': { hits: 1, runs: 1 }, - 'wrdn-authz': { hits: 1, runs: 1 }, + it('records a failed pass as errors on its entries, not misses', () => { + const results = createResults([testEntry('authz')]); + recordShardRun(results, { + skill: 'wrdn-authz', config: testConfig(), usage: testUsage(3), error: 'boom', hits: new Map(), errors: ['authz'], }); + expect(results.lanes['wrdn-authz']).toMatchObject({ runs: 0, failedRuns: 1, usage: { costUSD: 3 } }); + expect(results.entries['authz']!.lanes['wrdn-authz']).toEqual({ hits: 0, runs: 0, errors: 1 }); + expect(summarize(results)['wrdn-authz']).toMatchObject({ bugEntries: 0, recall: null, errors: 1 }); }); }); describe('output', () => { - it('renders the lane table and the entry x lane table', () => { - const results = createResults([entry('authz')]); - record(results, 'security-review', { authz: false }); - record(results, 'wrdn-authz', { authz: true }); + it('leads with verdicts and never prints a bare unique-catch count', () => { + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'security-review', { authz: 0 }, 1); + recordRepeats(results, 'wrdn-authz', { authz: 1 }, 1); results.summary = summarize(results); const markdown = renderMarkdown(results); - expect(markdown).toContain('| wrdn-authz | 1.00 | 1/1 | 1 | 1 | 0/0 | 1.00 | 10 | 1 |'); + expect(markdown).toContain('- **wrdn-authz: insufficient-evidence**.'); + expect(markdown).toContain('| wrdn-authz | 1.00 | 1/1 | 1 | 0/0 | 0 | 1.00 | 10 | 1 |'); expect(markdown).toContain('| authz | bug | 0/1 | 1/1 |'); + expect(markdown).not.toContain('missed by security-review'); + }); + + it('marks errored cells', () => { + const results = createResults([testEntry('authz')]); + recordShardRun(results, { + skill: 'security-review', config: testConfig(), usage: testUsage(), error: 'boom', hits: new Map(), errors: ['authz'], + }); + results.summary = summarize(results); + expect(renderMarkdown(results)).toContain('| authz | bug | 0/0 (1 err) |'); }); it('serializes with sorted keys', () => { - const results = createResults([entry('b'), entry('a')]); + const results = createResults([testEntry('b'), testEntry('a')]); expect(Object.keys(JSON.parse(serializeResults(results)).entries)).toEqual(['a', 'b']); }); }); diff --git a/packages/security-evals/src/results.ts b/packages/security-evals/src/results.ts index bc5f747d1..d35eb654d 100644 --- a/packages/security-evals/src/results.ts +++ b/packages/security-evals/src/results.ts @@ -1,19 +1,6 @@ import type { SuiteEntry } from './entries.js'; - -/** How a lane ran. Baseline lanes keep the config they were recorded with. */ -export interface LaneConfig { - runtime: string; - model: string; - effort: string; - auxiliaryModel: string; - auxiliaryEffort: string; - maxTurns: number; - judgeModel: string; - /** Pinned warden-skills ref for wrdn-* lanes. */ - skillsRef?: string; - /** Where the lane came from when it was not run by this suite. */ - origin?: string; -} +import type { LaneConfig } from './options.js'; +import { REFERENCE_SKILL, judgeSkill, type SkillVerdict } from './verdict.js'; export interface LaneUsage { inputTokens: number; @@ -26,21 +13,25 @@ export interface LaneUsage { export interface LaneResult { config: LaneConfig; usage: LaneUsage; - /** Passes completed: one per shard and repeat, or one per imported benchmark run. */ + /** Passes completed: one per shard and repeat. */ runs: number; - /** Shard passes that errored and were not scored. */ + /** Shard passes that errored. Their entries count as errors, not misses. */ failedRuns: number; + /** Set when the lane was copied from the baseline instead of run. */ + fromBaseline?: true; } -/** Hits over completed repeats for one entry in one lane. */ +/** One entry in one lane: hits over scored repeats, plus repeats that errored. */ export interface Tally { hits: number; runs: number; + errors: number; } export interface EntryResult { source: SuiteEntry['source']; kind: SuiteEntry['kind']; + repository: string; ref: string; targets: string[]; lanes: Record; @@ -48,18 +39,19 @@ export interface EntryResult { export interface LaneSummary { bugEntries: number; - /** Mean hit rate over bug entries this lane ran. */ + /** Mean hit rate over bug entries this lane scored. */ recall: number | null; found: number; /** Bug entries this lane found and every other lane that ran them missed. */ onlyThisLane: string[]; - /** Bug entries this lane found and security-review missed. The "do we need it" list. */ - missedBySecurityReview: string[]; - cleanFlagged: Tally; + cleanFlagged: { hits: number; runs: number }; + errors: number; + /** Whether a wrdn-* skill catches what security-review misses. Absent for security-review itself. */ + verdict?: SkillVerdict; } export interface SuiteResults { - version: 1; + version: 2; lanes: Record; entries: Record; summary: Record; @@ -70,20 +62,22 @@ export interface ShardRun { skill: string; config: LaneConfig; usage: LaneUsage; - /** Undefined when the pass errored; the entries then get no repeat. */ - hits?: Map; + /** Set when the whole pass failed. */ + error?: string; + hits: Map; + /** Entries this pass could not score. */ + errors: string[]; } -const SECURITY_REVIEW = 'security-review'; - /** Start an empty result set for the selected entries. */ export function createResults(entries: SuiteEntry[]): SuiteResults { return { - version: 1, + version: 2, lanes: {}, entries: Object.fromEntries(entries.map((entry) => [entry.id, { source: entry.source, kind: entry.kind, + repository: entry.repository, ref: entry.ref, targets: entry.targets, lanes: {}, @@ -103,48 +97,41 @@ export function recordShardRun(results: SuiteResults, run: ShardRun): void { for (const key of Object.keys(lane.usage) as (keyof LaneUsage)[]) { lane.usage[key] += run.usage[key]; } - if (!run.hits) { + if (run.error) { lane.failedRuns += 1; } else { lane.runs += 1; - for (const [entryId, hit] of run.hits) { - const entry = results.entries[entryId]; - if (!entry) continue; - const tally = entry.lanes[run.skill] ?? { hits: 0, runs: 0 }; - tally.runs += 1; - tally.hits += hit ? 1 : 0; - entry.lanes[run.skill] = tally; - } } - results.lanes[run.skill] = lane; -} - -/** - * Copy lanes this run skipped from a baseline, limited to the selected - * entries, so a candidate lane compares without rerunning the rest. - */ -export function mergeBaseline(results: SuiteResults, baseline: SuiteResults, ranSkills: string[]): void { - for (const [skill, lane] of Object.entries(baseline.lanes)) { - if (ranSkills.includes(skill)) continue; - results.lanes[skill] = lane; - for (const [entryId, entry] of Object.entries(results.entries)) { - const tally = baseline.entries[entryId]?.lanes[skill]; - if (tally) entry.lanes[skill] = tally; - } + const tally = (entryId: string): Tally | undefined => { + const entry = results.entries[entryId]; + if (!entry) return undefined; + entry.lanes[run.skill] ??= { hits: 0, runs: 0, errors: 0 }; + return entry.lanes[run.skill]; + }; + for (const [entryId, hit] of run.hits) { + const t = tally(entryId); + if (!t) continue; + t.runs += 1; + t.hits += hit ? 1 : 0; } + for (const entryId of run.errors) { + const t = tally(entryId); + if (t) t.errors += 1; + } + results.lanes[run.skill] = lane; } function hitRate(tally: Tally | undefined): number | undefined { return tally && tally.runs > 0 ? tally.hits / tally.runs : undefined; } -/** Compute per-lane recall, unique catches, and clean-case false positives. */ +/** Compute per-lane recall, unique catches, clean-case false positives, and each wrdn-* verdict. */ export function summarize(results: SuiteResults): Record { const entries = Object.entries(results.entries); const summary: Record = {}; for (const skill of Object.keys(results.lanes).sort()) { - // Entries this lane ran, with its hit rate on each. + // Entries this lane scored, with its hit rate on each. const ran = entries.flatMap(([id, entry]) => { const rate = hitRate(entry.lanes[skill]); return rate === undefined ? [] : [{ id, entry, rate }]; @@ -160,37 +147,44 @@ export function summarize(results: SuiteResults): Record { recall: bugs.length > 0 ? bugs.reduce((sum, { rate }) => sum + rate, 0) / bugs.length : null, found: found.length, onlyThisLane: found.filter(({ entry }) => othersMissed(entry)).map(({ id }) => id), - missedBySecurityReview: skill === SECURITY_REVIEW - ? [] - : found.filter(({ entry }) => entry.lanes[SECURITY_REVIEW]?.hits === 0).map(({ id }) => id), cleanFlagged: { hits: clean.filter(({ rate }) => rate > 0).length, runs: clean.length }, + errors: entries.reduce((sum, [, entry]) => sum + (entry.lanes[skill]?.errors ?? 0), 0), + ...(skill === REFERENCE_SKILL ? {} : { verdict: judgeSkill(results, skill) }), }; } return summary; } function cell(tally: Tally | undefined): string { - return tally ? `${tally.hits}/${tally.runs}` : '-'; + if (!tally) return '-'; + return `${tally.hits}/${tally.runs}${tally.errors > 0 ? ` (${tally.errors} err)` : ''}`; } -/** Render the summary and the entry x lane hit table as Markdown. */ +/** Render verdicts, the lane table, and the entry x lane hit table as Markdown. */ export function renderMarkdown(results: SuiteResults): string { const skills = Object.keys(results.lanes).sort(); - const lines = [ - '## Security evals', + const lines = ['## Security evals', '', '### Verdicts', '']; + for (const skill of skills) { + const verdict = results.summary[skill]?.verdict; + if (!verdict) continue; + const ids = verdict.uniqueCatches.length > 0 ? ` Unique: ${verdict.uniqueCatches.join(', ')}.` : ''; + lines.push(`- **${skill}: ${verdict.verdict}**. ${verdict.reason}.${ids}`); + } + + lines.push( '', - '| lane | recall | found | only this lane | missed by security-review | clean flagged | cost USD | input tokens | output tokens |', + '| lane | recall | found | only this lane | clean flagged | errors | cost USD | input tokens | output tokens |', '|---|---|---|---|---|---|---|---|---|', - ]; + ); for (const skill of skills) { const lane = results.lanes[skill]; const s = results.summary[skill]; if (!lane || !s) continue; const recall = s.recall === null ? '-' : s.recall.toFixed(2); lines.push( - `| ${skill}${lane.config.origin ? ' (baseline)' : ''} | ${recall} | ${s.found}/${s.bugEntries} ` - + `| ${s.onlyThisLane.length} | ${skill === SECURITY_REVIEW ? '-' : s.missedBySecurityReview.length} ` - + `| ${cell(s.cleanFlagged)} | ${lane.usage.costUSD.toFixed(2)} | ${lane.usage.inputTokens} | ${lane.usage.outputTokens} |`, + `| ${skill}${lane.fromBaseline ? ' (baseline)' : ''} | ${recall} | ${s.found}/${s.bugEntries} ` + + `| ${s.onlyThisLane.length} | ${s.cleanFlagged.hits}/${s.cleanFlagged.runs} | ${s.errors} ` + + `| ${lane.usage.costUSD.toFixed(2)} | ${lane.usage.inputTokens} | ${lane.usage.outputTokens} |`, ); } diff --git a/packages/security-evals/src/scaffold.test.ts b/packages/security-evals/src/scaffold.test.ts new file mode 100644 index 000000000..a4d6858af --- /dev/null +++ b/packages/security-evals/src/scaffold.test.ts @@ -0,0 +1,138 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { SUMMARY_TODO, loadCaseFiles } from './entries.js'; +import { MATRIX_SKILLS } from './options.js'; +import { createResults } from './results.js'; +import { inferTargets, parseChangeUrl, scaffoldCase, type ChangedFile, type GitHubReader } from './scaffold.js'; +import { validateSuite } from './validate.js'; + +const FIX = '57f0df6e319f9de3718e0e9804eb65ffc07aca3e'; +const PARENT = 'ee57ed6a'.padEnd(40, '0'); + +// Shaped like getsentry/sentry#108555, which bounded recursion in _encode_value. +const FILES: ChangedFile[] = [ + { filename: 'src/sentry/eventstream/item_helpers.py', status: 'modified' }, + { filename: 'tests/sentry/eventstream/test_item_helpers.py', status: 'modified' }, +]; + +function github(overrides: Partial = {}): GitHubReader { + return { + pull: async () => ({ title: 'fix(eap): Limit nesting depth in _encode_value', mergeCommit: FIX }), + commit: async () => ({ sha: FIX, subject: 'fix(eap): Limit nesting depth (#108555)', parents: [PARENT] }), + compare: async () => FILES, + ...overrides, + }; +} + +describe('parseChangeUrl', () => { + it('reads pull request and commit URLs', () => { + expect(parseChangeUrl('https://github.com/getsentry/sentry/pull/108555/files')) + .toEqual({ owner: 'getsentry', repo: 'sentry', pull: 108555 }); + expect(parseChangeUrl(`https://github.com/getsentry/sentry/commit/${FIX}`)) + .toEqual({ owner: 'getsentry', repo: 'sentry', commit: FIX }); + }); + + it('rejects anything else', () => { + expect(() => parseChangeUrl('https://github.com/getsentry/sentry/issues/1')).toThrow('pull request or commit URL'); + }); +}); + +describe('inferTargets', () => { + it('keeps modified non-test .py and .ts files and says why it skipped the rest', () => { + const { targets, skipped } = inferTargets([ + { filename: 'src/sentry/api/endpoints/system_options.py', status: 'modified' }, + { filename: 'static/app/utils/request.ts', status: 'modified' }, + { filename: 'src/sentry/new_helper.py', status: 'added' }, + { filename: 'src/sentry/old_helper.py', status: 'removed' }, + { filename: 'src/sentry/moved.py', status: 'renamed' }, + { filename: 'tests/sentry/api/test_system_options.py', status: 'modified' }, + { filename: 'src/sentry/testutils/factories.py', status: 'modified' }, + { filename: 'src/sentry/conftest.py', status: 'modified' }, + { filename: 'static/app/utils/request.spec.ts', status: 'modified' }, + { filename: 'static/app/views/page.test.ts', status: 'modified' }, + { filename: 'static/app/types/index.d.ts', status: 'modified' }, + { filename: 'CHANGES', status: 'modified' }, + ]); + expect(targets).toEqual(['src/sentry/api/endpoints/system_options.py', 'static/app/utils/request.ts']); + expect(Object.fromEntries(skipped.map(({ path, reason }) => [path, reason]))).toEqual({ + 'src/sentry/new_helper.py': 'added by the fix, so it is missing at one of the two refs', + 'src/sentry/old_helper.py': 'removed by the fix, so it is missing at one of the two refs', + 'src/sentry/moved.py': 'renamed by the fix, so it is missing at one of the two refs', + 'tests/sentry/api/test_system_options.py': 'test file', + 'src/sentry/testutils/factories.py': 'test file', + 'src/sentry/conftest.py': 'test file', + 'static/app/utils/request.spec.ts': 'test file', + 'static/app/views/page.test.ts': 'test file', + 'static/app/types/index.d.ts': 'not a .py or .ts file', + CHANGES: 'not a .py or .ts file', + }); + }); +}); + +describe('scaffoldCase', () => { + let casesDir: string; + + beforeEach(() => { + casesDir = mkdtempSync(join(tmpdir(), 'security-evals-cases-')); + }); + + afterEach(() => { + rmSync(casesDir, { recursive: true, force: true }); + }); + + it('writes the bug at the fix parent and the clean twin at the fix', async () => { + const url = 'https://github.com/getsentry/sentry/pull/108555'; + const result = await scaffoldCase({ url, skill: 'wrdn-dos-review', github: github(), casesDir }); + + expect(result).toMatchObject({ + bugId: 'wrdn-dos-review/fix-eap-limit-nesting-depth-in-encode-value', + cleanId: 'wrdn-dos-review/fix-eap-limit-nesting-depth-in-encode-value-fixed', + bugRef: PARENT, + fixRef: FIX, + targets: ['src/sentry/eventstream/item_helpers.py'], + }); + expect(JSON.parse(readFileSync(result.bugPath, 'utf8'))).toEqual({ + kind: 'bug', + summary: SUMMARY_TODO, + repository: 'getsentry/sentry', + targets: ['src/sentry/eventstream/item_helpers.py'], + source: url, + ref: PARENT, + }); + expect(JSON.parse(readFileSync(result.cleanPath, 'utf8'))).toMatchObject({ kind: 'clean', ref: FIX, source: url }); + }); + + it('writes a pair the validator rejects only for its TODO summary', async () => { + await scaffoldCase({ + url: `https://github.com/getsentry/sentry/commit/${FIX}`, + skill: 'wrdn-dos-review', + name: 'eap-item-encode', + github: github(), + casesDir, + }); + const problems = validateSuite({ + cases: loadCaseFiles(casesDir), corpus: [], skills: MATRIX_SKILLS, baseline: createResults([]), + }); + expect(problems).toEqual([ + 'wrdn-dos-review/eap-item-encode: summary is still the TODO placeholder', + 'wrdn-dos-review/eap-item-encode-fixed: summary is still the TODO placeholder', + ]); + }); + + it('refuses unmerged pull requests, unknown skills, fixes with no target, and overwrites', async () => { + const url = 'https://github.com/getsentry/sentry/pull/1'; + await expect(scaffoldCase({ + url, skill: 'wrdn-dos-review', casesDir, github: github({ pull: async () => ({ title: 't', mergeCommit: null }) }), + })).rejects.toThrow('is not merged'); + await expect(scaffoldCase({ url, skill: 'wrdn-nope', casesDir, github: github() })).rejects.toThrow('Unknown skill'); + await expect(scaffoldCase({ + url, skill: 'wrdn-dos-review', casesDir, github: github({ compare: async () => [FILES[1]!] }), + })).rejects.toThrow('modified no non-test .py or .ts file'); + + await scaffoldCase({ url, skill: 'wrdn-dos-review', name: 'same', casesDir, github: github() }); + await expect(scaffoldCase({ url, skill: 'wrdn-dos-review', name: 'same', casesDir, github: github() })) + .rejects.toThrow('Case already exists'); + }); +}); diff --git a/packages/security-evals/src/scaffold.ts b/packages/security-evals/src/scaffold.ts new file mode 100644 index 000000000..e87511720 --- /dev/null +++ b/packages/security-evals/src/scaffold.ts @@ -0,0 +1,183 @@ +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseGitHubPullRequestUrl, slugifyEvalName } from '../../evals/src/scaffold.js'; +import { CASES_DIR, CaseSchema, SUMMARY_TODO, type CaseData } from './entries.js'; +import { MATRIX_SKILLS } from './options.js'; + +/** A file the fix touched, as GitHub's compare API reports it. */ +export interface ChangedFile { + filename: string; + status: string; +} + +/** The three GitHub reads the scaffolder needs. Injected so tests run offline. */ +export interface GitHubReader { + pull(owner: string, repo: string, number: number): Promise<{ title: string; mergeCommit: string | null }>; + commit(owner: string, repo: string, ref: string): Promise<{ sha: string; subject: string; parents: string[] }>; + compare(owner: string, repo: string, base: string, head: string): Promise; +} + +export type ChangeRef = + | { owner: string; repo: string; pull: number; commit?: never } + | { owner: string; repo: string; commit: string; pull?: never }; + +/** Parse a GitHub pull request URL or commit URL. */ +export function parseChangeUrl(url: string): ChangeRef { + const commit = /^https:\/\/github\.com\/([\w.-]+)\/([\w.-]+)\/commit\/([0-9a-f]{7,40})\/?$/.exec(url); + if (commit?.[1] && commit[2] && commit[3]) { + return { owner: commit[1], repo: commit[2], commit: commit[3] }; + } + try { + const pull = parseGitHubPullRequestUrl(url); + return { owner: pull.owner, repo: pull.repo, pull: pull.pullNumber }; + } catch { + throw new Error(`Expected a GitHub pull request or commit URL, got ${url}`); + } +} + +const TEST_DIR = /(^|\/)(tests?|__tests__|testutils|fixtures)\//; +const TEST_FILE = /(^|\/)(test_[^/]*\.py|[^/]*_test\.py|conftest\.py|[^/]*\.(test|spec)\.ts)$/; + +/** + * Pick targets from the files a fix touched: non-test `.py` and `.ts` files + * that exist at both the bug ref and the fixed ref. Everything else is + * skipped with a reason, so the author can add a path by hand if needed. + */ +export function inferTargets(files: ChangedFile[]): { targets: string[]; skipped: { path: string; reason: string }[] } { + const targets: string[] = []; + const skipped: { path: string; reason: string }[] = []; + for (const { filename: path, status } of files) { + if (!/\.(py|ts)$/.test(path) || path.endsWith('.d.ts')) { + skipped.push({ path, reason: 'not a .py or .ts file' }); + } else if (TEST_DIR.test(path) || TEST_FILE.test(path)) { + skipped.push({ path, reason: 'test file' }); + } else if (status !== 'modified' && status !== 'changed') { + skipped.push({ path, reason: `${status} by the fix, so it is missing at one of the two refs` }); + } else { + targets.push(path); + } + } + return { targets: targets.sort(), skipped }; +} + +export interface ScaffoldedCase { + repository: string; + bugId: string; + cleanId: string; + bugPath: string; + cleanPath: string; + bugRef: string; + fixRef: string; + targets: string[]; + skipped: { path: string; reason: string }[]; +} + +/** + * Write a bug case at the fix's parent and its clean twin at the fix, both + * with the TODO summary the validator rejects until an author replaces it. + */ +export async function scaffoldCase(args: { + url: string; + skill: string; + github: GitHubReader; + name?: string; + force?: boolean; + casesDir?: string; +}): Promise { + const { url, skill, github, casesDir = CASES_DIR } = args; + if (!(MATRIX_SKILLS as readonly string[]).includes(skill)) { + throw new Error(`Unknown skill ${skill}. Choose from ${MATRIX_SKILLS.join(', ')}`); + } + const change = parseChangeUrl(url); + const { owner, repo } = change; + + const pull = change.pull === undefined ? undefined : await github.pull(owner, repo, change.pull); + const fixCommit = pull ? pull.mergeCommit : change.commit; + if (!fixCommit) { + throw new Error(`${url} is not merged. Pass the commit that fixed the bug instead.`); + } + const fix = await github.commit(owner, repo, fixCommit); + const fixRef = fix.sha; + const bugRef = fix.parents[0]; + if (!bugRef) { + throw new Error(`${fixRef} has no parent commit to hold the bug`); + } + + const { targets, skipped } = inferTargets(await github.compare(owner, repo, bugRef, fixRef)); + if (targets.length === 0) { + throw new Error(`The fix modified no non-test .py or .ts file. Write the case by hand.\n${ + skipped.map((file) => ` ${file.path}: ${file.reason}`).join('\n')}`); + } + + const name = args.name ?? slugifyEvalName(pull?.title ?? fix.subject); + if (!/^[a-z0-9][a-z0-9-]*$/.test(name)) { + throw new Error(`Invalid case name ${name}. Use lowercase letters, numbers, and dashes.`); + } + const repository = `${owner}/${repo}`; + const base = { summary: SUMMARY_TODO, repository, targets, source: url }; + const dir = join(casesDir, skill); + const bugPath = join(dir, `${name}.json`); + const cleanPath = join(dir, `${name}-fixed.json`); + const cases: [string, CaseData][] = [ + [bugPath, CaseSchema.parse({ kind: 'bug', ...base, ref: bugRef })], + [cleanPath, CaseSchema.parse({ kind: 'clean', ...base, ref: fixRef })], + ]; + const existing = cases.map(([path]) => path).filter((path) => existsSync(path)); + if (existing.length > 0 && !args.force) { + throw new Error(`Case already exists: ${existing.join(', ')}. Pass --force to overwrite.`); + } + mkdirSync(dir, { recursive: true }); + for (const [path, data] of cases) { + writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`); + } + + return { + repository, + bugId: `${skill}/${name}`, + cleanId: `${skill}/${name}-fixed`, + bugPath, + cleanPath, + bugRef, + fixRef, + targets, + skipped, + }; +} + +/** Read GitHub's REST API with `GITHUB_TOKEN`/`GH_TOKEN` when set. */ +export function createGitHubReader(token = process.env['GITHUB_TOKEN'] ?? process.env['GH_TOKEN']): GitHubReader { + const get = async (path: string): Promise => { + const response = await fetch(`https://api.github.com${path}`, { + headers: { + accept: 'application/vnd.github+json', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + }); + if (!response.ok) { + throw new Error(`GitHub ${path}: ${response.status} ${await response.text()}`); + } + return response.json() as Promise; + }; + return { + async pull(owner, repo, number) { + const pull = await get<{ title: string; merged: boolean; merge_commit_sha: string | null }>( + `/repos/${owner}/${repo}/pulls/${number}`, + ); + return { title: pull.title, mergeCommit: pull.merged ? pull.merge_commit_sha : null }; + }, + async commit(owner, repo, ref) { + const commit = await get<{ sha: string; commit: { message: string }; parents: { sha: string }[] }>( + `/repos/${owner}/${repo}/commits/${ref}`, + ); + return { + sha: commit.sha, + subject: commit.commit.message.split('\n')[0] ?? '', + parents: commit.parents.map((parent) => parent.sha), + }; + }, + async compare(owner, repo, base, head) { + const comparison = await get<{ files?: ChangedFile[] }>(`/repos/${owner}/${repo}/compare/${base}...${head}`); + return comparison.files ?? []; + }, + }; +} diff --git a/packages/security-evals/src/security.eval.ts b/packages/security-evals/src/security.eval.ts index 43c12bb6f..2760da875 100644 --- a/packages/security-evals/src/security.eval.ts +++ b/packages/security-evals/src/security.eval.ts @@ -1,28 +1,16 @@ -import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { afterAll, describe, it } from 'vitest'; import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries } from './entries.js'; -import { WARDEN_SKILLS_REF, resolveSuiteOptions } from './options.js'; -import { - createResults, - mergeBaseline, - recordShardRun, - renderMarkdown, - serializeResults, - summarize, - type SuiteResults, -} from './results.js'; +import { mergeBaseline, loadResultsFile } from './baseline.js'; +import { WARDEN_SKILLS_REF, laneConfig, resolveSuiteOptions } from './options.js'; +import { createResults, recordShardRun, renderMarkdown, serializeResults, summarize } from './results.js'; import { loadSkills, runShardPass } from './shard.js'; -import { SENTRY_CHECKOUT, SKILLS_CHECKOUT, ensureWorktree, removeWorktree, resolveCheckout } from './worktrees.js'; +import { SKILLS_CHECKOUT, ensureWorktree, removeWorktree, repoCheckout, resolveCheckout } from './worktrees.js'; const options = resolveSuiteOptions(); -const sentryRepo = resolveCheckout(SENTRY_CHECKOUT, options.sentryRepo); const entries = filterEntries([...loadCorpusEntries(), ...loadCaseEntries()], options.entries); const shards = groupShards(entries); -const foreign = shards.filter((shard) => shard.repository !== 'getsentry/sentry'); -if (foreign.length > 0) { - throw new Error(`Only getsentry/sentry entries are supported, got ${foreign.map((s) => s.repository).join(', ')}`); -} // Both worktree directory names are Roach values in roach.json; keep them in sync. const worktreesDir = join(options.cacheDir, 'security-evals-worktrees'); @@ -47,11 +35,13 @@ const PASS_TIMEOUT_MS = 3 * 60 * 60 * 1000; describe('security evals', () => { for (const shard of shards) { + // Resolved while vitest collects tests, so a missing clone fails before any pass runs. + const clone = resolveCheckout(repoCheckout(shard.repository), options.repos[shard.repository]); for (let repeat = 1; repeat <= options.repeats; repeat++) { const name = `${shard.ref.slice(0, 8)} pass ${repeat}/${options.repeats} ` + `(${shard.targets.length} files, ${shard.entries.length} entries)`; it(name, { timeout: PASS_TIMEOUT_MS }, async () => { - const worktree = ensureWorktree(sentryRepo, worktreesDir, shard.ref); + const worktree = ensureWorktree(clone, worktreesDir, shard.ref); try { const runs = await runShardPass({ shard, @@ -62,13 +52,14 @@ describe('security evals', () => { log: (message) => console.log(` ${message}`), }); for (const run of runs) recordShardRun(results, run); - const failed = runs.filter((run) => !run.hits).map((run) => run.skill); + const failed = runs.filter((run) => run.errors.length > 0) + .map((run) => `${run.skill} (${run.errors.length} entries)`); if (failed.length > 0) { - throw new Error(`Skill passes failed: ${failed.join(', ')}`); + throw new Error(`Entries errored, recorded as errors: ${failed.join(', ')}`); } } finally { if (options.removeWorktrees && repeat === options.repeats) { - removeWorktree(sentryRepo, worktree); + removeWorktree(clone, worktree); } } }); @@ -79,8 +70,9 @@ describe('security evals', () => { afterAll(() => { if (Object.keys(results.lanes).length === 0) return; if (options.baseline && existsSync(options.baseline)) { - const baseline = JSON.parse(readFileSync(options.baseline, 'utf8')) as SuiteResults; - mergeBaseline(results, baseline, options.skills); + const warnings = mergeBaseline(results, loadResultsFile(options.baseline), options.skills, + (skill) => laneConfig(skill, options)); + for (const warning of warnings) console.warn(warning); } results.summary = summarize(results); diff --git a/packages/security-evals/src/shard.test.ts b/packages/security-evals/src/shard.test.ts index 6dcb0b761..0d44aead3 100644 --- a/packages/security-evals/src/shard.test.ts +++ b/packages/security-evals/src/shard.test.ts @@ -1,8 +1,21 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { loadSkills } from './shard.js'; +import type * as Warden from '@sentry/warden'; +import { runSkill, type SkillReport } from '@sentry/warden'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { buildFileEventContext } from '../../warden/src/cli/context.js'; +import { resolveSuiteOptions } from './options.js'; +import { scoreEntry } from './score.js'; +import { loadSkills, runShardPass } from './shard.js'; +import { testEntry } from './testing.js'; + +vi.mock('@sentry/warden', async (importOriginal) => ({ + ...await importOriginal(), + runSkill: vi.fn(), +})); +vi.mock('../../warden/src/cli/context.js', () => ({ buildFileEventContext: vi.fn() })); +vi.mock('./score.js', () => ({ scoreEntry: vi.fn() })); function writeSkill(root: string, dir: string, name: string): void { mkdirSync(join(root, 'skills', dir), { recursive: true }); @@ -50,3 +63,77 @@ describe('loadSkills', () => { await expect(loadSkills(['wrdn-pii'], worktree)).rejects.toThrow('loaded wrdn-other'); }); }); + +describe('runShardPass', () => { + const usage = { inputTokens: 100, outputTokens: 10, costUSD: 0.5 }; + const shard = { + repository: 'getsentry/sentry', + ref: 'a'.repeat(40), + targets: ['a.py', 'b.py'], + entries: [testEntry('on-a', { targets: ['a.py'] }), testEntry('on-b', { targets: ['b.py'] })], + }; + const skill = { name: 'wrdn-authz' } as never; + + function pass(skills: string[] = ['wrdn-authz']) { + return runShardPass({ + shard, + worktree: '/worktree', + skills: new Map(skills.map((name) => [name, skill])), + options: resolveSuiteOptions({}), + apiKey: 'key', + log: () => undefined, + }); + } + + beforeEach(() => { + vi.mocked(buildFileEventContext).mockResolvedValue({ + pullRequest: { files: [{ filename: 'a.py' }, { filename: 'b.py' }] }, + } as never); + vi.mocked(scoreEntry).mockResolvedValue({ hit: true, usage: { costUSD: 0.01 } } as never); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + it('scores every entry when the pass succeeds', async () => { + vi.mocked(runSkill).mockResolvedValue({ findings: [], usage } as unknown as SkillReport); + const [run] = await pass(); + expect(run).toMatchObject({ skill: 'wrdn-authz', errors: [], usage: { costUSD: 0.5, judgeCostUSD: 0.02 } }); + expect([...run!.hits]).toEqual([['on-a', true], ['on-b', true]]); + }); + + it('records entries on a failed chunk as errors and still scores the rest', async () => { + vi.mocked(runSkill).mockResolvedValue({ + findings: [], usage, hunkFailures: [{ filename: 'b.py' }], + } as unknown as SkillReport); + const [run] = await pass(); + expect(run).toMatchObject({ errors: ['on-b'] }); + expect([...run!.hits.keys()]).toEqual(['on-a']); + }); + + it('records every entry as an error when the skill reports or throws one, keeping its spend', async () => { + vi.mocked(runSkill).mockResolvedValueOnce({ + findings: [], usage, error: { message: 'provider 429' }, + } as unknown as SkillReport); + vi.mocked(runSkill).mockRejectedValueOnce(new Error('GnuTLS recv error')); + const runs = await pass(['wrdn-authz', 'security-review']); + expect(runs.map(({ skill, error, errors, hits }) => ({ skill, error, errors, hits: hits.size }))).toEqual([ + { skill: 'wrdn-authz', error: 'provider 429', errors: ['on-a', 'on-b'], hits: 0 }, + { skill: 'security-review', error: 'GnuTLS recv error', errors: ['on-a', 'on-b'], hits: 0 }, + ]); + expect(runs[0]!.usage.costUSD).toBe(0.5); + }); + + it('records a judge failure as errors, not misses', async () => { + vi.mocked(runSkill).mockResolvedValue({ findings: [], usage } as unknown as SkillReport); + vi.mocked(scoreEntry).mockRejectedValue(new Error('judge timed out')); + const [run] = await pass(); + expect(run).toMatchObject({ error: 'judge timed out', errors: ['on-a', 'on-b'] }); + }); + + it('refuses a worktree that is missing a target', async () => { + vi.mocked(buildFileEventContext).mockResolvedValue({ pullRequest: { files: [{ filename: 'a.py' }] } } as never); + await expect(pass()).rejects.toThrow('missing targets: b.py'); + }); +}); diff --git a/packages/security-evals/src/shard.ts b/packages/security-evals/src/shard.ts index 7e237f712..55fc289d3 100644 --- a/packages/security-evals/src/shard.ts +++ b/packages/security-evals/src/shard.ts @@ -3,8 +3,8 @@ import { resolveSkillAsync, runSkill, type SkillReport, type UsageStats } from ' // Not on the public surface; the CLI's file mode builds its context the same way. import { buildFileEventContext } from '../../warden/src/cli/context.js'; import type { SuiteShard } from './entries.js'; -import { WARDEN_SKILLS_REF, type SuiteOptions } from './options.js'; -import type { LaneConfig, LaneUsage, ShardRun } from './results.js'; +import { laneConfig, type SuiteOptions } from './options.js'; +import type { LaneUsage, ShardRun } from './results.js'; import { scoreEntry } from './score.js'; type SkillDefinition = Awaited>; @@ -44,20 +44,6 @@ export async function loadSkills(names: string[], skillsWorktree: string | undef return skills; } -/** Lane config recorded with each result. */ -function laneConfig(skill: string, options: SuiteOptions): LaneConfig { - return { - runtime: options.runtime, - model: options.model, - effort: options.effort, - auxiliaryModel: options.auxiliaryModel, - auxiliaryEffort: options.auxiliaryEffort, - maxTurns: options.maxTurns, - judgeModel: options.judgeModel, - ...(skill === 'security-review' ? {} : { skillsRef: `getsentry/warden-skills@${WARDEN_SKILLS_REF}` }), - }; -} - function reportUsage(report: SkillReport): LaneUsage { const parts: UsageStats[] = [report.usage, ...Object.values(report.auxiliaryUsage ?? {})] .filter((usage): usage is UsageStats => usage !== undefined); @@ -72,8 +58,9 @@ function reportUsage(report: SkillReport): LaneUsage { /** * One Warden pass over a shard's worktree: build the file context once, run - * every selected skill against it, then score each entry per skill. Entries in - * a file with a failed chunk get no repeat instead of a false miss. + * every selected skill against it, then score each entry per skill. An entry + * the pass could not score (failed chunk, failed skill, failed judge) is + * recorded as an error for that skill, never as a miss. */ export async function runShardPass(args: { shard: SuiteShard; @@ -93,37 +80,48 @@ export async function runShardPass(args: { return Promise.all([...skills].map(async ([name, skill]): Promise => { const config = laneConfig(name, options); - const report = await runSkill(skill, context, { - apiKey: apiKey || undefined, - runtime: options.runtime, - model: options.model, - effort: options.effort, - auxiliaryModel: options.auxiliaryModel, - auxiliaryEffort: options.auxiliaryEffort, - synthesisModel: options.auxiliaryModel, - maxTurns: options.maxTurns, - concurrency: options.concurrency, - verifyFindings: true, - }); - const usage = reportUsage(report); - if (report.error) { - log(`${name} @ ${shard.ref.slice(0, 8)} failed: ${report.error.message}`); - return { skill: name, config, usage }; - } + const allEntries = shard.entries.map((entry) => entry.id); + let usage: LaneUsage = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, costUSD: 0, judgeCostUSD: 0 }; + const failed = (error: unknown): ShardRun => { + const message = error instanceof Error ? error.message : String(error); + log(`${name} @ ${shard.ref.slice(0, 8)} failed: ${message}`); + return { skill: name, config, usage, error: message, hits: new Map(), errors: allEntries }; + }; + + try { + const report = await runSkill(skill, context, { + apiKey: apiKey || undefined, + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + synthesisModel: options.auxiliaryModel, + maxTurns: options.maxTurns, + concurrency: options.concurrency, + verifyFindings: true, + }); + usage = reportUsage(report); + if (report.error) return failed(report.error.message); - const failedFiles = new Set(report.hunkFailures?.map((failure) => failure.filename)); - const hits = new Map(); - for (const entry of shard.entries) { - if (entry.targets.some((target) => failedFiles.has(target))) { - log(`${name} @ ${shard.ref.slice(0, 8)}: ${entry.id} skipped, a target chunk failed`); - continue; + const failedFiles = new Set(report.hunkFailures?.map((failure) => failure.filename)); + const hits = new Map(); + const errors: string[] = []; + for (const entry of shard.entries) { + if (entry.targets.some((target) => failedFiles.has(target))) { + log(`${name} @ ${shard.ref.slice(0, 8)}: ${entry.id} errored, a target chunk failed`); + errors.push(entry.id); + continue; + } + const verdict = await scoreEntry(entry, report.findings, { model: options.judgeModel, apiKey }); + usage.judgeCostUSD += verdict.usage?.costUSD ?? 0; + hits.set(entry.id, verdict.hit); } - const verdict = await scoreEntry(entry, report.findings, { model: options.judgeModel, apiKey }); - usage.judgeCostUSD += verdict.usage?.costUSD ?? 0; - hits.set(entry.id, verdict.hit); + log(`${name} @ ${shard.ref.slice(0, 8)}: ${report.findings.length} findings, ` + + `${[...hits.values()].filter(Boolean).length}/${hits.size} entries hit, ${errors.length} errored`); + return { skill: name, config, usage, hits, errors }; + } catch (error) { + return failed(error); } - log(`${name} @ ${shard.ref.slice(0, 8)}: ${report.findings.length} findings, ` - + `${[...hits.values()].filter(Boolean).length}/${hits.size} entries hit`); - return { skill: name, config, usage, hits }; })); } diff --git a/packages/security-evals/src/testing.ts b/packages/security-evals/src/testing.ts new file mode 100644 index 000000000..3c95a36ee --- /dev/null +++ b/packages/security-evals/src/testing.ts @@ -0,0 +1,53 @@ +import type { SuiteEntry } from './entries.js'; +import type { LaneConfig } from './options.js'; +import { recordShardRun, type LaneUsage, type SuiteResults } from './results.js'; + +/** Test fixtures shared by the results, verdict, and baseline tests. */ +export const testConfig = (overrides: Partial = {}): LaneConfig => ({ + runtime: 'pi', + model: 'm', + effort: 'high', + auxiliaryModel: 'a', + auxiliaryEffort: 'high', + maxTurns: 100, + judgeModel: 'j', + wardenVersion: '1.0.0', + skillRef: 'builtin', + ...overrides, +}); + +export const testUsage = (costUSD = 1): LaneUsage => ({ + inputTokens: 10, outputTokens: 1, cacheReadInputTokens: 5, costUSD, judgeCostUSD: 0.01, +}); + +export function testEntry(id: string, overrides: Partial = {}): SuiteEntry { + return { + id, + source: id.includes('/') ? 'case' : 'corpus', + kind: 'bug', + repository: 'getsentry/sentry', + ref: 'a'.repeat(40), + targets: ['x.py'], + summary: id, + ...overrides, + }; +} + +/** Record `repeats` passes of one lane, hitting each entry in the first `hits[id]` passes. */ +export function recordRepeats( + results: SuiteResults, + skill: string, + hits: Record, + repeats: number, + config: LaneConfig = testConfig(), +): void { + for (let pass = 0; pass < repeats; pass++) { + recordShardRun(results, { + skill, + config, + usage: testUsage(), + hits: new Map(Object.entries(hits).map(([id, count]) => [id, pass < count])), + errors: [], + }); + } +} diff --git a/packages/security-evals/src/validate.test.ts b/packages/security-evals/src/validate.test.ts new file mode 100644 index 000000000..0ed794131 --- /dev/null +++ b/packages/security-evals/src/validate.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest'; +import { loadResultsFile } from './baseline.js'; +import { SUMMARY_TODO, loadCaseFiles, loadCorpusEntries, type CaseData, type CaseFile } from './entries.js'; +import { DEFAULT_BASELINE, MATRIX_SKILLS } from './options.js'; +import { createResults, recordShardRun } from './results.js'; +import { testConfig, testEntry, testUsage } from './testing.js'; +import { validateSuite } from './validate.js'; + +// The committed suite. This is the check that keeps it honest as cases are added. +describe('the committed suite', () => { + it('has no problems', () => { + expect(validateSuite({ + cases: loadCaseFiles(), + corpus: loadCorpusEntries(), + skills: MATRIX_SKILLS, + baseline: loadResultsFile(DEFAULT_BASELINE), + })).toEqual([]); + }); +}); + +const BUG = 'b'.repeat(40); +const FIX = 'f'.repeat(40); + +function caseFile(id: string, data: Partial = {}): CaseFile { + return { + id, + owner: id.split('/')[0]!, + data: { + kind: 'bug', + summary: 'Unbounded recursion on nested input', + repository: 'getsentry/sentry', + ref: BUG, + targets: ['src/a.py'], + source: 'https://github.com/getsentry/sentry/commit/ffff', + ...data, + }, + }; +} + +const pair = [caseFile('wrdn-dos-review/bug'), caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX })]; + +function check(cases: CaseFile[], extra: Partial[0]> = {}): string[] { + return validateSuite({ cases, corpus: [], skills: MATRIX_SKILLS, baseline: createResults([]), ...extra }); +} + +describe('validateSuite', () => { + it('accepts a bug case with its clean twin', () => { + expect(check(pair)).toEqual([]); + }); + + it('accepts a bug case with a noClean reason and no twin', () => { + expect(check([caseFile('wrdn-authz/catch', { noClean: 'PR catch, fix not pinned' })])).toEqual([]); + }); + + it('rejects a bug case with neither a twin nor a noClean reason', () => { + expect(check([pair[0]!])).toEqual([ + 'wrdn-dos-review/bug: no clean twin (same source and targets at the fixed ref) and no noClean reason', + ]); + }); + + it('rejects a twin whose targets or source drifted', () => { + const moved = caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, targets: ['src/b.py'] }); + expect(check([pair[0]!, moved])).toHaveLength(1); + }); + + it('rejects a twin with a different summary, and noClean next to a twin', () => { + expect(check([ + caseFile('wrdn-dos-review/bug', { noClean: 'stale' }), + caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, summary: 'Something else' }), + ])).toEqual([ + 'wrdn-dos-review/bug: has clean twin wrdn-dos-review/bug-fixed and a noClean reason; drop noClean', + 'wrdn-dos-review/bug: clean twin wrdn-dos-review/bug-fixed has a different summary; they describe the same bug', + ]); + }); + + it('rejects a clean case pinned at the bug ref', () => { + const problems = check([pair[0]!, caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean' })]); + expect(problems).toContain('wrdn-dos-review/bug: clean case wrdn-dos-review/bug-fixed pins the same code'); + }); + + it('rejects the TODO summary the scaffolder writes', () => { + const problems = check([ + caseFile('wrdn-dos-review/bug', { summary: SUMMARY_TODO }), + caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, summary: SUMMARY_TODO }), + ]); + expect(problems).toEqual([ + 'wrdn-dos-review/bug: summary is still the TODO placeholder', + 'wrdn-dos-review/bug-fixed: summary is still the TODO placeholder', + ]); + }); + + it('rejects a case directory that is not a matrix skill', () => { + expect(check([caseFile('wrdn-nope/bug', { noClean: 'x' })])[0]).toContain('cases/wrdn-nope is not a matrix skill'); + }); + + it('rejects a case that duplicates a corpus entry and a duplicate id', () => { + const corpus = [testEntry('sentry-vuln-001', { ref: BUG, targets: ['src/a.py'] }), testEntry('sentry-vuln-001')]; + expect(check(pair, { corpus })).toEqual([ + 'sentry-vuln-001: duplicate entry id', + 'wrdn-dos-review/bug: duplicates corpus entry sentry-vuln-001 (src/a.py at bbbbbbbb)', + ]); + }); + + it('rejects baseline entries that no longer exist or moved, and unknown lanes', () => { + const baseline = createResults([ + testEntry('wrdn-dos-review/gone'), + testEntry('wrdn-dos-review/bug', { ref: 'c'.repeat(40) }), + ]); + recordShardRun(baseline, { + skill: 'wrdn-retired', config: testConfig(), usage: testUsage(), hits: new Map(), errors: [], + }); + expect(check(pair, { baseline })).toEqual([ + 'baseline: lane wrdn-retired is not a matrix skill', + 'baseline: wrdn-dos-review/gone is not a corpus entry or case', + 'baseline: wrdn-dos-review/bug was recorded at getsentry/sentry@cccccccc, the entry now pins getsentry/sentry@bbbbbbbb', + ]); + }); +}); diff --git a/packages/security-evals/src/validate.ts b/packages/security-evals/src/validate.ts new file mode 100644 index 000000000..18f6ff421 --- /dev/null +++ b/packages/security-evals/src/validate.ts @@ -0,0 +1,80 @@ +import { SUMMARY_TODO, type CaseFile, type SuiteEntry } from './entries.js'; +import type { SuiteResults } from './results.js'; + +const sameTargets = (a: string[], b: string[]): boolean => + a.length === b.length && [...a].sort().every((target, index) => target === [...b].sort()[index]); + +/** + * Check everything about the suite that needs no model and no clone. Returns + * one message per problem; an empty list means cases, corpus, and baseline + * agree. + */ +export function validateSuite(args: { + cases: CaseFile[]; + corpus: SuiteEntry[]; + skills: readonly string[]; + baseline: SuiteResults; +}): string[] { + const { cases, corpus, skills, baseline } = args; + const problems: string[] = []; + + const ids = [...corpus.map((entry) => entry.id), ...cases.map((file) => file.id)]; + for (const id of new Set(ids.filter((id, index) => ids.indexOf(id) !== index))) { + problems.push(`${id}: duplicate entry id`); + } + + for (const { id, owner, data } of cases) { + if (!skills.includes(owner)) { + problems.push(`${id}: cases/${owner} is not a matrix skill (${skills.join(', ')})`); + } + if (data.summary.trim() === SUMMARY_TODO || data.summary.trim().startsWith('TODO')) { + problems.push(`${id}: summary is still the TODO placeholder`); + } + for (const entry of corpus) { + const shared = data.targets.filter((target) => entry.targets.includes(target)); + if (entry.repository === data.repository && entry.ref === data.ref && shared.length > 0) { + problems.push(`${id}: duplicates corpus entry ${entry.id} (${shared.join(', ')} at ${data.ref.slice(0, 8)})`); + } + } + } + + const clean = cases.filter((file) => file.data.kind === 'clean'); + for (const { id, owner, data } of cases.filter((file) => file.data.kind === 'bug')) { + for (const other of clean) { + if (other.data.repository === data.repository && other.data.ref === data.ref + && other.data.targets.some((target) => data.targets.includes(target))) { + problems.push(`${id}: clean case ${other.id} pins the same code`); + } + } + const twins = clean.filter((other) => other.owner === owner && other.data.source === data.source + && other.data.repository === data.repository && other.data.ref !== data.ref + && sameTargets(other.data.targets, data.targets)); + const [twin] = twins; + if (!twin && !data.noClean) { + problems.push(`${id}: no clean twin (same source and targets at the fixed ref) and no noClean reason`); + } + if (twin && data.noClean) { + problems.push(`${id}: has clean twin ${twin.id} and a noClean reason; drop noClean`); + } + for (const twin of twins.filter((other) => other.data.summary !== data.summary)) { + problems.push(`${id}: clean twin ${twin.id} has a different summary; they describe the same bug`); + } + } + + const pinned = new Map(); + for (const entry of corpus) pinned.set(entry.id, entry); + for (const file of cases) pinned.set(file.id, file.data); + for (const skill of Object.keys(baseline.lanes)) { + if (!skills.includes(skill)) problems.push(`baseline: lane ${skill} is not a matrix skill`); + } + for (const [id, recorded] of Object.entries(baseline.entries)) { + const entry = pinned.get(id); + if (!entry) { + problems.push(`baseline: ${id} is not a corpus entry or case`); + } else if (entry.repository !== recorded.repository || entry.ref !== recorded.ref) { + problems.push(`baseline: ${id} was recorded at ${recorded.repository}@${recorded.ref.slice(0, 8)}, ` + + `the entry now pins ${entry.repository}@${entry.ref.slice(0, 8)}`); + } + } + return problems; +} diff --git a/packages/security-evals/src/verdict.test.ts b/packages/security-evals/src/verdict.test.ts new file mode 100644 index 000000000..dcf8910f5 --- /dev/null +++ b/packages/security-evals/src/verdict.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { createResults, recordShardRun, summarize } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; +import { VERDICT_RULES, judgeSkill } from './verdict.js'; + +const SKILL = 'wrdn-dos-review'; +const owned = (count: number) => Array.from({ length: count }, (_, index) => testEntry(`${SKILL}/case-${index}`)); + +/** Five owned bug cases, both lanes at `repeats`, with the given hit counts (out of `repeats`). */ +function matrix(args: { repeats: number; mine?: Record; reference?: Record; cases?: number }) { + const entries = owned(args.cases ?? VERDICT_RULES.minOwnedBugCases); + const results = createResults(entries); + const none = Object.fromEntries(entries.map((entry) => [entry.id, 0])); + recordRepeats(results, 'security-review', { ...none, ...args.reference }, args.repeats); + recordRepeats(results, SKILL, { ...none, ...args.mine }, args.repeats); + return results; +} + +describe('judgeSkill', () => { + it('earns-keep for a bug caught in 2 of 3 repeats that security-review never caught', () => { + const verdict = judgeSkill(matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 2 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'earns-keep', uniqueCatches: [`${SKILL}/case-0`] }); + }); + + it('earns-keep even with few cases, since one real unique catch is enough', () => { + const verdict = judgeSkill(matrix({ repeats: 3, cases: 1, mine: { [`${SKILL}/case-0`]: 3 } }), SKILL); + expect(verdict.verdict).toBe('earns-keep'); + }); + + it('no-unique-catches only with 5 owned bug cases at 3 repeats and nothing it beat security-review on', () => { + const verdict = judgeSkill(matrix({ + repeats: 3, + mine: { [`${SKILL}/case-0`]: 3 }, + reference: { [`${SKILL}/case-0`]: 3 }, + }), SKILL); + expect(verdict).toMatchObject({ verdict: 'no-unique-catches', uniqueCatches: [], ownedBugCases: { evidence: 5 } }); + }); + + it('insufficient-evidence with fewer than 5 owned bug cases', () => { + const verdict = judgeSkill(matrix({ repeats: 3, cases: 4 }), SKILL); + expect(verdict.verdict).toBe('insufficient-evidence'); + expect(verdict.reason).toContain('needs 5'); + }); + + it('insufficient-evidence below 3 repeats, even with 5 cases and no catches', () => { + expect(judgeSkill(matrix({ repeats: 2 }), SKILL).verdict).toBe('insufficient-evidence'); + }); + + it('a single-repeat catch is not a unique catch', () => { + const verdict = judgeSkill(matrix({ repeats: 1, mine: { [`${SKILL}/case-0`]: 1 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', uniqueCatches: [], unsettled: [`${SKILL}/case-0`] }); + }); + + it('a bug it caught once that security-review missed blocks no-unique-catches', () => { + const verdict = judgeSkill(matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 1 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', unsettled: [`${SKILL}/case-0`] }); + }); + + it('a bug security-review caught once is unsettled, not unique', () => { + const verdict = judgeSkill(matrix({ + repeats: 3, + mine: { [`${SKILL}/case-0`]: 3 }, + reference: { [`${SKILL}/case-0`]: 1 }, + }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', uniqueCatches: [], unsettled: [`${SKILL}/case-0`] }); + }); + + it('an errored entry blocks every verdict, including earns-keep', () => { + const results = matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 3 } }); + recordShardRun(results, { + skill: SKILL, config: testConfig(), usage: testUsage(), error: 'boom', hits: new Map(), errors: [`${SKILL}/case-1`], + }); + expect(judgeSkill(results, SKILL)).toMatchObject({ verdict: 'insufficient-evidence', errors: 1 }); + }); + + it('security-review errors also block the verdict', () => { + const results = matrix({ repeats: 3 }); + recordShardRun(results, { + skill: 'security-review', config: testConfig(), usage: testUsage(), hits: new Map(), errors: [`${SKILL}/case-2`], + }); + expect(judgeSkill(results, SKILL).verdict).toBe('insufficient-evidence'); + }); + + it('needs a security-review lane to compare against', () => { + const results = createResults(owned(5)); + recordRepeats(results, SKILL, { [`${SKILL}/case-0`]: 3 }, 3); + expect(judgeSkill(results, SKILL).reason).toContain('no security-review lane'); + }); + + it('counts only bug cases the skill owns toward the case minimum', () => { + const entries = [...owned(4), testEntry('wrdn-authz/other'), testEntry('sentry-vuln-001')]; + const results = createResults(entries); + const none = Object.fromEntries(entries.map((entry) => [entry.id, 0])); + recordRepeats(results, 'security-review', none, 3); + recordRepeats(results, SKILL, none, 3); + expect(judgeSkill(results, SKILL)).toMatchObject({ + verdict: 'insufficient-evidence', + ownedBugCases: { evidence: 4, selected: 4 }, + }); + }); + + it('is attached to every wrdn-* lane summary and never to security-review', () => { + const summary = summarize(matrix({ repeats: 3 })); + expect(summary['security-review']?.verdict).toBeUndefined(); + expect(summary[SKILL]?.verdict?.verdict).toBe('no-unique-catches'); + }); +}); diff --git a/packages/security-evals/src/verdict.ts b/packages/security-evals/src/verdict.ts new file mode 100644 index 000000000..60a70ce99 --- /dev/null +++ b/packages/security-evals/src/verdict.ts @@ -0,0 +1,93 @@ +import type { SuiteResults, Tally } from './results.js'; + +/** The lane every other skill has to beat. */ +export const REFERENCE_SKILL = 'security-review'; + +/** Evidence a skill needs before the suite says anything about keeping it. */ +export const VERDICT_RULES = { + /** Repeats both lanes need on an entry before it counts as evidence. */ + minRepeats: 3, + /** Share of repeats that makes a catch reliable: 2 of 3. */ + reliableHitRate: 2 / 3, + /** Bug cases under `cases//` with full evidence before zero unique catches means anything. */ + minOwnedBugCases: 5, +} as const; + +export type VerdictName = 'earns-keep' | 'no-unique-catches' | 'insufficient-evidence'; + +export interface SkillVerdict { + verdict: VerdictName; + reason: string; + /** Bugs this skill caught reliably that security-review never caught. */ + uniqueCatches: string[]; + /** Bugs this skill caught at least once that security-review did not catch reliably. More repeats settle them. */ + unsettled: string[]; + /** Bug cases the skill owns: how many have full evidence, out of how many were selected. */ + ownedBugCases: { evidence: number; selected: number }; + /** Entry runs that errored in this lane or in security-review. Any error blocks a verdict. */ + errors: number; +} + +const rate = (tally: Tally): number => tally.hits / tally.runs; +const plural = (count: number, word: string): string => `${count} ${word}${count === 1 ? '' : 's'}`; + +/** + * Decide whether a wrdn-* skill catches bugs security-review misses. Only two + * outcomes are conclusive, and both need repeats; anything short of that is + * insufficient evidence, never a quiet zero. + */ +export function judgeSkill(results: SuiteResults, skill: string): SkillVerdict { + const { minRepeats, reliableHitRate, minOwnedBugCases } = VERDICT_RULES; + const entries = Object.entries(results.entries); + const errors = entries.reduce((sum, [, entry]) => sum + + (entry.lanes[skill]?.errors ?? 0) + (entry.lanes[REFERENCE_SKILL]?.errors ?? 0), 0); + + // Bug entries both lanes scored at least once. + const compared = entries.flatMap(([id, entry]) => { + const mine = entry.lanes[skill]; + const reference = entry.lanes[REFERENCE_SKILL]; + return entry.kind === 'bug' && mine && reference && mine.runs > 0 && reference.runs > 0 + ? [{ id, mine, reference, full: mine.runs >= minRepeats && reference.runs >= minRepeats }] + : []; + }); + const uniqueCatches = compared + .filter(({ mine, reference, full }) => full && rate(mine) >= reliableHitRate && reference.hits === 0) + .map(({ id }) => id); + const unsettled = compared + .filter(({ id, mine, reference }) => !uniqueCatches.includes(id) + && mine.hits > 0 && rate(reference) < reliableHitRate) + .map(({ id }) => id); + const owned = entries.filter(([id, entry]) => entry.source === 'case' && entry.kind === 'bug' + && id.startsWith(`${skill}/`)); + const ownedEvidence = compared.filter(({ id, full }) => full && id.startsWith(`${skill}/`)).length; + + const verdict = (name: VerdictName, reason: string): SkillVerdict => ({ + verdict: name, + reason, + uniqueCatches, + unsettled, + ownedBugCases: { evidence: ownedEvidence, selected: owned.length }, + errors, + }); + + if (!results.lanes[REFERENCE_SKILL]) { + return verdict('insufficient-evidence', `no ${REFERENCE_SKILL} lane to compare against`); + } + if (errors > 0) { + return verdict('insufficient-evidence', `${plural(errors, 'entry run')} errored; rerun before reading anything into it`); + } + if (uniqueCatches.length > 0) { + return verdict('earns-keep', `${plural(uniqueCatches.length, 'bug')} caught in at least 2 of 3 repeats ` + + `that ${REFERENCE_SKILL} never caught`); + } + if (unsettled.length > 0) { + return verdict('insufficient-evidence', `${plural(unsettled.length, 'bug')} it caught that ${REFERENCE_SKILL} ` + + `did not catch reliably; more repeats settle ${unsettled.length === 1 ? 'it' : 'them'}`); + } + if (ownedEvidence < minOwnedBugCases) { + return verdict('insufficient-evidence', `${ownedEvidence} owned bug cases with ${minRepeats}+ repeats in both lanes, ` + + `needs ${minOwnedBugCases} (${owned.length} selected)`); + } + return verdict('no-unique-catches', `0 unique catches across ${ownedEvidence} owned bug cases and ` + + `${plural(compared.length, 'bug entry')} overall at ${minRepeats}+ repeats`); +} diff --git a/packages/security-evals/src/worktrees.test.ts b/packages/security-evals/src/worktrees.test.ts index 1a872fe79..9eab66b94 100644 --- a/packages/security-evals/src/worktrees.test.ts +++ b/packages/security-evals/src/worktrees.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'no import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { SENTRY_CHECKOUT, SKILLS_CHECKOUT, ensureWorktree, removeWorktree, resolveCheckout } from './worktrees.js'; +import { SKILLS_CHECKOUT, ensureWorktree, removeWorktree, repoCheckout, resolveCheckout } from './worktrees.js'; function configuredEmail(): string { try { @@ -89,8 +89,9 @@ describe('ensureWorktree', () => { describe('resolveCheckout', () => { it('fails fast without a clone instead of cloning', () => { - expect(() => resolveCheckout(SENTRY_CHECKOUT, undefined)).toThrow('WARDEN_SECURITY_EVALS_SENTRY_REPO (or --sentry-repo)'); + expect(() => resolveCheckout(repoCheckout('getsentry/getsentry'), undefined)) + .toThrow('WARDEN_SECURITY_EVALS_REPOS (or --repo getsentry/getsentry=) must point at a local getsentry/getsentry clone'); expect(() => resolveCheckout(SKILLS_CHECKOUT, undefined)).toThrow('getsentry/warden-skills clone'); - expect(() => resolveCheckout(SENTRY_CHECKOUT, tmpdir())).toThrow('is not a git checkout'); + expect(() => resolveCheckout(repoCheckout('getsentry/sentry'), tmpdir())).toThrow('is not a git checkout'); }); }); diff --git a/packages/security-evals/src/worktrees.ts b/packages/security-evals/src/worktrees.ts index 2c08ba168..876a51d80 100644 --- a/packages/security-evals/src/worktrees.ts +++ b/packages/security-evals/src/worktrees.ts @@ -10,11 +10,10 @@ export interface CheckoutSpec { flag: string; } -export const SENTRY_CHECKOUT: CheckoutSpec = { - repository: 'getsentry/sentry', - env: 'WARDEN_SECURITY_EVALS_SENTRY_REPO', - flag: '--sentry-repo', -}; +/** A scanned repository. Entries name it; `--repo owner/name=` supplies the clone. */ +export function repoCheckout(repository: string): CheckoutSpec { + return { repository, env: 'WARDEN_SECURITY_EVALS_REPOS', flag: `--repo ${repository}=` }; +} export const SKILLS_CHECKOUT: CheckoutSpec = { repository: 'getsentry/warden-skills', From 2d53f7ecd4bef34d98e8decb9ca4e9143d5abaf5 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:22:41 +0000 Subject: [PATCH 06/11] docs(security-evals): Use $HOME in --repo examples Tilde does not expand after = in a plain argument. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/security-evals/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index 925acd6ac..fe526b76f 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -34,10 +34,10 @@ git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills ```bash # Every lane over every entry -pnpm evals:security --repo getsentry/sentry=~/src/sentry --skills-repo ~/src/warden-skills +pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills # One candidate lane on its own cases, three passes; other lanes come from the baseline -pnpm evals:security --repo getsentry/sentry=~/src/sentry --skills-repo ~/src/warden-skills \ +pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills \ --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3 ``` From 64ef275fe5c249707582cfec3d32e5c66894ac80 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:32:21 +0000 Subject: [PATCH 07/11] test(security-evals): Add authz team-role downgrade case Fix commit getsentry/sentry@d714026543 (contributor could demote an org admin's inherited team role). Brings wrdn-authz to 5 bug cases. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/security-evals/README.md | 6 +++--- .../sentry-team-role-downgrade-fail-open-fixed.json | 10 ++++++++++ .../sentry-team-role-downgrade-fail-open.json | 10 ++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json create mode 100644 packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index fe526b76f..f072ca12e 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -165,9 +165,9 @@ adding cases you want compared for free. ## Known gaps -- Owned bug cases today: wrdn-authz 4, wrdn-dos-review 3, wrdn-data-exfil 2, - wrdn-code-execution 1, wrdn-pii 0. Every skill is below 5, so every lane - reads `insufficient-evidence` unless it earns its keep outright. +- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 3, wrdn-data-exfil 2, + wrdn-code-execution 1, wrdn-pii 0. A skill below 5 reads + `insufficient-evidence` unless it earns its keep outright. - The corpus has almost no DoS or exfiltration entries. Cases have to carry those skills. - Next: harvest candidate cases from production Warden PR comments that led diff --git a/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json new file mode 100644 index 000000000..64d28fa8c --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "OrganizationMemberTeamDetailsEndpoint.put treats a member with no explicit team role as freely changeable (can_set_old_role defaults to True), so a team contributor can rewrite the team role of an org admin or manager who inherits team admin from their org role, demoting them on that team", + "repository": "getsentry/sentry", + "ref": "d714026543ec355782969c8e702ca7e9d56251fd", + "targets": [ + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d714026543ec355782969c8e702ca7e9d56251fd" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json new file mode 100644 index 000000000..243b83ba1 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "OrganizationMemberTeamDetailsEndpoint.put treats a member with no explicit team role as freely changeable (can_set_old_role defaults to True), so a team contributor can rewrite the team role of an org admin or manager who inherits team admin from their org role, demoting them on that team", + "repository": "getsentry/sentry", + "ref": "1dd5d51234118842768ed532b75fb2913b28daac", + "targets": [ + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d714026543ec355782969c8e702ca7e9d56251fd" +} From b1bdc42b39633ec17908f377bae4d65306d34ff6 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:35:58 +0000 Subject: [PATCH 08/11] test(security-evals): Add three dos-review cases Fix commits in getsentry/sentry: - a00270ad6d: unbounded RE2 compile cost for logs regex filters - b61768dc55: unbounded commits list processed under the release lock - 1e21009cb0: full=true issue events page serializes 100 full events Brings wrdn-dos-review to 6 bug cases, each with a clean twin. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/security-evals/README.md | 2 +- .../sentry-group-events-full-page-size-fixed.json | 10 ++++++++++ .../sentry-group-events-full-page-size.json | 10 ++++++++++ .../sentry-logs-regex-filter-compile-cost-fixed.json | 10 ++++++++++ .../sentry-logs-regex-filter-compile-cost.json | 10 ++++++++++ .../sentry-release-set-commits-unbounded-fixed.json | 11 +++++++++++ .../sentry-release-set-commits-unbounded.json | 11 +++++++++++ 7 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json create mode 100644 packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index f072ca12e..aea6b023e 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -165,7 +165,7 @@ adding cases you want compared for free. ## Known gaps -- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 3, wrdn-data-exfil 2, +- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 2, wrdn-code-execution 1, wrdn-pii 0. A skill below 5 reads `insufficient-evidence` unless it earns its keep outright. - The corpus has almost no DoS or exfiltration entries. Cases have to carry diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json new file mode 100644 index 000000000..9aee29428 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "GroupEventsEndpoint with full=true binds and serializes the complete nodestore body of every event with EventSerializer at the default per_page of 100, so a single request against native events with many threads and frames builds hundreds of megabytes of JSON and holds a web worker for a long time", + "repository": "getsentry/sentry", + "ref": "1e21009cb0f03dd93714a4ac50356053176e5b6a", + "targets": [ + "src/sentry/issues/endpoints/group_events.py" + ], + "source": "https://github.com/getsentry/sentry/commit/1e21009cb0f03dd93714a4ac50356053176e5b6a" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json new file mode 100644 index 000000000..5e6e4ab59 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "GroupEventsEndpoint with full=true binds and serializes the complete nodestore body of every event with EventSerializer at the default per_page of 100, so a single request against native events with many threads and frames builds hundreds of megabytes of JSON and holds a web worker for a long time", + "repository": "getsentry/sentry", + "ref": "b597a2926d9b321b1da0ffc1018a41c283f4f781", + "targets": [ + "src/sentry/issues/endpoints/group_events.py" + ], + "source": "https://github.com/getsentry/sentry/commit/1e21009cb0f03dd93714a4ac50356053176e5b6a" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json new file mode 100644 index 000000000..7ed380e34 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "event_search.validate_regex_pattern compiles every logs key://pattern// filter with RE2 on the web worker with only a 64-character length cap, no RE2 max_mem and no limit on filters per query, so a short pattern like \\p{L}{100} repeated across many filters makes each search request burn CPU and memory compiling huge programs before any Snuba query runs", + "repository": "getsentry/sentry", + "ref": "a00270ad6dfe4beb9637e21582ad163c3284cfe3", + "targets": [ + "src/sentry/api/event_search.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a00270ad6dfe4beb9637e21582ad163c3284cfe3" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json new file mode 100644 index 000000000..26ae18b93 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "event_search.validate_regex_pattern compiles every logs key://pattern// filter with RE2 on the web worker with only a 64-character length cap, no RE2 max_mem and no limit on filters per query, so a short pattern like \\p{L}{100} repeated across many filters makes each search request burn CPU and memory compiling huge programs before any Snuba query runs", + "repository": "getsentry/sentry", + "ref": "453b3a60dab0cc4c12d1b080a2be39be13acdb37", + "targets": [ + "src/sentry/api/event_search.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a00270ad6dfe4beb9637e21582ad163c3284cfe3" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json new file mode 100644 index 000000000..89bf4dc66 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "set_commits processes every commit in the release create/update commits list with no size limit, issuing several queries per commit inside the web request while holding the release lock for up to 10 minutes, so one request with a huge commits array ties up a worker, the database, and the lock", + "repository": "getsentry/sentry", + "ref": "b61768dc5582d1018ee2ad8a582591ddae110a53", + "targets": [ + "src/sentry/models/releases/set_commits.py", + "src/sentry/api/serializers/rest_framework/release.py" + ], + "source": "https://github.com/getsentry/sentry/commit/b61768dc5582d1018ee2ad8a582591ddae110a53" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json new file mode 100644 index 000000000..2e1171fbe --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "set_commits processes every commit in the release create/update commits list with no size limit, issuing several queries per commit inside the web request while holding the release lock for up to 10 minutes, so one request with a huge commits array ties up a worker, the database, and the lock", + "repository": "getsentry/sentry", + "ref": "bac9728af44bcfd338a4b3e424732a6f5157a451", + "targets": [ + "src/sentry/models/releases/set_commits.py", + "src/sentry/api/serializers/rest_framework/release.py" + ], + "source": "https://github.com/getsentry/sentry/commit/b61768dc5582d1018ee2ad8a582591ddae110a53" +} From 769253ec045c4defff63ffa33202b50cbb6b4c23 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:40:47 +0000 Subject: [PATCH 09/11] test(security-evals): Add three data-exfil cases Fix commits in getsentry/sentry: - dd85c7170d: f-string SQL in the missing org members query - be42554450: caller-chosen ClickHouse function name in fn_span_count - d84bf17e96: SCIM bearer token rendered in full on the SSO settings page Brings wrdn-data-exfil to 5 bug cases, each with a clean twin. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/security-evals/README.md | 2 +- ...cover-fn-span-count-function-injection-fixed.json | 10 ++++++++++ ...ry-discover-fn-span-count-function-injection.json | 10 ++++++++++ .../sentry-missing-members-domain-sqli-fixed.json | 10 ++++++++++ .../sentry-missing-members-domain-sqli.json | 10 ++++++++++ ...try-scim-token-plaintext-settings-page-fixed.json | 12 ++++++++++++ .../sentry-scim-token-plaintext-settings-page.json | 12 ++++++++++++ 7 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json create mode 100644 packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index aea6b023e..a66e25b3e 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -165,7 +165,7 @@ adding cases you want compared for free. ## Known gaps -- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 2, +- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 5, wrdn-code-execution 1, wrdn-pii 0. A skill below 5 reads `insufficient-evidence` unless it earns its keep outright. - The corpus has almost no DoS or exfiltration entries. Cases have to carry diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json new file mode 100644 index 000000000..af88d49aa --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "DiscoverDatasetConfig's fn_span_count declares its fn argument as an SnQLStringArg with no allowed_strings, so the caller's value passes through unchanged and becomes the name of the generated ClickHouse function, letting an events API caller call arbitrary ClickHouse functions in the query Sentry sends to Snuba", + "repository": "getsentry/sentry", + "ref": "be425544501ac37ccc6730d78d39b830bf69021e", + "targets": [ + "src/sentry/search/events/datasets/discover.py" + ], + "source": "https://github.com/getsentry/sentry/commit/be425544501ac37ccc6730d78d39b830bf69021e" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json new file mode 100644 index 000000000..e0286dbee --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "DiscoverDatasetConfig's fn_span_count declares its fn argument as an SnQLStringArg with no allowed_strings, so the caller's value passes through unchanged and becomes the name of the generated ClickHouse function, letting an events API caller call arbitrary ClickHouse functions in the query Sentry sends to Snuba", + "repository": "getsentry/sentry", + "ref": "d336e8ecfccf6334d52d6b3b0845f979ade9997c", + "targets": [ + "src/sentry/search/events/datasets/discover.py" + ], + "source": "https://github.com/getsentry/sentry/commit/be425544501ac37ccc6730d78d39b830bf69021e" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json new file mode 100644 index 000000000..42bbffe7f --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_get_missing_organization_members in organization_missing_org_members.py f-string interpolates shared_domain, the email domain shared by the org owners, into the raw CommitAuthor SQL LIKE clause instead of passing it as a query parameter, so an owner email whose domain carries SQL (via import, relocation, or any path that skips Django email validation) injects into a query against the shared database", + "repository": "getsentry/sentry", + "ref": "dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc", + "targets": [ + "src/sentry/api/endpoints/organization_missing_org_members.py" + ], + "source": "https://github.com/getsentry/sentry/commit/dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json new file mode 100644 index 000000000..e0a5729bd --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_get_missing_organization_members in organization_missing_org_members.py f-string interpolates shared_domain, the email domain shared by the org owners, into the raw CommitAuthor SQL LIKE clause instead of passing it as a query parameter, so an owner email whose domain carries SQL (via import, relocation, or any path that skips Django email validation) injects into a query against the shared database", + "repository": "getsentry/sentry", + "ref": "b74117a6cd90e4d11bc44dc4ef542cebb480857b", + "targets": [ + "src/sentry/api/endpoints/organization_missing_org_members.py" + ], + "source": "https://github.com/getsentry/sentry/commit/dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json new file mode 100644 index 000000000..be09da4b8 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json @@ -0,0 +1,12 @@ +{ + "kind": "clean", + "summary": "organization_auth_settings.handle_existing_provider puts auth_provider.get_scim_token() into the SSO settings page context, and the template renders the full SCIM bearer token in plaintext to every manager-level viewer for as long as SCIM stays enabled, so anyone who can open the page can copy a long-lived credential that provisions and deprovisions org members", + "repository": "getsentry/sentry", + "ref": "d84bf17e962269389a9f0fe6521e2f87c785f31d", + "targets": [ + "src/sentry/web/frontend/organization_auth_settings.py", + "src/sentry/models/authprovider.py", + "src/sentry/auth/services/auth/model.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d84bf17e962269389a9f0fe6521e2f87c785f31d" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json new file mode 100644 index 000000000..d9611910a --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json @@ -0,0 +1,12 @@ +{ + "kind": "bug", + "summary": "organization_auth_settings.handle_existing_provider puts auth_provider.get_scim_token() into the SSO settings page context, and the template renders the full SCIM bearer token in plaintext to every manager-level viewer for as long as SCIM stays enabled, so anyone who can open the page can copy a long-lived credential that provisions and deprovisions org members", + "repository": "getsentry/sentry", + "ref": "6b546fd93d8a2dcd533c655b6415b5f233ac818b", + "targets": [ + "src/sentry/web/frontend/organization_auth_settings.py", + "src/sentry/models/authprovider.py", + "src/sentry/auth/services/auth/model.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d84bf17e962269389a9f0fe6521e2f87c785f31d" +} From 3c6082fd66196d147d5c6075cb89ced69395c0da Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:45:14 +0000 Subject: [PATCH 10/11] test(security-evals): Add six pii cases, clone sentry-python in CI Fix commits: - getsentry/sentry 58b51b430f: UserSerializer returns other users' secondary emails - getsentry/sentry aedeffac61: activity emails ignore enhanced_privacy - getsentry/sentry 2204fde687: AI suggested fix ignores the org opt-out - getsentry/sentry-python f17dad95f3: ASGI query string sent without send_default_pii - getsentry/sentry-python 5606bb3e58: user attributes on logs and metrics without send_default_pii - getsentry/sentry-python e52c3b117a: LangChain tool calls recorded without include_prompts The sentry-python cases need a second clone, so the manual workflow now prepares both with prepare-clone.sh

. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .github/workflows/security-evals.yml | 8 ++++++-- packages/security-evals/README.md | 9 ++++++--- ...-email-ignores-enhanced-privacy-fixed.json | 10 ++++++++++ ...tivity-email-ignores-enhanced-privacy.json | 10 ++++++++++ ...ggested-fix-ignores-org-opt-out-fixed.json | 10 ++++++++++ ...-ai-suggested-fix-ignores-org-opt-out.json | 10 ++++++++++ ...sentry-python-asgi-query-string-fixed.json | 10 ++++++++++ .../sentry-python-asgi-query-string.json | 10 ++++++++++ ...thon-langchain-tool-call-output-fixed.json | 10 ++++++++++ ...try-python-langchain-tool-call-output.json | 10 ++++++++++ ...on-logs-metrics-user-attributes-fixed.json | 10 ++++++++++ ...y-python-logs-metrics-user-attributes.json | 10 ++++++++++ ...ser-serializer-secondary-emails-fixed.json | 10 ++++++++++ ...ntry-user-serializer-secondary-emails.json | 10 ++++++++++ .../security-evals/scripts/prepare-clone.sh | 20 +++++++++++++++++++ .../scripts/prepare-sentry-clone.sh | 14 ------------- 16 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json create mode 100644 packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json create mode 100755 packages/security-evals/scripts/prepare-clone.sh delete mode 100755 packages/security-evals/scripts/prepare-sentry-clone.sh diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml index 6ed0ece3b..3c4d5d36b 100644 --- a/.github/workflows/security-evals.yml +++ b/.github/workflows/security-evals.yml @@ -39,6 +39,7 @@ jobs: timeout-minutes: 360 env: SENTRY_CLONE: ${{ github.workspace }}/../sentry + SENTRY_PYTHON_CLONE: ${{ github.workspace }}/../sentry-python # Must match WARDEN_SKILLS_REF in packages/security-evals/src/options.ts. WARDEN_SKILLS_REF: daf01f970a28450627a8d2d61d9915a742a735e2 SKILLS_CHECKOUT: packages/security-evals/.cache/warden-skills @@ -69,13 +70,16 @@ jobs: restore-keys: sentry-blobless- - name: Prepare sentry clone - run: packages/security-evals/scripts/prepare-sentry-clone.sh + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry "$SENTRY_CLONE" + + - name: Prepare sentry-python clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-python "$SENTRY_PYTHON_CLONE" - name: Run security evals env: WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} - WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }} + WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }},getsentry/sentry-python=${{ env.SENTRY_PYTHON_CLONE }} WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }} WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index a66e25b3e..f97e4b650 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -25,16 +25,19 @@ classes, and permission checks the way they do in production. ## Running it You need a local clone of every repository the selected entries pin (today -only getsentry/sentry) and of getsentry/warden-skills. Blobless is enough: +getsentry/sentry, plus getsentry/sentry-python for some `wrdn-pii` cases) and +of getsentry/warden-skills. Blobless is enough: ```bash git clone --filter=blob:none https://github.com/getsentry/sentry.git ~/src/sentry +git clone --filter=blob:none https://github.com/getsentry/sentry-python.git ~/src/sentry-python git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills ``` ```bash # Every lane over every entry -pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills +pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry \ + --repo getsentry/sentry-python=$HOME/src/sentry-python --skills-repo ~/src/warden-skills # One candidate lane on its own cases, three passes; other lanes come from the baseline pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills \ @@ -166,7 +169,7 @@ adding cases you want compared for free. ## Known gaps - Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 5, - wrdn-code-execution 1, wrdn-pii 0. A skill below 5 reads + wrdn-code-execution 1, wrdn-pii 6. A skill below 5 reads `insufficient-evidence` unless it earns its keep outright. - The corpus has almost no DoS or exfiltration entries. Cases have to carry those skills. diff --git a/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json new file mode 100644 index 000000000..b56595bcb --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "ActivityNotification.get_subject and the generic activity email templates put the issue title and group details in the subject, preheader, and text body even when the organization has enhanced_privacy enabled, so regression and other activity emails ship event-derived titles, which often carry user data, to inboxes the org asked to keep them out of", + "repository": "getsentry/sentry", + "ref": "aedeffac61f00a3a7c39b3df8fc62f263c2686ca", + "targets": [ + "src/sentry/notifications/notifications/activity/base.py" + ], + "source": "https://github.com/getsentry/sentry/commit/aedeffac61f00a3a7c39b3df8fc62f263c2686ca" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json new file mode 100644 index 000000000..ba456e049 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "ActivityNotification.get_subject and the generic activity email templates put the issue title and group details in the subject, preheader, and text body even when the organization has enhanced_privacy enabled, so regression and other activity emails ship event-derived titles, which often carry user data, to inboxes the org asked to keep them out of", + "repository": "getsentry/sentry", + "ref": "660849756331cc9e2a232ee8a2133ea7e2267b29", + "targets": [ + "src/sentry/notifications/notifications/activity/base.py" + ], + "source": "https://github.com/getsentry/sentry/commit/aedeffac61f00a3a7c39b3df8fc62f263c2686ca" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json new file mode 100644 index 000000000..4d3c21c64 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "EventAiSuggestedFixEndpoint.get decides whether to send an event's message and stacktrace to OpenAI from the OpenAI policy and per-request consent flags only and never checks the organization's sentry:ai_suggested_solution option, so event data, including any user data inside it, goes to a third-party model for orgs that turned the feature off", + "repository": "getsentry/sentry", + "ref": "2204fde6870055ee4fcd7551464559282973c188", + "targets": [ + "src/sentry/api/endpoints/event_ai_suggested_fix.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2204fde6870055ee4fcd7551464559282973c188" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json new file mode 100644 index 000000000..977f42cc0 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "EventAiSuggestedFixEndpoint.get decides whether to send an event's message and stacktrace to OpenAI from the OpenAI policy and per-request consent flags only and never checks the organization's sentry:ai_suggested_solution option, so event data, including any user data inside it, goes to a third-party model for orgs that turned the feature off", + "repository": "getsentry/sentry", + "ref": "d8ff0cb911134cf4a8ad24e179247a2e96c06d20", + "targets": [ + "src/sentry/api/endpoints/event_ai_suggested_fix.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2204fde6870055ee4fcd7551464559282973c188" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json new file mode 100644 index 000000000..b8bdfcf69 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_asgi_common._get_request_attributes records http.query and a url.full that carries the query string on every ASGI request span without checking should_send_default_pii, so apps that never opted into PII still ship query strings, which routinely hold emails, tokens, and user IDs, to Sentry", + "repository": "getsentry/sentry-python", + "ref": "f17dad95f33f59aa65bf023a4435e059c4145d37", + "targets": [ + "sentry_sdk/integrations/_asgi_common.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/f17dad95f33f59aa65bf023a4435e059c4145d37" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json new file mode 100644 index 000000000..019fed294 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_asgi_common._get_request_attributes records http.query and a url.full that carries the query string on every ASGI request span without checking should_send_default_pii, so apps that never opted into PII still ship query strings, which routinely hold emails, tokens, and user IDs, to Sentry", + "repository": "getsentry/sentry-python", + "ref": "400963132d9c8fb07ab7b4824e5a44c75cb35dbb", + "targets": [ + "sentry_sdk/integrations/_asgi_common.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/f17dad95f33f59aa65bf023a4435e059c4145d37" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json new file mode 100644 index 000000000..ca73252f1 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "SentryLangchainCallback.on_llm_end records the model's tool_calls, with their arguments, as gen_ai.response.tool_calls span data without checking should_send_default_pii or include_prompts, so apps that disabled prompt capture still send tool-call payloads built from user conversations to Sentry", + "repository": "getsentry/sentry-python", + "ref": "e52c3b117a404fb7755feced02680d29e421b0ee", + "targets": [ + "sentry_sdk/integrations/langchain.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/e52c3b117a404fb7755feced02680d29e421b0ee" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json new file mode 100644 index 000000000..628161f21 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "SentryLangchainCallback.on_llm_end records the model's tool_calls, with their arguments, as gen_ai.response.tool_calls span data without checking should_send_default_pii or include_prompts, so apps that disabled prompt capture still send tool-call payloads built from user conversations to Sentry", + "repository": "getsentry/sentry-python", + "ref": "3f4e2a8497ae942b380ce4d2459cac7bb364506e", + "targets": [ + "sentry_sdk/integrations/langchain.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/e52c3b117a404fb7755feced02680d29e421b0ee" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json new file mode 100644 index 000000000..9e4defad4 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "Client._capture_log and Client._capture_metric copy the isolation scope user's id, username, and email onto every log and metric as user.* attributes without checking should_send_default_pii, so apps with send_default_pii off still send user identities with each log line and metric", + "repository": "getsentry/sentry-python", + "ref": "5606bb3e58e1c27e2c1e48d6dc69e09e30469897", + "targets": [ + "sentry_sdk/client.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/5606bb3e58e1c27e2c1e48d6dc69e09e30469897" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json new file mode 100644 index 000000000..a6f43238b --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "Client._capture_log and Client._capture_metric copy the isolation scope user's id, username, and email onto every log and metric as user.* attributes without checking should_send_default_pii, so apps with send_default_pii off still send user identities with each log line and metric", + "repository": "getsentry/sentry-python", + "ref": "6046f2d04632176654415d5ad16ad17d558a81a3", + "targets": [ + "sentry_sdk/client.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/5606bb3e58e1c27e2c1e48d6dc69e09e30469897" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json new file mode 100644 index 000000000..156a653e1 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "UserSerializer.serialize returns every serialized user's full list of secondary email addresses (with verification state) no matter who is asking, so any endpoint that serializes another user without popping emails, such as organization user details, hands org members each other's private secondary emails", + "repository": "getsentry/sentry", + "ref": "58b51b430f0883cd5560eca37dbb7a8427e5a8c1", + "targets": [ + "src/sentry/users/api/serializers/user.py" + ], + "source": "https://github.com/getsentry/sentry/commit/58b51b430f0883cd5560eca37dbb7a8427e5a8c1" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json new file mode 100644 index 000000000..b9ba7dc3c --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "UserSerializer.serialize returns every serialized user's full list of secondary email addresses (with verification state) no matter who is asking, so any endpoint that serializes another user without popping emails, such as organization user details, hands org members each other's private secondary emails", + "repository": "getsentry/sentry", + "ref": "8b34c7c654c24b2b2a5630097d965044b7d71d57", + "targets": [ + "src/sentry/users/api/serializers/user.py" + ], + "source": "https://github.com/getsentry/sentry/commit/58b51b430f0883cd5560eca37dbb7a8427e5a8c1" +} diff --git a/packages/security-evals/scripts/prepare-clone.sh b/packages/security-evals/scripts/prepare-clone.sh new file mode 100755 index 000000000..ffa5073ec --- /dev/null +++ b/packages/security-evals/scripts/prepare-clone.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Usage: prepare-clone.sh +# Ensure holds a blobless clone of the GitHub repository. A restored +# cache is used as is: the suite fetches any pinned SHA the clone lacks. +set -eu + +if [ "$#" -ne 2 ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi +repository=$1 +dir=$2 + +if [ -d "$dir/.git" ]; then + printf 'Using cached %s clone at %s\n' "$repository" "$dir" + exit 0 +fi + +timeout 1800 git clone --quiet --filter=blob:none --no-checkout \ + "https://github.com/$repository.git" "$dir" diff --git a/packages/security-evals/scripts/prepare-sentry-clone.sh b/packages/security-evals/scripts/prepare-sentry-clone.sh deleted file mode 100755 index 0412a6c02..000000000 --- a/packages/security-evals/scripts/prepare-sentry-clone.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# Ensure $SENTRY_CLONE holds a blobless getsentry/sentry clone. A restored -# cache is used as is: the suite fetches any pinned SHA the clone lacks. -set -eu - -: "${SENTRY_CLONE:?SENTRY_CLONE must name the clone directory}" - -if [ -d "$SENTRY_CLONE/.git" ]; then - printf 'Using cached clone at %s\n' "$SENTRY_CLONE" - exit 0 -fi - -timeout 1800 git clone --quiet --filter=blob:none --no-checkout \ - https://github.com/getsentry/sentry.git "$SENTRY_CLONE" From 107e13c73027cbaca86fd19506f4cb1b5e6ae3e9 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:51:38 +0000 Subject: [PATCH 11/11] test(security-evals): Add two code-execution cases Fix commits: - getsentry/sentry-wizard 9a4f3c9282: changed file names interpolated into formatter shell commands - getsentry/dotagents 461da846ce: skill source URL and ref passed to git without a -- separator The manual workflow clones both repositories. wrdn-code-execution has 3 bug cases; the README lists why it is still short. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .github/workflows/security-evals.yml | 10 +++++++- packages/security-evals/README.md | 24 ++++++++++++------- ...otagents-git-argument-injection-fixed.json | 11 +++++++++ .../dotagents-git-argument-injection.json | 11 +++++++++ ...rd-formatter-filename-injection-fixed.json | 11 +++++++++ ...y-wizard-formatter-filename-injection.json | 11 +++++++++ 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json create mode 100644 packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json create mode 100644 packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json create mode 100644 packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml index 3c4d5d36b..7ce3a9995 100644 --- a/.github/workflows/security-evals.yml +++ b/.github/workflows/security-evals.yml @@ -40,6 +40,8 @@ jobs: env: SENTRY_CLONE: ${{ github.workspace }}/../sentry SENTRY_PYTHON_CLONE: ${{ github.workspace }}/../sentry-python + SENTRY_WIZARD_CLONE: ${{ github.workspace }}/../sentry-wizard + DOTAGENTS_CLONE: ${{ github.workspace }}/../dotagents # Must match WARDEN_SKILLS_REF in packages/security-evals/src/options.ts. WARDEN_SKILLS_REF: daf01f970a28450627a8d2d61d9915a742a735e2 SKILLS_CHECKOUT: packages/security-evals/.cache/warden-skills @@ -75,11 +77,17 @@ jobs: - name: Prepare sentry-python clone run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-python "$SENTRY_PYTHON_CLONE" + - name: Prepare sentry-wizard clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-wizard "$SENTRY_WIZARD_CLONE" + + - name: Prepare dotagents clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/dotagents "$DOTAGENTS_CLONE" + - name: Run security evals env: WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} - WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }},getsentry/sentry-python=${{ env.SENTRY_PYTHON_CLONE }} + WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }},getsentry/sentry-python=${{ env.SENTRY_PYTHON_CLONE }},getsentry/sentry-wizard=${{ env.SENTRY_WIZARD_CLONE }},getsentry/dotagents=${{ env.DOTAGENTS_CLONE }} WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }} WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md index f97e4b650..c079fd35c 100644 --- a/packages/security-evals/README.md +++ b/packages/security-evals/README.md @@ -24,20 +24,25 @@ classes, and permission checks the way they do in production. ## Running it -You need a local clone of every repository the selected entries pin (today -getsentry/sentry, plus getsentry/sentry-python for some `wrdn-pii` cases) and -of getsentry/warden-skills. Blobless is enough: +You need a local clone of every repository the selected entries pin and of +getsentry/warden-skills. Today that is getsentry/sentry for the corpus and most +cases, getsentry/sentry-python for some `wrdn-pii` cases, and +getsentry/sentry-wizard and getsentry/dotagents for some `wrdn-code-execution` +cases. Blobless is enough: ```bash -git clone --filter=blob:none https://github.com/getsentry/sentry.git ~/src/sentry -git clone --filter=blob:none https://github.com/getsentry/sentry-python.git ~/src/sentry-python +for repo in sentry sentry-python sentry-wizard dotagents; do + git clone --filter=blob:none https://github.com/getsentry/$repo.git ~/src/$repo +done git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills ``` ```bash # Every lane over every entry pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry \ - --repo getsentry/sentry-python=$HOME/src/sentry-python --skills-repo ~/src/warden-skills + --repo getsentry/sentry-python=$HOME/src/sentry-python \ + --repo getsentry/sentry-wizard=$HOME/src/sentry-wizard \ + --repo getsentry/dotagents=$HOME/src/dotagents --skills-repo ~/src/warden-skills # One candidate lane on its own cases, three passes; other lanes come from the baseline pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills \ @@ -169,8 +174,11 @@ adding cases you want compared for free. ## Known gaps - Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 5, - wrdn-code-execution 1, wrdn-pii 6. A skill below 5 reads - `insufficient-evidence` unless it earns its keep outright. + wrdn-code-execution 3, wrdn-pii 6. A skill below 5 reads + `insufficient-evidence` unless it earns its keep outright. Code execution is + short because fixed, attacker-reachable command, template, or deserialization + bugs are rare in getsentry history; most "injection" fixes are workflow YAML + or latent hardening with no untrusted input. - The corpus has almost no DoS or exfiltration entries. Cases have to carry those skills. - Next: harvest candidate cases from production Warden PR comments that led diff --git a/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json new file mode 100644 index 000000000..4f75943fc --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "sources/git.ts passes the skill source URL and ref from agents.toml to git clone and git fetch as positional arguments with no -- separator, and the config schema accepts any git: string, so a source or ref starting with --upload-pack= (or another option) turns into a git option that runs an attacker-chosen command when someone installs skills in a repo carrying that agents.toml", + "repository": "getsentry/dotagents", + "ref": "461da846ce520b49b17f66ce00c5859a1a04a644", + "targets": [ + "src/sources/git.ts", + "src/config/schema.ts" + ], + "source": "https://github.com/getsentry/dotagents/commit/461da846ce520b49b17f66ce00c5859a1a04a644" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json new file mode 100644 index 000000000..5d96d96e9 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "sources/git.ts passes the skill source URL and ref from agents.toml to git clone and git fetch as positional arguments with no -- separator, and the config schema accepts any git: string, so a source or ref starting with --upload-pack= (or another option) turns into a git option that runs an attacker-chosen command when someone installs skills in a repo carrying that agents.toml", + "repository": "getsentry/dotagents", + "ref": "f17bb9edbe132b67e1987c74d9991d167b16dcf9", + "targets": [ + "src/sources/git.ts", + "src/config/schema.ts" + ], + "source": "https://github.com/getsentry/dotagents/commit/461da846ce520b49b17f66ce00c5859a1a04a644" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json new file mode 100644 index 000000000..5a10a289f --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "getFormatterTargetFiles joins the changed and untracked file names from git status into one space-separated string, and the formatter step interpolates it into childProcess.exec command lines (npx prettier --write ${targetFiles}, npx @biomejs/biome ...), so a file in the target repo named with shell syntax such as $(touch${IFS}pwned).js runs arbitrary commands on the developer machine when they run the wizard", + "repository": "getsentry/sentry-wizard", + "ref": "9a4f3c9282d4cb47978e01d391b0ecc87f6f437a", + "targets": [ + "src/utils/clack/index.ts", + "src/utils/git.ts" + ], + "source": "https://github.com/getsentry/sentry-wizard/commit/9a4f3c9282d4cb47978e01d391b0ecc87f6f437a" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json new file mode 100644 index 000000000..f6192a904 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "getFormatterTargetFiles joins the changed and untracked file names from git status into one space-separated string, and the formatter step interpolates it into childProcess.exec command lines (npx prettier --write ${targetFiles}, npx @biomejs/biome ...), so a file in the target repo named with shell syntax such as $(touch${IFS}pwned).js runs arbitrary commands on the developer machine when they run the wizard", + "repository": "getsentry/sentry-wizard", + "ref": "0144d69f365c5812f927a692e8433eb20304ce9c", + "targets": [ + "src/utils/clack/index.ts", + "src/utils/git.ts" + ], + "source": "https://github.com/getsentry/sentry-wizard/commit/9a4f3c9282d4cb47978e01d391b0ecc87f6f437a" +}