From d3146dac070ab62b9c9889b25422e609807581cb Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 4 Jun 2026 11:28:27 -0400 Subject: [PATCH 01/22] test From 824a1a8f127758194f7241180b54decbc137e3b2 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Fri, 12 Jun 2026 12:01:48 -0400 Subject: [PATCH 02/22] create new project on pipeline run --- azure-pipelines-templates/run-tests.yml | 50 ++++++++++++++++++------- tests/base.py | 2 +- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index 1288f112..fffb0e97 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -123,20 +123,42 @@ jobs: # This will give a user name like 'something macOS 2.7' SG_HUMAN_NAME: $(python_api_human_name) ${{ parameters.os_name }} ${{ parameters.python_version }} SG_HUMAN_PASSWORD: $(python_api_human_password) - # So, first, we need to make sure that two builds running at the same time do not manipulate - # the same entities, so we're sandboxing build nodes based on their name. - SG_PROJECT_NAME: Python API CI - $(Agent.Name) - # The entities created and then reused between tests assume that the same user is always - # manipulating them. Because different builds will be assigned different agents and therefore - # different projects, it means each project needs to have an entity specific to a given user. - # Again, this would have been a lot simpler if we could simply have had a login based on the - # agent name, but alas, the agent name has a space in it which needs to be replaced to something - # else and string substitution can't be made on build variables, only template parameters. - SG_ASSET_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_VERSION_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_SHOT_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_TASK_CONTENT: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} + # Each job gets its own ephemeral project, scoped to this build + OS + Python version. + # This eliminates state bleed between concurrent runs and across successive builds on the + # same agent. The project is retired in the "Cleanup test project" step below. + SG_PROJECT_NAME: Python API CI - $(Build.BuildId) - ${{ parameters.os_name }} - ${{ parameters.python_version }} + # Entity codes only need to be unique within the project, so fixed strings are fine. + SG_ASSET_CODE: CI-asset + SG_VERSION_CODE: CI-version + SG_SHOT_CODE: CI-shot + SG_TASK_CONTENT: CI-task + SG_PLAYLIST_CODE: CI-playlist + + - task: Bash@3 + displayName: Cleanup test project + condition: always() + inputs: + targetType: inline + script: | + python -c " + import os, shotgun_api3 + sg = shotgun_api3.Shotgun( + os.environ['SG_SERVER_URL'], + os.environ['SG_SCRIPT_NAME'], + os.environ['SG_API_KEY'], + ) + project = sg.find_one('Project', [['name', 'is', os.environ['SG_PROJECT_NAME']]]) + if project: + sg.delete('Project', project['id']) + print('Retired project:', os.environ['SG_PROJECT_NAME']) + else: + print('Project not found, nothing to clean up.') + " + env: + SG_SERVER_URL: $(ci_site) + SG_SCRIPT_NAME: $(ci_site_script_name) + SG_API_KEY: $(ci_site_script_key) + SG_PROJECT_NAME: Python API CI - $(Build.BuildId) - ${{ parameters.os_name }} - ${{ parameters.python_version }} # Explicit call to PublishTestResults@2 and PublishCodeCoverageResults@2 here # instead of relying on pytest-azurepipelines because pytest-azurepipelines diff --git a/tests/base.py b/tests/base.py index eea47fad..fc0c9778 100644 --- a/tests/base.py +++ b/tests/base.py @@ -289,7 +289,7 @@ def _setup_db(cls, config, sg): cls.human_user = _find_or_create_entity(sg, "HumanUser", data) data = {"code": cls.config.asset_code, "project": cls.project} - keys = ["code"] + keys = ["code", "project"] cls.asset = _find_or_create_entity(sg, "Asset", data, keys) data = { From 5b0cfc3bb4e91f88218b46f3425bfdc7ce26b75e Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 16 Jun 2026 17:06:29 -0400 Subject: [PATCH 03/22] flaky test fix attempt --- tests/test_api.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index d0e8407e..81dcd3e5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3329,22 +3329,24 @@ def test_modify_visibility(self): project_1 = projects[0] project_2 = projects[1] + def assert_visibility(project, expected, retries=5, delay=1): + """Poll until schema_field_read reflects the expected visibility value.""" + result = None + for _ in range(retries): + result = self.sg.schema_field_read("Asset", field_name, project)[ + field_name + ]["visible"] + if result == expected: + return + time.sleep(delay) + self.assertEqual(expected, result) + # First, reset the field visibility in a known state, i.e. visible for both projects, # in case the last test run failed midway through. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": True, "editable": True}) self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_2) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_2)[field_name][ - "visible" - ], - ) + assert_visibility(project_2, {"value": True, "editable": True}) # Built-in fields should remain not editable. self.assertFalse( @@ -3360,12 +3362,7 @@ def test_modify_visibility(self): # Hide the field on project 1 self.sg.schema_field_update("Asset", field_name, {"visible": False}, project_1) # It should not be visible anymore. - self.assertEqual( - {"value": False, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": False, "editable": True}) # The field should be visible on the second project. self.assertEqual( @@ -3377,12 +3374,7 @@ def test_modify_visibility(self): # Restore the visibility on the field. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": True, "editable": True}) class TestLibImports(base.LiveTestBase): From 53c39be8dd2034556db20ff9376472066c7077f4 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Wed, 17 Jun 2026 10:00:35 -0400 Subject: [PATCH 04/22] re-run flaky tests --- azure-pipelines-templates/run-tests.yml | 2 ++ tests/requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index fffb0e97..be290641 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -102,6 +102,8 @@ jobs: --durations=0 \ --nunit-xml=test-results.xml \ --verbose \ + --reruns 1 \ + --reruns-delay 2 \ env: # Tell Pytest that we're running in a CI environment CI: 1 diff --git a/tests/requirements.txt b/tests/requirements.txt index 82f6c626..4df3c018 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,3 +11,4 @@ pytest pytest-cov pytest-nunit +pytest-rerunfailures From e72e6cdb0fd458b5de2242a12615fd584f2cc89a Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Wed, 17 Jun 2026 11:41:03 -0400 Subject: [PATCH 05/22] CI test 1 From fa96a4baaac396b41dbd2a757f0c94c507e9447c Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Wed, 17 Jun 2026 11:42:26 -0400 Subject: [PATCH 06/22] CI test 2 From bc58c711a06ff611a0d8bf24a59206d4f5458ed4 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Wed, 17 Jun 2026 11:42:48 -0400 Subject: [PATCH 07/22] CI test 3 From bd27f37f946cb69fbfb326e71d9354bccfe5fc48 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 13:24:56 -0400 Subject: [PATCH 08/22] SG-43825 use ephemeral project as control for schema visibility tests --- tests/test_api.py | 50 ++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 81dcd3e5..f0cd0e1d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3322,31 +3322,27 @@ def test_modify_visibility(self): if field_name not in schema: self.sg.schema_field_create("Asset", "text", "Project Visibility Test") - # Grab any two projects that we can use for toggling the visible property with. - projects = self.sg.find( - "Project", [], order=[{"field_name": "id", "direction": "asc"}] - ) - project_1 = projects[0] - project_2 = projects[1] - - def assert_visibility(project, expected, retries=5, delay=1): - """Poll until schema_field_read reflects the expected visibility value.""" - result = None - for _ in range(retries): - result = self.sg.schema_field_read("Asset", field_name, project)[ - field_name - ]["visible"] - if result == expected: - return - time.sleep(delay) - self.assertEqual(expected, result) + # Hide/show on the ephemeral CI project so concurrent matrix jobs do not race. + # Use the oldest site project as the control. + project_1 = self.project + project_2 = self.sg.find_one("Project", []) # First, reset the field visibility in a known state, i.e. visible for both projects, # in case the last test run failed midway through. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - assert_visibility(project_1, {"value": True, "editable": True}) + self.assertEqual( + {"value": True, "editable": True}, + self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ + "visible" + ], + ) self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_2) - assert_visibility(project_2, {"value": True, "editable": True}) + self.assertEqual( + {"value": True, "editable": True}, + self.sg.schema_field_read("Asset", field_name, project_2)[field_name][ + "visible" + ], + ) # Built-in fields should remain not editable. self.assertFalse( @@ -3362,7 +3358,12 @@ def assert_visibility(project, expected, retries=5, delay=1): # Hide the field on project 1 self.sg.schema_field_update("Asset", field_name, {"visible": False}, project_1) # It should not be visible anymore. - assert_visibility(project_1, {"value": False, "editable": True}) + self.assertEqual( + {"value": False, "editable": True}, + self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ + "visible" + ], + ) # The field should be visible on the second project. self.assertEqual( @@ -3374,7 +3375,12 @@ def assert_visibility(project, expected, retries=5, delay=1): # Restore the visibility on the field. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - assert_visibility(project_1, {"value": True, "editable": True}) + self.assertEqual( + {"value": True, "editable": True}, + self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ + "visible" + ], + ) class TestLibImports(base.LiveTestBase): From 3f89923aa8416833102ac0a0caeedcd80c4eb770 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 14:00:28 -0400 Subject: [PATCH 09/22] CI test 4 From e9f22c6854f7d9bb1e120d8ba46627608e56ecde Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 14:00:38 -0400 Subject: [PATCH 10/22] CI test 5 From dd6c045d95b0f2705a3229f3ee83a6a8a8a48b90 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 14:00:55 -0400 Subject: [PATCH 11/22] CI test 6 From ec99632a704a0e768f87667b71f8e6ac33e8e22a Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 14:29:31 -0400 Subject: [PATCH 12/22] CI test 7 From 1bd63727748501abcd69ca1b2d3659a66d18a1d3 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 15:01:08 -0400 Subject: [PATCH 13/22] CI test 8 From 28d55c02fb5bc0791817a29e5cae52f8f1af9b0b Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Tue, 23 Jun 2026 15:01:18 -0400 Subject: [PATCH 14/22] CI test 9 From 975d46d26467d19adbe75f673408245c76844128 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 14:00:19 -0400 Subject: [PATCH 15/22] use task: PythonScript@0 --- azure-pipelines-templates/run-tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index be290641..88d87d7f 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -136,13 +136,12 @@ jobs: SG_TASK_CONTENT: CI-task SG_PLAYLIST_CODE: CI-playlist - - task: Bash@3 + - task: PythonScript@0 displayName: Cleanup test project condition: always() inputs: - targetType: inline + scriptSource: inline script: | - python -c " import os, shotgun_api3 sg = shotgun_api3.Shotgun( os.environ['SG_SERVER_URL'], @@ -155,7 +154,6 @@ jobs: print('Retired project:', os.environ['SG_PROJECT_NAME']) else: print('Project not found, nothing to clean up.') - " env: SG_SERVER_URL: $(ci_site) SG_SCRIPT_NAME: $(ci_site_script_name) From 31ecccdd7c772732eb03afdf5c6324614b2ceff9 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 14:04:36 -0400 Subject: [PATCH 16/22] remove useless entity code overrides (using ephemeral projects anyways) --- azure-pipelines-templates/run-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index 88d87d7f..cac2667a 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -129,12 +129,6 @@ jobs: # This eliminates state bleed between concurrent runs and across successive builds on the # same agent. The project is retired in the "Cleanup test project" step below. SG_PROJECT_NAME: Python API CI - $(Build.BuildId) - ${{ parameters.os_name }} - ${{ parameters.python_version }} - # Entity codes only need to be unique within the project, so fixed strings are fine. - SG_ASSET_CODE: CI-asset - SG_VERSION_CODE: CI-version - SG_SHOT_CODE: CI-shot - SG_TASK_CONTENT: CI-task - SG_PLAYLIST_CODE: CI-playlist - task: PythonScript@0 displayName: Cleanup test project From 30103a39158741ae4331894a68d88beee561ed96 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 14:09:01 -0400 Subject: [PATCH 17/22] make sure projects are different for test_modify_visibility --- tests/test_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index f0cd0e1d..bbcfb6a0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3323,9 +3323,13 @@ def test_modify_visibility(self): self.sg.schema_field_create("Asset", "text", "Project Visibility Test") # Hide/show on the ephemeral CI project so concurrent matrix jobs do not race. - # Use the oldest site project as the control. + # Use the oldest site project as the control (find_one defaults to sorting by "id" ascending) project_1 = self.project - project_2 = self.sg.find_one("Project", []) + project_2 = self.sg.find_one("Project", [["id", "is_not", project_1["id"]]]) + self.assertIsNotNone( + project_2, + "A second project is required to test per-project field visibility.", + ) # First, reset the field visibility in a known state, i.e. visible for both projects, # in case the last test run failed midway through. From 8ff127dbd68f58c89fcae1f55b04b777bb9a549e Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 14:53:25 -0400 Subject: [PATCH 18/22] fix ModuleNotFound using PythonScript@0 --- azure-pipelines-templates/run-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index cac2667a..a4bc97de 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -135,6 +135,7 @@ jobs: condition: always() inputs: scriptSource: inline + workingDirectory: $(Build.SourcesDirectory) script: | import os, shotgun_api3 sg = shotgun_api3.Shotgun( @@ -149,6 +150,7 @@ jobs: else: print('Project not found, nothing to clean up.') env: + PYTHONPATH: $(Build.SourcesDirectory) SG_SERVER_URL: $(ci_site) SG_SCRIPT_NAME: $(ci_site_script_name) SG_API_KEY: $(ci_site_script_key) From a3d814607e7f9a6668e1b9a6537fab24d5297816 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 14:55:22 -0400 Subject: [PATCH 19/22] use named paramters when available --- azure-pipelines-templates/run-tests.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index a4bc97de..ee08afbf 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -139,13 +139,16 @@ jobs: script: | import os, shotgun_api3 sg = shotgun_api3.Shotgun( - os.environ['SG_SERVER_URL'], - os.environ['SG_SCRIPT_NAME'], - os.environ['SG_API_KEY'], + base_url=os.environ['SG_SERVER_URL'], + script_name=os.environ['SG_SCRIPT_NAME'], + api_key=os.environ['SG_API_KEY'], + ) + project = sg.find_one( + entity_type='Project', + filters=[['name', 'is', os.environ['SG_PROJECT_NAME']]], ) - project = sg.find_one('Project', [['name', 'is', os.environ['SG_PROJECT_NAME']]]) if project: - sg.delete('Project', project['id']) + sg.delete(entity_type='Project', entity_id=project['id']) print('Retired project:', os.environ['SG_PROJECT_NAME']) else: print('Project not found, nothing to clean up.') From 4d16f8da26222bc8c106852e43ef4e07b9e32ad6 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Thu, 25 Jun 2026 15:00:03 -0400 Subject: [PATCH 20/22] fail fast on post-job project cleanup --- azure-pipelines-templates/run-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index ee08afbf..4d379edb 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -147,11 +147,11 @@ jobs: entity_type='Project', filters=[['name', 'is', os.environ['SG_PROJECT_NAME']]], ) - if project: - sg.delete(entity_type='Project', entity_id=project['id']) - print('Retired project:', os.environ['SG_PROJECT_NAME']) - else: + if not project: print('Project not found, nothing to clean up.') + sys.exit(0) + sg.delete(entity_type='Project', entity_id=project['id']) + print('Retired project:', os.environ['SG_PROJECT_NAME']) env: PYTHONPATH: $(Build.SourcesDirectory) SG_SERVER_URL: $(ci_site) From 0ff6f44991813aea0b03a7d9bc4dc90d2bc71af2 Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Fri, 26 Jun 2026 08:53:26 -0400 Subject: [PATCH 21/22] SG-43825 add sys import for project cleanup --- azure-pipelines-templates/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index 4d379edb..3856a5ba 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -137,7 +137,7 @@ jobs: scriptSource: inline workingDirectory: $(Build.SourcesDirectory) script: | - import os, shotgun_api3 + import os, shotgun_api3, sys sg = shotgun_api3.Shotgun( base_url=os.environ['SG_SERVER_URL'], script_name=os.environ['SG_SCRIPT_NAME'], From 6203060e5e54dd119c45fe827bf56f92daed489e Mon Sep 17 00:00:00 2001 From: Jay Roebuck Date: Fri, 26 Jun 2026 13:19:40 -0400 Subject: [PATCH 22/22] SG-43825 cleanup Shotgun init in run-tests.yml --- azure-pipelines-templates/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index 3856a5ba..bfc4e949 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -139,7 +139,7 @@ jobs: script: | import os, shotgun_api3, sys sg = shotgun_api3.Shotgun( - base_url=os.environ['SG_SERVER_URL'], + os.environ['SG_SERVER_URL'], script_name=os.environ['SG_SCRIPT_NAME'], api_key=os.environ['SG_API_KEY'], )