feat(gapic-generator): add support for resumable uploads

.github/workflows/docs.yml

@@ -15,17 +15,18 @@ name: docs
 permissions:
   contents: read
 
+env:
+  NOX_DEFAULT_VENV_BACKEND: "uv"
+  UV_VENV_SEED: "1"
+
 jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -34,7 +35,7 @@ jobs:
     - name: Install nox
       run: |
         python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox
+        python -m pip install nox uv
     - name: Run docs
       env:
         BUILD_TYPE: presubmit
@@ -50,11 +51,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -63,7 +61,7 @@ jobs:
     - name: Install nox
       run: |
         python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox
+        python -m pip install nox uv
     - name: Run docfx
       env:
         BUILD_TYPE: presubmit

.github/workflows/gapic-generator-tests.yml

@@ -13,9 +13,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  SHOWCASE_VERSION: 0.35.0
+  SHOWCASE_VERSION: 0.44.0
   PROTOC_VERSION: 33.5
   LATEST_STABLE_PYTHON: 3.14
+  OLDEST_PYTHON: '3.10'
   PRERELEASE_PYTHON: 3.15
   ALL_PYTHON: "['3.10', '3.11', '3.12', '3.13', '3.14', '3.15']"
   TRIMMED_PYTHON: "['3.10', '3.14', '3.15']"
@@ -50,6 +51,7 @@ jobs:
     outputs:
       all_python: ${{ env.ALL_PYTHON }}
       latest_stable_python: ${{ env.LATEST_STABLE_PYTHON }}
+      oldest_python: ${{ env.OLDEST_PYTHON }}
       prerelease_python: ${{ env.PRERELEASE_PYTHON }}
       trimmed_python: ${{ env.TRIMMED_PYTHON }}
     steps:
@@ -94,6 +96,57 @@ jobs:
           cd packages/gapic-generator
           nox -s showcase_unit${{ matrix.variant }}-${MATRIX_PYTHON}
 
+  showcase:
+    needs: python_config
+    strategy:
+      # Run showcase tests on the lowest and highest supported runtimes
+      matrix:
+        # TODO(https://github.com/googleapis/gapic-generator-python/issues/2121) Remove `showcase_w_rest_async` target when async rest is GA.
+        python: ["${{ needs.python_config.outputs.oldest_python }}", "${{ needs.python_config.outputs.latest_stable_python }}"]
+        target: [showcase, showcase_w_rest_async]
+        logging_scope: ["", "google"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+      - name: Set up Python "${{ matrix.python }}"
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "${{ matrix.python }}"
+          cache: 'pip'
+      - name: Install system dependencies.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl pandoc unzip
+      - name: Install showcase
+        run: |
+          sudo mkdir -p /usr/src/showcase
+          sudo chown -R ${USER} /usr/src/
+          curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
+          cd /usr/src/showcase/
+          tar -xf showcase-*
+          ./gapic-showcase run &
+          cd -
+      - name: Install nox.
+        run: python -m pip install nox
+      - name: Install protoc
+        run: |
+            sudo mkdir -p /usr/src/protoc/
+            sudo chown -R ${USER} /usr/src/
+            curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc-${PROTOC_VERSION}.zip
+            cd /usr/src/protoc/
+            unzip protoc-${PROTOC_VERSION}.zip
+            sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+      - name: Run showcase tests.
+        env:
+          # TODO(https://github.com/googleapis/gapic-generator-python/issues/2286): Construct nox sessions with logging enabled.
+          GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
+          MATRIX_PYTHON: ${{ matrix.python }}
+        run: |
+          cd packages/gapic-generator
+          nox -s ${{ matrix.target }}-${MATRIX_PYTHON}
+
   showcase-mypy:
     needs: python_config
     runs-on: ubuntu-latest

.github/workflows/lint.yml

@@ -68,11 +68,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Check for unit_test:all_packages label
       id: check-label
@@ -133,11 +130,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6

.github/workflows/unittest.yml

@@ -113,11 +113,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -187,11 +184,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -247,11 +241,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
       with:
-        fetch-depth: 2
+        fetch-depth: 0
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5

ci/run_conditional_tests.sh

@@ -44,6 +44,16 @@ TARGET_BRANCH="${TARGET_BRANCH:-main}"
 git config --global url."${PROJECT_ROOT}".insteadOf "https://github.com/googleapis/google-cloud-python"
 git config --global url."${PROJECT_ROOT}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
 
+# Ensure remote-tracking branches (e.g. origin/feat/...) are also exposed under
+# refs/heads/* in PROJECT_ROOT so that `pip` clones redirected via `insteadOf`
+# can resolve branch references.
+for ref in $(git for-each-ref --format='%(refname:short)' refs/remotes/origin/ 2>/dev/null); do
+    branch="${ref#origin/}"
+    if [ "${branch}" != "HEAD" ] && ! git show-ref --verify --quiet "refs/heads/${branch}"; then
+        git branch "${branch}" "${ref}" 2>/dev/null || true
+    fi
+done
+
 # A script file for running the test in a sub project.
 test_script="${PROJECT_ROOT}/ci/run_single_test.sh"
 
@@ -54,7 +64,7 @@ elif [[ ${BUILD_TYPE} == "presubmit" ]]; then
     # For presubmit build, we want to know the difference from the
     # common commit in the target branch.
     if [ -n "${TARGET_BRANCH}" ]; then
-        if [[ "${TEST_TYPE}" == "import_profile" ]]; then
+        if [[ "${TEST_TYPE}" == "import_profile" ]] || [ ! -f "$(git rev-parse --git-dir)/shallow" ]; then
             git fetch --no-tags --quiet origin "${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" || true
         else
             git fetch --no-tags --quiet origin "${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" --depth=200 || true
@@ -186,6 +196,7 @@ AVAIL_CORES=$(nproc 2>/dev/null || echo 4)
 if [ "${PARALLEL_WORKERS}" -gt "${AVAIL_CORES}" ]; then
     PARALLEL_WORKERS="${AVAIL_CORES}"
 fi
+export PARALLEL_WORKERS
 
 echo "Running tests across ${#dirs_to_test[@]} package(s) using ${PARALLEL_WORKERS} parallel worker(s)..."
 printf "%s\0" "${dirs_to_test[@]}" | xargs -0 -P "${PARALLEL_WORKERS}" -I {} bash -c 'run_test_in_dir "$@"' _ {}

packages/gapic-generator/WORKSPACE

@@ -21,6 +21,15 @@ http_archive(
     urls = ["https://github.com/googleapis/googleapis/archive/{0}.zip".format(_googleapis_sha256)],
 )
 
+# Load com_google_gapic_showcase used for testing
+_gapic_showcase_sha256 = "5158967e8a41c751b15776142d190428ddb7e99f"
+http_archive(
+    name = "com_google_gapic_showcase",
+    build_file = "//tests/integration:showcase.BUILD.bazel",
+    strip_prefix = "gapic-showcase-{}".format(_gapic_showcase_sha256),
+    urls = ["https://github.com/googleapis/gapic-showcase/archive/{}.zip".format(_gapic_showcase_sha256)],
+)
+
 # Load rules_gapic v1.0.0. We patch rules_gapic's .bzl files to inject the
 # explicit load for Starlark ProtoInfo from @rules_proto to maintain symbol
 # resolution under rules_proto / Protobuf 35.1.

packages/gapic-generator/gapic/generator/generator.py

@@ -352,12 +352,14 @@ def _render_template(
                         and not api_schema.all_library_settings[
                             api_schema.naming.proto_package
                         ].python_settings.experimental_features.rest_async_io_enabled
+                        and not service.has_resumable_upload_methods
                     )
                     or (
                         "rest_asyncio" in template_name
                         and not api_schema.all_library_settings[
                             api_schema.naming.proto_package
                         ].python_settings.experimental_features.rest_async_io_enabled
+                        and not service.has_resumable_upload_methods
                     )
                     or ("rest_base" in template_name and "rest" not in opts.transport)
                 ):

packages/gapic-generator/gapic/samplegen/samplegen.py

@@ -1183,14 +1183,20 @@ def _get_sample_imports(sample: Dict, rpc: wrappers.Method) -> List[str]:
     module_name = sample["module_name"]
     module_import = f"from {module_namespace} import {module_name}"
 
+    imports = [module_import]
     address = rpc.input.meta.address
     # This checks if the request message is part of the service proto package.
     # If not, we should try to include a separate import statement.
-    if address.proto_package.startswith(address.api_naming.proto_package):
-        return [module_import]
-    else:
-        request_import = str(address.python_import)
-        return sorted([module_import, request_import])
+    if not address.proto_package.startswith(address.api_naming.proto_package):
+        imports.append(str(address.python_import))
+
+    if rpc.is_resumable_upload:
+        imports.append(
+            "from google.api_core.resumable_transfer import ResumableUploadConfig"
+        )
+        imports.append("import io")
+
+    return sorted(imports)
 
 
 def generate_sample(
@@ -1222,6 +1228,7 @@ def generate_sample(
 
     calling_form = types.CallingForm.method_default(rpc)
     sample["is_internal"] = rpc.is_internal
+    sample["is_resumable_upload"] = rpc.is_resumable_upload
 
     v = Validator(rpc, api_schema)
     # Tweak some small aspects of the sample to set defaults for optional

packages/gapic-generator/gapic/schema/api.py

@@ -1065,6 +1065,13 @@ def has_operations_mixin(self) -> bool:
             > 0
         )
 
+    @cached_property
+    def has_resumable_upload_methods(self) -> bool:
+        return any(
+            service.has_resumable_upload_methods
+            for service in self.services.values()
+        )
+
     @cached_property
     def _has_iam_overrides(self) -> bool:
         if not self.has_iam_mixin:
@@ -1345,7 +1352,8 @@ def _load_children(
             wrapped = loader(
                 child, address=address, path=path + (i,), resources=resources
             )
-            answer[wrapped.name] = wrapped
+            if wrapped is not None:
+                answer[wrapped.name] = wrapped
         return answer
 
     def _get_oneofs(
@@ -1651,6 +1659,7 @@ def _get_methods(
                 output=self.api_messages[meth_pb.output_type.lstrip(".")],
                 retry=retry,
                 timeout=timeout,
+                resumable_upload_prefix=self.opts.resumable_upload_prefix,
             )
 
         # Done; return the answer.

packages/gapic-generator/gapic/schema/wrappers.py

@@ -1499,6 +1499,7 @@ class Method:
     meta: metadata.Metadata = dataclasses.field(
         default_factory=metadata.Metadata,
     )
+    resumable_upload_prefix: str = "resumable/upload"
 
     def __getattr__(self, name):
         return getattr(self.method_pb, name)
@@ -1728,6 +1729,29 @@ def http_opt(self) -> Optional[Dict[str, str]]:
         # TODO(yon-mg): enums for http verbs?
         return answer
 
+    @property
+    def is_resumable_upload(self) -> bool:
+        """Return True if this method is a resumable upload method."""
+        try:
+            if hasattr(self, "options") and self.options:
+                http = self.options.Extensions[annotations_pb2.http]
+                if getattr(http, "media_upload", None) and getattr(http.media_upload, "enabled", False):
+                    return True
+                for binding in getattr(http, "additional_bindings", ()):
+                    if getattr(binding, "media_upload", None) and getattr(binding.media_upload, "enabled", False):
+                        return True
+        except Exception:
+            pass
+
+        # TODO: TEMPORARY - Remove this hardcoded fallback once
+        # the media_upload annotation is published and added to gapic-showcase proto.
+        pb_name = getattr(self.method_pb, "name", "")
+        method_name = getattr(self, "name", "")
+        if pb_name == "UploadMedia" or method_name == "upload_media":
+            return True
+
+        return False
+
     @property
     def path_params(self) -> Sequence[str]:
         """Return the path parameters found in the http annotation path template"""
@@ -2208,6 +2232,11 @@ def has_pagers(self) -> bool:
         """Return whether the service has paged methods."""
         return any(m.paged_result_field for m in self.methods.values())
 
+    @property
+    def has_resumable_upload_methods(self) -> bool:
+        """Return whether the service has resumable upload methods."""
+        return any(m.is_resumable_upload for m in self.methods.values())
+
     @property
     def host(self) -> str:
         """Return the hostname for this service, if specified.
@@ -2241,7 +2270,7 @@ def shortname(self) -> str:
         # Get the shortname from the host
         # Real APIs are expected to have format:
         # "{api_shortname}.googleapis.com"
-        return self.host.split(".")[0]
+        return self.host.split(".")[0].split(":")[0]
 
     @property
     def oauth_scopes(self) -> Sequence[str]: