Support pagination when retrieving GitHub installations and repositories

backend/services/authService.js

@@ -16,6 +16,7 @@ const cookieSession = require('cookie-session');
 const passport = require('passport');
 const GitHubStrategy = require('passport-github2').Strategy;
 const { Octokit } = require('@octokit/rest');
+const { collectAllGitHubPages, findInGitHubPages } = require('./githubPagination');
 
 const GOOGLE_NOT_AVAILABLE = 'Google auth is not available until Phase 3';
 
@@ -131,18 +132,29 @@ class AuthService {
       }
     }
 
-    const { data } = await userOctokit.rest.apps.listInstallationsForAuthenticatedUser({
-      per_page: 100,
+    const installations = await collectAllGitHubPages(async (page, perPage) => {
+      const { data } = await userOctokit.rest.apps.listInstallationsForAuthenticatedUser({
+        per_page: perPage,
+        page,
+      });
+      return data.installations ?? [];
     });
 
-    for (const installation of data.installations ?? []) {
-      const { data: reposData } =
-        await userOctokit.rest.apps.listInstallationReposForAuthenticatedUser({
-          installation_id: installation.id,
-          per_page: 100,
-        });
+    for (const installation of installations) {
+      const matched = await findInGitHubPages(
+        async (page, perPage) => {
+          const { data: reposData } =
+            await userOctokit.rest.apps.listInstallationReposForAuthenticatedUser({
+              installation_id: installation.id,
+              per_page: perPage,
+              page,
+            });
+          return reposData.repositories ?? [];
+        },
+        (entry) => entry.full_name === fullName,
+      );
 
-      if (reposData.repositories?.some((entry) => entry.full_name === fullName)) {
+      if (matched) {
         return installation.id;
       }
     }

backend/services/githubPagination.js

@@ -0,0 +1,77 @@
+/**
+ * Shared GitHub list pagination helpers.
+ *
+ * GitHub caps each page at 100 items. Callers that previously fetched only
+ * page 1 silently dropped everything after that — these helpers walk pages
+ * until a short/empty page, keeping the common ≤100 case as a single request.
+ */
+
+const DEFAULT_PER_PAGE = 100;
+const DEFAULT_MAX_PAGES = 50;
+
+/**
+ * Collect every item across paginated GitHub list responses.
+ *
+ * @template T
+ * @param {(page: number, perPage: number) => Promise<T[]>} fetchPage
+ * @param {{ perPage?: number, maxPages?: number }} [options]
+ * @returns {Promise<T[]>}
+ */
+async function collectAllGitHubPages(fetchPage, options = {}) {
+  const perPage = options.perPage ?? DEFAULT_PER_PAGE;
+  const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+  /** @type {T[]} */
+  const all = [];
+
+  for (let page = 1; page <= maxPages; page += 1) {
+    const items = await fetchPage(page, perPage);
+    if (!Array.isArray(items) || items.length === 0) {
+      break;
+    }
+    all.push(...items);
+    if (items.length < perPage) {
+      break;
+    }
+  }
+
+  return all;
+}
+
+/**
+ * Scan paginated results until `predicate` matches, then stop.
+ * Prefer this when looking for a single repo so large installations
+ * do not force a full crawl after a hit.
+ *
+ * @template T
+ * @param {(page: number, perPage: number) => Promise<T[]>} fetchPage
+ * @param {(item: T) => boolean} predicate
+ * @param {{ perPage?: number, maxPages?: number }} [options]
+ * @returns {Promise<T | null>}
+ */
+async function findInGitHubPages(fetchPage, predicate, options = {}) {
+  const perPage = options.perPage ?? DEFAULT_PER_PAGE;
+  const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+
+  for (let page = 1; page <= maxPages; page += 1) {
+    const items = await fetchPage(page, perPage);
+    if (!Array.isArray(items) || items.length === 0) {
+      return null;
+    }
+    const hit = items.find(predicate);
+    if (hit) {
+      return hit;
+    }
+    if (items.length < perPage) {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+module.exports = {
+  DEFAULT_PER_PAGE,
+  DEFAULT_MAX_PAGES,
+  collectAllGitHubPages,
+  findInGitHubPages,
+};

backend/services/storageService.js

@@ -8,6 +8,7 @@
 const crypto = require('crypto');
 const { randomUUID } = require('crypto');
 const { normalizeGitHubRepoName } = require('../../shared/githubRepoName');
+const { collectAllGitHubPages, findInGitHubPages } = require('./githubPagination');
 
 const MANIFEST_PATH = 'vizably.json';
 /** Pre-rename store root — still loadable; rewritten to `MANIFEST_PATH` on load. */
@@ -41,11 +42,15 @@ class StorageService {
    * @returns {Promise<Array<{ id: string, full_name: string, private: boolean, html_url: string }>>}
    */
   async listGitHubRepos(githubClient) {
-    const { data } = await githubClient.rest.repos.listForAuthenticatedUser({
-      visibility: 'all',
-      affiliation: 'owner,collaborator,organization_member',
-      per_page: 100,
-      sort: 'updated',
+    const data = await collectAllGitHubPages(async (page, perPage) => {
+      const { data: pageData } = await githubClient.rest.repos.listForAuthenticatedUser({
+        visibility: 'all',
+        affiliation: 'owner,collaborator,organization_member',
+        per_page: perPage,
+        page,
+        sort: 'updated',
+      });
+      return pageData;
     });
 
     return data.map((repo) => ({
@@ -223,35 +228,46 @@ class StorageService {
    */
   async _isRepoOnWritableInstallation(octokit, owner, repo) {
     const fullName = `${owner}/${repo}`;
-    let data;
+    let installations;
     try {
-      ({ data } = await octokit.rest.apps.listInstallationsForAuthenticatedUser({
-        per_page: 100,
-      }));
+      installations = await collectAllGitHubPages(async (page, perPage) => {
+        const { data } = await octokit.rest.apps.listInstallationsForAuthenticatedUser({
+          per_page: perPage,
+          page,
+        });
+        return data.installations ?? [];
+      });
     } catch (err) {
       throw this._formatGitHubInstallationProbeError(err);
     }
 
-    for (const installation of data.installations ?? []) {
+    for (const installation of installations) {
       if (installation.permissions?.contents !== 'write') {
         continue;
       }
       if (installation.repository_selection === 'all') {
         return true;
       }
 
-      let reposData;
+      let matched;
       try {
-        ({ data: reposData } =
-          await octokit.rest.apps.listInstallationReposForAuthenticatedUser({
-            installation_id: installation.id,
-            per_page: 100,
-          }));
+        matched = await findInGitHubPages(
+          async (page, perPage) => {
+            const { data: reposData } =
+              await octokit.rest.apps.listInstallationReposForAuthenticatedUser({
+                installation_id: installation.id,
+                per_page: perPage,
+                page,
+              });
+            return reposData.repositories ?? [];
+          },
+          (entry) => entry.full_name === fullName,
+        );
       } catch (err) {
         throw this._formatGitHubInstallationProbeError(err);
       }
 
-      if (reposData.repositories?.some((r) => r.full_name === fullName)) {
+      if (matched) {
         return true;
       }
     }
@@ -1195,24 +1211,33 @@ class StorageService {
     let canWrite = false;
 
     try {
-      const { data } = await octokit.rest.apps.listInstallationsForAuthenticatedUser({
-        per_page: 100,
+      const installations = await collectAllGitHubPages(async (page, perPage) => {
+        const { data } = await octokit.rest.apps.listInstallationsForAuthenticatedUser({
+          per_page: perPage,
+          page,
+        });
+        return data.installations ?? [];
       });
 
-      for (const installation of data.installations ?? []) {
+      for (const installation of installations) {
         const contents = installation.permissions?.contents;
         if (!contents || contents === 'none') {
           continue;
         }
 
-        const { data: reposData } =
-          await octokit.rest.apps.listInstallationReposForAuthenticatedUser({
-            installation_id: installation.id,
-            per_page: 100,
-          });
-
-        const included = reposData.repositories?.some((r) => r.full_name === fullName);
-        if (!included) {
+        const matched = await findInGitHubPages(
+          async (page, perPage) => {
+            const { data: reposData } =
+              await octokit.rest.apps.listInstallationReposForAuthenticatedUser({
+                installation_id: installation.id,
+                per_page: perPage,
+                page,
+              });
+            return reposData.repositories ?? [];
+          },
+          (entry) => entry.full_name === fullName,
+        );
+        if (!matched) {
           continue;
         }
 

backend/tests/authService.test.js

@@ -246,3 +246,49 @@ test('getInstallationClientForRepo requires storageRef.id', async () => {
   );
   assert.equal(reposGetCalled, false);
 });
+
+test('_findInstallationIdForRepo paginates user installations and repos', async () => {
+  const authService = new AuthService({
+    sessionSecret: TEST_SESSION_SECRET,
+    encryptionKey: TEST_ENCRYPTION_KEY,
+  });
+
+  const installations = Array.from({ length: 101 }, (_, i) => ({ id: i + 1 }));
+  const reposForTarget = Array.from({ length: 101 }, (_, i) => ({
+    full_name: i === 100 ? 'sam/site-audits' : `sam/other-${i}`,
+  }));
+  const calls = { installations: [], repos: [] };
+
+  const userOctokit = {
+    rest: {
+      apps: {
+        listInstallationsForAuthenticatedUser: async ({ page = 1, per_page = 100 } = {}) => {
+          calls.installations.push({ page, per_page });
+          const start = (page - 1) * per_page;
+          return {
+            data: { installations: installations.slice(start, start + per_page) },
+          };
+        },
+        listInstallationReposForAuthenticatedUser: async ({
+          installation_id,
+          page = 1,
+          per_page = 100,
+        }) => {
+          calls.repos.push({ installation_id, page, per_page });
+          if (installation_id !== 101) {
+            return { data: { repositories: [] } };
+          }
+          const start = (page - 1) * per_page;
+          return {
+            data: { repositories: reposForTarget.slice(start, start + per_page) },
+          };
+        },
+      },
+    },
+  };
+
+  const id = await authService._findInstallationIdForRepo(userOctokit, 'sam/site-audits');
+  assert.equal(id, 101);
+  assert.equal(calls.installations.length, 2);
+  assert.ok(calls.repos.some((c) => c.installation_id === 101 && c.page === 2));
+});