Fix scope-deleting polls, migration lockout, and cancellation gaps

cmd/db_export.go

@@ -131,6 +131,7 @@ func exportCSV(entries []storage.Entry) error {
 		"base_category",
 		"disabled",
 		"is_ignored",
+		"base_in_scope",
 	}
 	if err := writer.Write(header); err != nil {
 		return err
@@ -153,6 +154,10 @@ func exportCSV(entries []storage.Entry) error {
 			e.BaseCategory,
 			fmt.Sprintf("%t", e.Disabled),
 			fmt.Sprintf("%t", e.IsIgnored),
+			"",
+		}
+		if e.BaseInScope != nil {
+			record[len(record)-1] = fmt.Sprintf("%t", *e.BaseInScope)
 		}
 		if err := writer.Write(record); err != nil {
 			return err

cmd/db_import.go

@@ -208,11 +208,17 @@ func groupEntriesForImport(entries []storage.Entry) ([]programKey, map[programKe
 		items := grouped[key]
 		pos, exists := index[key][identity]
 		if !exists {
+			// An "ai" row's InScope is the variant's effective value; the base
+			// target's own flag travels in BaseInScope (absent in old backups).
+			baseInScope := e.InScope
+			if e.BaseInScope != nil {
+				baseInScope = *e.BaseInScope
+			}
 			items = append(items, storage.TargetItem{
 				URI:         raw,
 				Category:    baseCat,
 				Description: e.Description,
-				InScope:     e.InScope,
+				InScope:     baseInScope,
 				IsBBP:       e.IsBBP,
 			})
 			pos = len(items) - 1
@@ -234,7 +240,8 @@ func groupEntriesForImport(entries []storage.Entry) ([]programKey, map[programKe
 				v.HasCategory = true
 				v.Category = e.Category
 			}
-			v.HasInScope = true
+			// Only an effective value that differs from the base is an override.
+			v.HasInScope = e.BaseInScope == nil || *e.BaseInScope != e.InScope
 			v.InScope = e.InScope
 			item.Variants = append(item.Variants, v)
 		}
@@ -381,6 +388,12 @@ func parseimportCSV(r io.Reader) ([]storage.Entry, error) {
 			inScope = parseCSVBool(raw)
 		}
 
+		var baseInScope *bool
+		if raw := field(record, "base_in_scope"); raw != "" {
+			b := parseCSVBool(raw)
+			baseInScope = &b
+		}
+
 		entries = append(entries, storage.Entry{
 			ProgramURL:       field(record, "program_url"),
 			Platform:         field(record, "platform"),
@@ -390,6 +403,7 @@ func parseimportCSV(r io.Reader) ([]storage.Entry, error) {
 			Category:         field(record, "category"),
 			BaseCategory:     field(record, "base_category"),
 			BaseTargetRaw:    field(record, "base_target_raw"),
+			BaseInScope:      baseInScope,
 			Description:      field(record, "description"),
 			InScope:          inScope,
 			IsBBP:            parseCSVBool(field(record, "is_bbp")),

cmd/db_roundtrip_test.go

@@ -372,3 +372,59 @@ func TestIntegration_ImportMergesWithoutDeletingLiveTargets(t *testing.T) {
 		t.Fatalf("replace import should leave only the file contents, got %#v", got)
 	}
 }
+
+// An AI row's in_scope is the variant's effective value. Import used it as the
+// base target's in_scope, so an in-scope target with an out-of-scope variant
+// came back out of scope.
+func TestIntegration_ImportKeepsBaseInScopeUnderVariantOverride(t *testing.T) {
+	db, raw := openRoundtripDB(t)
+	ctx := context.Background()
+	platform := "itest_basescope_" + strconv.FormatInt(time.Now().UnixNano(), 36)
+	programURL := "https://example.com/" + platform + "/a"
+	cleanup := func() {
+		_, _ = raw.ExecContext(ctx, `DELETE FROM targets_ai_enhanced WHERE target_id IN (
+			SELECT tr.id FROM targets_raw tr JOIN programs p ON tr.program_id = p.id WHERE p.platform = $1)`, platform)
+		_, _ = raw.ExecContext(ctx, `DELETE FROM targets_raw WHERE program_id IN (SELECT id FROM programs WHERE platform = $1)`, platform)
+		_, _ = raw.ExecContext(ctx, `DELETE FROM scope_changes WHERE platform = $1`, platform)
+		_, _ = raw.ExecContext(ctx, `DELETE FROM programs WHERE platform = $1`, platform)
+	}
+	cleanup()
+	t.Cleanup(cleanup)
+
+	built, err := storage.BuildEntries(programURL, platform, "a", []storage.TargetItem{{
+		URI: "*.base.example.com", Category: "wildcard", InScope: true,
+		Variants: []storage.TargetVariant{{Value: "legacy.base.example.com", HasInScope: true, InScope: false}},
+	}})
+	if err != nil {
+		t.Fatalf("BuildEntries: %v", err)
+	}
+	if _, err := db.UpsertProgramEntriesWithOptions(ctx, programURL, platform, "a", built,
+		storage.UpsertOptions{SkipChangeLog: true}); err != nil {
+		t.Fatalf("seeding: %v", err)
+	}
+	exported, err := db.ListEntries(ctx, storage.ListOptions{Platform: platform, IncludeOOS: true})
+	if err != nil {
+		t.Fatalf("ListEntries: %v", err)
+	}
+
+	cleanup()
+	if _, failed := importEntries(ctx, db, exported, false); failed != 0 {
+		t.Fatalf("import failed for %d entries", failed)
+	}
+
+	var baseInScope int
+	var variantInScope sql.NullInt64
+	if err := raw.QueryRowContext(ctx, `
+		SELECT tr.in_scope, a.in_scope FROM targets_raw tr
+		JOIN programs p ON tr.program_id = p.id
+		JOIN targets_ai_enhanced a ON a.target_id = tr.id
+		WHERE p.platform = $1`, platform).Scan(&baseInScope, &variantInScope); err != nil {
+		t.Fatalf("reading restored target: %v", err)
+	}
+	if baseInScope != 1 {
+		t.Errorf("base in_scope = %d after round trip, want 1", baseInScope)
+	}
+	if !variantInScope.Valid || variantInScope.Int64 != 0 {
+		t.Errorf("variant in_scope = %+v after round trip, want 0", variantInScope)
+	}
+}

cmd/exit_codes_test.go

@@ -3,17 +3,26 @@ package cmd
 import (
 	"context"
 	"errors"
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"github.com/zalando/go-keyring"
 )
 
+// TestMain swaps in the in-memory keyring so tests never read a developer's
+// real credentials or block on a desktop Secret Service unlock prompt.
+func TestMain(m *testing.M) {
+	keyring.MockInit()
+	os.Exit(m.Run())
+}
+
 // runPollSubcommand invokes one poll subcommand's RunE directly with no
 // credentials available. Going through Execute would traverse up to the root
 // command and print its help instead of running this subcommand. Credentials
-// resolve through viper here because the test environment has no OS keychain.
+// resolve through viper here because TestMain mocks the OS keychain empty.
 func runPollSubcommand(t *testing.T, cmd *cobra.Command) error {
 	t.Helper()
 	viper.Reset()

cmd/poll.go

@@ -41,18 +41,27 @@ var pollCmd = &cobra.Command{
 		proxyURL, _ := cmd.Flags().GetString("proxy")
 		// Parent poll includes all configured platforms (including Immunefi).
 		pollers, err := buildPollersFromConfig(cmd.Context(), proxyURL, nil)
-		if err != nil {
-			return err
-		}
-		if len(pollers) == 0 {
-			cmd.SilenceUsage = true
-			return fmt.Errorf("no platforms to poll; configure credentials or use a platform subcommand")
-		}
-
-		return runPollWithPollers(cmd, pollers)
+		return pollAvailable(cmd, pollers, err)
 	},
 }
 
+// pollAvailable polls every poller that was built even when others failed to
+// authenticate, so one banned or expired login cannot skip the remaining
+// platforms. The build error is still returned to keep the exit code non-zero.
+func pollAvailable(cmd *cobra.Command, pollers []platforms.PlatformPoller, buildErr error) error {
+	if len(pollers) == 0 {
+		if buildErr != nil {
+			return buildErr
+		}
+		cmd.SilenceUsage = true
+		return fmt.Errorf("no platforms to poll; configure credentials or use a platform subcommand")
+	}
+	if buildErr != nil {
+		utils.Log.Warnf("Polling the platforms that authenticated; others failed: %v", buildErr)
+	}
+	return errors.Join(buildErr, runPollWithPollers(cmd, pollers))
+}
+
 func init() {
 	rootCmd.AddCommand(pollCmd)
 
@@ -235,7 +244,9 @@ func runPollWithPollers(cmd *cobra.Command, pollers []platforms.PlatformPoller)
 			continue
 		}
 
-		if useDB {
+		if useDB && pollIsFiltered(opts) {
+			utils.Log.Infof("Scope filters are active; skipping removal sync for %s so filtered-out scope is kept", p.Name())
+		} else if useDB {
 			// After processing all programs for a platform, sync the state.
 			// This will mark any programs that were not in the latest poll as disabled.
 			removedProgramChanges, err := db.SyncPlatformPrograms(ctx, p.Name(), polledProgramURLs)
@@ -261,6 +272,14 @@ func runPollWithPollers(cmd *cobra.Command, pollers []platforms.PlatformPoller)
 	return errors.Join(runErrs...)
 }
 
+// pollIsFiltered reports whether the poll returns a filtered subset of the
+// platform's scope. A subset must be merged, never reconciled: treating it as
+// the full scope deletes every filtered-out target and disables every
+// filtered-out program.
+func pollIsFiltered(opts platforms.PollOptions) bool {
+	return opts.BountyOnly || opts.PrivateOnly || scope.GetAllStringsForCategories(opts.Categories) != nil
+}
+
 // emptyListingWouldWipe reports whether syncing an empty handle list would
 // disable programs already stored for the platform. Any non-zero stored count
 // is enough: a one-program platform is exactly the case a truncated listing
@@ -346,51 +365,52 @@ func processProgramsConcurrently(ctx context.Context, cmd *cobra.Command, p plat
 				var aiCandidates []storage.TargetItem
 				var aiEnhancements map[string][]storage.TargetVariant
 
-				if aiNormalizer != nil && len(allItems) > 0 {
+				// Always reattach stored AI variants, not only under --ai: the
+				// upsert treats an entry without variants as "none desired" and
+				// would delete every stored variant on a plain --db poll.
+				if len(allItems) > 0 {
 					var err error
 					aiEnhancements, err = db.ListAIEnhancements(ctx, pd.Url)
 					if err != nil {
-						utils.Log.Warnf("Failed to load AI enhancements for %s: %v", pd.Url, err)
-						aiEnhancements = nil
+						utils.Log.Warnf("Failed to load AI enhancements for %s: %v; skipping program to keep stored variants", pd.Url, err)
+						errorMu.Lock()
+						if firstError == nil {
+							firstError = err
+						}
+						errorMu.Unlock()
+						continue
 					}
 				}
 
-				if aiNormalizer != nil && len(allItems) > 0 {
-					aiCandidates = make([]storage.TargetItem, 0, len(allItems))
-					for _, item := range allItems {
-						key := storage.BuildTargetCategoryKey(item.URI, item.Category)
-						if variants, ok := aiEnhancements[key]; ok && len(variants) > 0 {
-							clone := item
-							clone.Variants = append([]storage.TargetVariant(nil), variants...)
-							processedItems = append(processedItems, clone)
-							continue
-						}
-						aiCandidates = append(aiCandidates, item)
+				for _, item := range allItems {
+					key := storage.BuildTargetCategoryKey(item.URI, item.Category)
+					if variants := aiEnhancements[key]; len(variants) > 0 {
+						clone := item
+						clone.Variants = append([]storage.TargetVariant(nil), variants...)
+						processedItems = append(processedItems, clone)
+						continue
 					}
-
-					if len(aiCandidates) > 0 {
-						normalized, err := aiNormalizer.NormalizeTargets(ctx, ai.ProgramInfo{
-							ProgramURL: pd.Url,
-							Platform:   p.Name(),
-							Handle:     h,
-						}, aiCandidates)
-						if err != nil {
-							utils.Log.Warnf("AI normalization failed for %s: %v", pd.Url, err)
-							processedItems = append(processedItems, aiCandidates...)
-						} else if len(normalized) > 0 {
-							processedItems = append(processedItems, normalized...)
-						} else {
-							processedItems = append(processedItems, aiCandidates...)
-						}
+					if aiNormalizer == nil {
+						processedItems = append(processedItems, item)
+						continue
 					}
+					aiCandidates = append(aiCandidates, item)
+				}
 
-					// if there were no candidates but also no pre-existing enhancements,
-					// ensure raw items still get processed
-					if len(processedItems) == 0 {
-						processedItems = append(processedItems, allItems...)
+				if len(aiCandidates) > 0 {
+					normalized, err := aiNormalizer.NormalizeTargets(ctx, ai.ProgramInfo{
+						ProgramURL: pd.Url,
+						Platform:   p.Name(),
+						Handle:     h,
+					}, aiCandidates)
+					if err != nil {
+						utils.Log.Warnf("AI normalization failed for %s: %v", pd.Url, err)
+						processedItems = append(processedItems, aiCandidates...)
+					} else if len(normalized) > 0 {
+						processedItems = append(processedItems, normalized...)
+					} else {
+						processedItems = append(processedItems, aiCandidates...)
 					}
-				} else if len(processedItems) == 0 {
-					processedItems = append(processedItems, allItems...)
 				}
 
 				entries, err := storage.BuildEntries(pd.Url, p.Name(), h, processedItems)
@@ -412,14 +432,20 @@ func processProgramsConcurrently(ctx context.Context, cmd *cobra.Command, p plat
 					p.Name(),
 					h,
 					entries,
-					storage.UpsertOptions{SkipChangeLog: isFirstRunForPlatform},
+					storage.UpsertOptions{SkipChangeLog: isFirstRunForPlatform, MergeOnly: pollIsFiltered(opts)},
 				)
 
 				if err != nil {
 					if errors.Is(err, storage.ErrAbortingScopeWipe) {
 						utils.Log.Warnf("Potential scope wipe detected for program %s. Skipping update. This might be due to a broken poller or a platform API change.", pd.Url)
 						continue // Don't treat this as a fatal error for the whole poll
 					}
+					if errors.Is(err, storage.ErrProgramURLOwned) {
+						// e.g. `db add -u` stored this URL as a custom program. Failing the
+						// platform here would skip its sync on every future run.
+						utils.Log.Warnf("Skipping program %s: %v", pd.Url, err)
+						continue
+					}
 					// For other errors, log but continue processing
 					utils.Log.Warnf("Database error for program %s: %v", pd.Url, err)
 					errorMu.Lock()