Fall back to a pre-release when no final release is in range

CHANGELOG.md

@@ -13,6 +13,12 @@ served it.
 
 ## [Unreleased]
 
+### Fixed
+
+- **When no final release satisfies a requirement's range, a pre-release is now chosen, as pip
+  and uv do.** A final release in range is still preferred. Some resolutions that used to fail or
+  backtrack now pick a pre-release or a newer version instead.
+
 ## [0.11.0] - 2026-09-18
 
 ### Breaking

candidate/doc.go

@@ -6,12 +6,16 @@
 // Two separable things live here, and keeping them separate is the point:
 //
 //   - Admission is a yes/no about a single version, decided once before
-//     solving so it cannot move while the solver backtracks. Today that is the
-//     pre-release rule: see PrereleaseSet.
+//     solving so it cannot move while the solver backtracks.
 //   - Ranking is a total order over the admissible versions, supplied by the
 //     caller through Policy so that an embedder -- Package Manager, say -- can
 //     demote versions it would rather not install without making them
-//     unavailable. Newest is the default.
+//     unavailable. Newest is the default. The pre-release rule lives here too:
+//     see PrereleaseSet. A package's pre-releases rank alongside its finals if
+//     the caller enabled it, and after every final release otherwise, so a
+//     pre-release is chosen only when nothing final in range is usable -- the
+//     pip/uv in-range fallback. Nothing is ever inadmissible for being a
+//     pre-release.
 //
 // # Ranking must never remove a version
 //

candidate/prerelease.go

@@ -8,10 +8,11 @@ import (
 	"github.com/posit-dev/go-python-packaging/version"
 )
 
-// PrereleaseSet records which packages may have pre-release versions offered.
+// PrereleaseSet records which packages rank their pre-releases alongside
+// their final releases, rather than after all of them.
 //
 // Keys are PEP 503-canonical names. A package absent from the set, or present
-// with a false value, gets final releases only.
+// with a false value, still offers its pre-releases -- only after its finals.
 type PrereleaseSet map[index.PackageName]bool
 
 // EnabledPrereleases derives the set from the requirements a resolution starts
@@ -23,11 +24,18 @@ type PrereleaseSet map[index.PackageName]bool
 // canonicalized here; allow is assumed to hold values already built with
 // index.NewPackageName.
 //
+// For an enabled package, its pre-releases are ranked right alongside its
+// final releases by Policy. For a package that is not enabled, Admits still
+// says every version exists (see below); the caller ranks its pre-releases
+// after every final release instead, so a pre-release is only ever chosen
+// there when nothing final is usable. That in-range fallback matches pip and
+// current uv.
+//
 // The set is computed ONCE, before solving, and must not be recomputed as the
-// solver narrows a package's allowed range. That is what makes pre-release
-// admission a fact about a version rather than a fact about the current search
-// state, and go-pubgrub caches derivations on the assumption that the facts
-// behind them do not move.
+// solver narrows a package's allowed range. That is what keeps a version's
+// pre-release status a fact about the version rather than a fact about the
+// current search state, and go-pubgrub caches derivations on the assumption
+// that the facts behind them do not move.
 //
 // ⚠️ Detection uses Specifiers.PreReleases, NOT Specifiers.FilterVersions, and
 // the difference is not stylistic. FilterVersions implements pip's fuller rule,
@@ -37,6 +45,8 @@ type PrereleaseSet map[index.PackageName]bool
 // admit a pre-release under one range and reject it under a wider one, and a
 // cached incompatibility derived from the earlier answer would then be wrong.
 // Do not call FilterVersions here, and do not call it from Candidates either.
+// The pip/uv in-range fallback described above is implemented as ranking
+// instead, precisely so it can react to the range without moving admission.
 //
 // Note that "!=1.0a1" does not enable pre-releases even though it names one,
 // and neither does "==1.*"; both match pypa/packaging's own derivation, which
@@ -59,17 +69,15 @@ func EnabledPrereleases(reqs []requirement.Requirement, allow []index.PackageNam
 	return set
 }
 
-// Admits reports whether v may be offered for pkg.
-//
-// Every final release is admitted; a pre-release is admitted only for a
-// package the set enables. "Pre-release" here is version.IsPreRelease, so a
-// development release (2.0.dev1) counts and a post-release (2.0.post1) does
-// not -- confirmed against go-python-packaging v0.5.0 rather than assumed.
-//
-// This is admission, not ranking: a version this method rejects is one the
-// resolution genuinely may not use, and the reason does not change while the
-// solver runs. Anything that merely makes a version less desirable belongs in
-// a Policy, where it cannot make the version look nonexistent.
+// Admits now decides ORDERING, not admission: every version of pkg is
+// admissible for existence, whatever this returns. What it decides is
+// whether v should be ranked no worse than pkg's final releases. A final
+// release always is; a pre-release is only for a package the set enables --
+// everyone else's pre-releases get ranked after every final release, so one
+// is picked only when nothing final is usable (see EnabledPrereleases).
+// "Pre-release" here is version.IsPreRelease, so a development release
+// (2.0.dev1) counts and a post-release (2.0.post1) does not -- confirmed
+// against go-python-packaging v0.5.0 rather than assumed.
 func (s PrereleaseSet) Admits(pkg index.PackageName, v version.Version) bool {
 	if !v.IsPreRelease() {
 		return true

provider/export_test.go

@@ -73,25 +73,58 @@ func (p *Provider) ExactCandidates(pkg Package, allowed pep440set.Set) (pep440se
 		return pep440set.Empty(), false, 0, err
 	}
 
-	admissible := make([]version.Version, 0, len(all))
+	// Admission is range alone now -- nothing is excluded for being a
+	// pre-release, so every in-range version is tested for usability.
+	// finalsInRange is the SAME raw count Candidates uses for rank, kept
+	// separately so the count returned below can mirror it exactly rather
+	// than merely bound it.
+	inRange := make([]version.Version, 0, len(all))
+	finalsInRange := 0
 	for _, v := range all {
-		if !allowed.Contains(v) || !p.opts.Prereleases.Admits(pkg.Name, v) {
+		if !allowed.Contains(v) {
 			continue
 		}
+		inRange = append(inRange, v)
+		if !v.IsPreRelease() {
+			finalsInRange++
+		}
+	}
+
+	ranked := finalsFirst(pkg.Name, rankBySortRef(pkg.Name, inRange, p.opts.Policy), p.opts.Prereleases)
+
+	var (
+		best                    version.Version
+		found                   bool
+		finalsUsable, preUsable int
+	)
+	for _, v := range ranked {
 		ok, err := p.usable(pkg, v)
 		if err != nil {
 			return pep440set.Empty(), false, 0, err
 		}
-		if ok {
-			admissible = append(admissible, v)
+		if !ok {
+			continue
+		}
+		if v.IsPreRelease() {
+			preUsable++
+		} else {
+			finalsUsable++
+		}
+		if !found {
+			best, found = v, true
 		}
 	}
-	if len(admissible) == 0 {
+	if !found {
 		return pep440set.Empty(), false, 0, nil
 	}
-
-	ranked := rankBySortRef(pkg.Name, admissible, p.opts.Policy)
-	return pep440set.Exactly(ranked[0]), true, len(ranked), nil
+	// Mirrors Candidates' own rank rule (provider.go) exactly, not merely an
+	// upper bound of it: finalsInRange (raw) here equals the count Candidates
+	// itself computes, so finalsUsable/preUsable can never exceed it.
+	count := finalsUsable
+	if finalsInRange == 0 {
+		count = preUsable
+	}
+	return pep440set.Exactly(best), true, count, nil
 }
 
 // InRangeRanked is the ranked in-range version list Candidates walks, before any
@@ -109,10 +142,10 @@ func (p *Provider) InRangeRanked(pkg Package, allowed pep440set.Set) ([]version.
 
 	inRange := make([]version.Version, 0, len(all))
 	for _, v := range all {
-		if !allowed.Contains(v) || !p.opts.Prereleases.Admits(pkg.Name, v) {
-			continue
+		if allowed.Contains(v) {
+			inRange = append(inRange, v)
 		}
-		inRange = append(inRange, v)
 	}
-	return rankBySortRef(pkg.Name, inRange, p.opts.Policy), nil
+	ranked := rankBySortRef(pkg.Name, inRange, p.opts.Policy)
+	return finalsFirst(pkg.Name, ranked, p.opts.Prereleases), nil
 }

provider/provider.go

@@ -251,20 +251,23 @@ func (p *Provider) Candidates(pkg Package, allowed pep440set.Set) (pep440set.Set
 	// a weakening of the rule the error path exists for: nothing is being reported as
 	// unavailable on the strength of an outage. It is simply not being looked at.
 	var (
-		best    version.Version
-		found   bool
-		inRange int
+		best                             version.Version
+		found                            bool
+		finalsInRange, prereleaseInRange int
 	)
 	for _, v := range ranked {
-		// The cheap half of admission: range and pre-release policy, no metadata
-		// and no I/O. What passes both is what rank counts.
+		// The cheap half of admission is range alone now: rankedVersions already
+		// ordered pre-releases after finals for a package that is not enabled, so
+		// nothing needs skipping here -- the fallback is a ranking effect, not an
+		// admission one. See candidate.PrereleaseSet.
 		if !allowed.Contains(v) {
 			continue
 		}
-		if !p.opts.Prereleases.Admits(pkg.Name, v) {
-			continue
+		if v.IsPreRelease() {
+			prereleaseInRange++
+		} else {
+			finalsInRange++
 		}
-		inRange++
 		if found {
 			// best is settled; the rest of the walk only counts, which costs no
 			// metadata read.
@@ -285,7 +288,18 @@ func (p *Provider) Candidates(pkg Package, allowed pep440set.Set) (pep440set.Set
 		// everything.
 		return pep440set.Empty(), false, 0, nil
 	}
-	return pep440set.Exactly(best), true, inRange, nil
+	// rank counts finals only when there are any, and pre-releases otherwise --
+	// NOT their sum -- so a package's rank does not grow just because it also
+	// publishes pre-releases the walk above would never prefer while a final is
+	// usable. See the package's pre-release fallback in rankedVersions; without
+	// this every package with pre-releases would report a larger rank than
+	// before this change, for a reason unrelated to it, and could shift the
+	// solver's package order (TestResolutionTranscriptMatchesGolden).
+	rank := finalsInRange
+	if rank == 0 {
+		rank = prereleaseInRange
+	}
+	return pep440set.Exactly(best), true, rank, nil
 }
 
 // rankedVersions returns pkg's full published version list in Policy order,
@@ -415,10 +429,32 @@ func (p *Provider) rankedVersions(pkg Package) ([]version.Version, error) {
 	// is unchanged. provider/differential_test.go checks that against the exact
 	// reference rather than leaving it as an argument.
 	r := candidate.Rank(pkg.Name, all, p.opts.Policy)
+	r = finalsFirst(pkg.Name, r, p.opts.Prereleases)
 	p.ranked[pkg.Name] = r
 	return r, nil
 }
 
+// finalsFirst stably moves the pre-releases of a package that is not enabled
+// to the end of an already-ranked list, leaving everything else untouched.
+//
+// This is the pip/uv in-range fallback (candidate.PrereleaseSet's doc
+// comment), implemented as ranking rather than admission so the enabled set
+// can stay fixed for the whole resolution: candidate.Rank is a stable sort,
+// so this partition preserves Policy order within each half, and for an
+// enabled package (Admits true for every version) it is a no-op.
+func finalsFirst(pkg index.PackageName, ranked []version.Version, prereleases candidate.PrereleaseSet) []version.Version {
+	out := make([]version.Version, 0, len(ranked))
+	var after []version.Version
+	for _, v := range ranked {
+		if prereleases.Admits(pkg, v) {
+			out = append(out, v)
+		} else {
+			after = append(after, v)
+		}
+	}
+	return append(out, after...)
+}
+
 // singleVersion answers for a package with exactly one version and no index
 // behind it: the root and the interpreter.
 func singleVersion(v version.Version, allowed pep440set.Set) (pep440set.Set, bool, int, error) {

provider/provider_test.go

@@ -161,8 +161,8 @@ func TestCandidatesPrereleaseAdmission(t *testing.T) {
 			t.Fatal("found = false, want true")
 		}
 		if rank != 1 {
-			t.Errorf("rank = %d, want 1 (the release candidate is not admissible, and "+
-				"pre-release admission is part of the in-range filter rank counts)", rank)
+			t.Errorf("rank = %d, want 1 (rank counts finals only when any are in range, "+
+				"even though the release candidate is still in range as a fallback)", rank)
 		}
 		if got := bestVersion(t, best); got.String() != "1.0" {
 			t.Errorf("best = %s, want 1.0", got)
@@ -181,8 +181,9 @@ func TestCandidatesPrereleaseAdmission(t *testing.T) {
 		if !found {
 			t.Fatal("found = false, want true")
 		}
-		if rank != 2 {
-			t.Errorf("rank = %d, want 2", rank)
+		if rank != 1 {
+			t.Errorf("rank = %d, want 1 (finals-in-range count, per the rank rule -- "+
+				"the pre-release still wins the ranking, via Policy, not via rank)", rank)
 		}
 		if got := bestVersion(t, best); got.String() != "2.0rc1" {
 			t.Errorf("best = %s, want 2.0rc1", got)

resolver/packse_scenario_test.go

@@ -89,8 +89,8 @@ type tomlResolverOptions struct {
 
 // packseScenario is one loaded scenario, named by its path relative to
 // testdata/packse (e.g. "requires_python/python-less-than-current"), which is
-// what the classification lists (outOfScope, knownFail, unsupportedOption) key
-// on.
+// what the classification lists (outOfScope, knownFail, intentionalDivergence,
+// unsupportedOption) key on.
 type packseScenario struct {
 	relName  string // category/scenario-name, no extension
 	path     string