From 7156e77ee0b7eec64f7476faa7b692ab262b014e Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Wed, 24 Jun 2026 15:16:29 -0400 Subject: [PATCH 1/6] wip: tooling: update fg promo tooling for install features Signed-off-by: Bryce Palmer --- .../codegen/cmd/featuregate-test-analyzer.go | 158 +++++++++++++++++- 1 file changed, 154 insertions(+), 4 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 92e07bcece7..396d40a3ff5 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -247,7 +247,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { summaryMarkdown := md.ExactBytes() if len(o.OutputDir) > 0 { filename := filepath.Join(o.OutputDir, "feature-promotion-summary.md") - if err := os.WriteFile(filename, summaryMarkdown, 0644); err != nil { + if err := os.WriteFile(filename, summaryMarkdown, 0o644); err != nil { errs = append(errs, err) } @@ -343,7 +343,6 @@ func buildHTMLFeatureGateData(name string, testingResults map[JobVariant]*Testin } func writeHTMLFromTemplate(filename string, featureGateHTMLData []utils.HTMLFeatureGate) error { - data := utils.HTMLTemplateData{ FeatureGates: featureGateHTMLData, } @@ -486,7 +485,6 @@ func writeTestingMarkDown(testingResults map[JobVariant]*TestingResults, md *uti } md.Text("") md.Text("") - } var ( @@ -649,7 +647,7 @@ func (a OrderedJobVariants) Less(i, j int) bool { // Map these to an ordered list of strings so that we can define the order // rather than them being alphabetical. - var networkStackOrder = map[string]string{ + networkStackOrder := map[string]string{ "": "0", "ipv4": "1", "ipv6": "2", @@ -991,3 +989,155 @@ func matchTwoNodeFeatureGates(featureGate string, topology string) bool { } return false } + +func verifyInstallFeatureGatePromotion(featureGate string, variant JobVariant) (*TestingResults, error) { + // TEMPORARY: construct periodic job name based on pattern: + // periodic-ci-openshift-installer-release-${release-version}-periodics-e2e-${platform}${-network-topology}${-variant}-${lower(feature-gate)} + ocpRelease, err := getRelease() + if err != nil { + return nil, fmt.Errorf("getting release version: %w", err) + } + + jobName := fmt.Sprintf( + "periodic-ci-openshift-installer-release-%s-periodics-e2e-%s%s%s-%s", + ocpRelease, + variant.Cloud, + variant.NetworkStack, + topologyDisplayName(variant.Topology), + strings.ToLower(featureGate), + ) + + defaultTransport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + sippyClient := &http.Client{ + Timeout: 2 * time.Minute, + Transport: defaultTransport, + } + + results, err := verifyJobPassRate(sippyClient, ocpRelease, jobName) + if err != nil { + return nil, fmt.Errorf("verifying job pass rate: %w", err) + } + + return &TestingResults{ + JobVariant: variant, + TestResults: []TestResults{*results}, + }, nil +} + +func verifyJobPassRate(client *http.Client, release, jobName string) (*TestResults, error) { + jobRuns, err := getJobRunsFromSippy(client, release, jobName) + if err != nil { + return nil, fmt.Errorf("getting job %q results from sippy: %w", jobName, err) + } + + testResults := &TestResults{ + TestName: fmt.Sprintf("job:%s", jobName), // TODO: should this be a common name to prevent different test names per platform? + TotalRuns: len(jobRuns), + } + + for _, jobRun := range jobRuns { + if (jobRun.Failed || jobRun.OverallResult == "F") && !jobRun.KnownFailure { + // TODO: check failed test names for known regressed test names + testResults.FailedRuns++ + continue + } + + testResults.SuccessfulRuns++ + } + + return testResults, nil +} + +type sippyJobRun struct { + Variants []string `json:"variants,omitempty"` + Failed bool `json:"failed"` + FailedTestNames []string `json:"failed_test_names,omitempty"` + KnownFailure bool `json:"known_failure"` + OverallResult string `json:"overall_result,omitempty"` +} + +type sippyFilter struct { + Items []sippyFilterQuery `json:"items,omitempty"` +} + +type sippyFilterQuery struct { + ColumnField string `json:"columnField,omitempty"` + OperatorValue string `json:"operatorValue,omitempty"` + Value string `json:"value,omitempty"` +} + +func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJobRun, error) { + reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs/runs") + if err != nil { + panic(fmt.Sprintf("couldn't parse sippy jobs url: %v", err)) + } + + filter := &sippyFilter{ + Items: []sippyFilterQuery{ + { + ColumnField: "name", + OperatorValue: "equals", + Value: jobName, + }, + { + ColumnField: "timestamp", + OperatorValue: ">=", + Value: fmt.Sprintf("%d", time.Now().Add(-1*(14*(24*time.Hour))).Unix()), + }, + }, + } + + filterBytes, err := json.Marshal(filter) + if err != nil { + panic(fmt.Sprintf("couldn't marshal jobs filter: %v", err)) + } + + queryValues := url.Values{ + "release": []string{release}, + "filter": []string{string(filterBytes)}, + "sortField": []string{"timestamp"}, + "sort": []string{"desc"}, + "perPage": []string{"14"}, // Limit to the most recent 14 runs within the last 2 weeks. + "page": []string{"0"}, + } + + reqURL.RawQuery = queryValues.Encode() + + resp, err := client.Get(reqURL.String()) + if err != nil { + return nil, fmt.Errorf("getting job info: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + + defer resp.Body.Close() + + jobs := struct { + Rows []sippyJobRun `json:"rows,omitempty"` + }{} + err = json.Unmarshal(body, jobs) + if err != nil { + return nil, fmt.Errorf("unmarshalling response body: %w", err) + } + + // should always have exactly one job + return jobs.Rows, nil +} From 4682907b06229efb02cbb7a7bd758fe097b885ff Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Wed, 24 Jun 2026 16:03:44 -0400 Subject: [PATCH 2/6] wip: more changes Signed-off-by: Bryce Palmer --- tools/codegen/cmd/featuregate-test-analyzer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 396d40a3ff5..4648322aa10 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -868,7 +868,7 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi // Feature gates used by the installer don't need separate tests, use the overall install tests if strings.Contains(featureGate, "Install") { - testPattern = fmt.Sprintf("install should succeed") + return verifyInstallFeatureGatePromotion(featureGate, jobVariant) } fmt.Printf("Query sippy for all test run results for pattern %q on variant %#v\n", testPattern, jobVariant) @@ -998,6 +998,7 @@ func verifyInstallFeatureGatePromotion(featureGate string, variant JobVariant) ( return nil, fmt.Errorf("getting release version: %w", err) } + // TODO: get list of job names to evaluate for the feature gate jobName := fmt.Sprintf( "periodic-ci-openshift-installer-release-%s-periodics-e2e-%s%s%s-%s", ocpRelease, @@ -1047,7 +1048,7 @@ func verifyJobPassRate(client *http.Client, release, jobName string) (*TestResul } for _, jobRun := range jobRuns { - if (jobRun.Failed || jobRun.OverallResult == "F") && !jobRun.KnownFailure { + if jobRun.OverallResult == "F" && !jobRun.KnownFailure { // TODO: check failed test names for known regressed test names testResults.FailedRuns++ continue @@ -1130,7 +1131,7 @@ func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJ defer resp.Body.Close() - jobs := struct { + jobs := &struct { Rows []sippyJobRun `json:"rows,omitempty"` }{} err = json.Unmarshal(body, jobs) From c9bd9c7ec588a71f39ad5db4b2c48464369e7792 Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Thu, 25 Jun 2026 11:17:04 -0400 Subject: [PATCH 3/6] wip: more changes Signed-off-by: Bryce Palmer --- .../codegen/cmd/featuregate-test-analyzer.go | 224 +++++++++++++++--- 1 file changed, 187 insertions(+), 37 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 4648322aa10..9136fe65e51 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -990,24 +990,12 @@ func matchTwoNodeFeatureGates(featureGate string, topology string) bool { return false } -func verifyInstallFeatureGatePromotion(featureGate string, variant JobVariant) (*TestingResults, error) { - // TEMPORARY: construct periodic job name based on pattern: - // periodic-ci-openshift-installer-release-${release-version}-periodics-e2e-${platform}${-network-topology}${-variant}-${lower(feature-gate)} +func verifyInstallFeatureGatePromotion(featureGate string, jobVariant JobVariant) (*TestingResults, error) { ocpRelease, err := getRelease() if err != nil { return nil, fmt.Errorf("getting release version: %w", err) } - // TODO: get list of job names to evaluate for the feature gate - jobName := fmt.Sprintf( - "periodic-ci-openshift-installer-release-%s-periodics-e2e-%s%s%s-%s", - ocpRelease, - variant.Cloud, - variant.NetworkStack, - topologyDisplayName(variant.Topology), - strings.ToLower(featureGate), - ) - defaultTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, ForceAttemptHTTP2: true, @@ -1025,32 +1013,57 @@ func verifyInstallFeatureGatePromotion(featureGate string, variant JobVariant) ( Transport: defaultTransport, } - results, err := verifyJobPassRate(sippyClient, ocpRelease, jobName) + jobs, err := getJobsForFeatureGateFromSippy(sippyClient, ocpRelease, featureGate, jobVariant) if err != nil { - return nil, fmt.Errorf("verifying job pass rate: %w", err) + return nil, fmt.Errorf("getting jobs for feature-gate %q for variant %v : %w", featureGate, jobVariant, err) + } + + testResults := []TestResults{} + + for _, job := range jobs { + results, err := verifyJobPassRate(sippyClient, ocpRelease, job) + if err != nil { + return nil, fmt.Errorf("verifying job pass rate for job %q: %w", job.Name, err) + } + + testResults = append(testResults, *results) } return &TestingResults{ - JobVariant: variant, - TestResults: []TestResults{*results}, + JobVariant: jobVariant, + TestResults: testResults, }, nil } -func verifyJobPassRate(client *http.Client, release, jobName string) (*TestResults, error) { - jobRuns, err := getJobRunsFromSippy(client, release, jobName) +func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*TestResults, error) { + // TODO: use other job fields for early-return calculations + + jobRuns, err := getJobRunsFromSippy(client, release, job.Name) if err != nil { - return nil, fmt.Errorf("getting job %q results from sippy: %w", jobName, err) + return nil, fmt.Errorf("getting job %q results from sippy: %w", job.Name, err) } testResults := &TestResults{ - TestName: fmt.Sprintf("job:%s", jobName), // TODO: should this be a common name to prevent different test names per platform? + TestName: job.Name, TotalRuns: len(jobRuns), } + triagedTestFailures, err := getTriagedTestFailuresFromSippy(client, release) + for _, jobRun := range jobRuns { if jobRun.OverallResult == "F" && !jobRun.KnownFailure { - // TODO: check failed test names for known regressed test names - testResults.FailedRuns++ + + untriagedTestFailures := []string{} + for _, failure := range jobRun.FailedTestNames { + if !triagedTestFailures.Has(failure) { + untriagedTestFailures = append(untriagedTestFailures, failure) + } + } + + if len(untriagedTestFailures) > 0 { + testResults.FailedRuns++ + } + continue } @@ -1060,6 +1073,99 @@ func verifyJobPassRate(client *http.Client, release, jobName string) (*TestResul return testResults, nil } +type sippyJob struct { + Name string `json:"name,omitempty"` + Variants []string `json:"variants,omitempty"` + CurrentRuns int `json:"current_runs,omitempty"` + CurrentPasses int `json:"current_passes,omitempty"` + PreviousRuns int `json:"previous_runs,omitempty"` + PreviousPasses int `json:"previous_passes,omitempty"` +} + +func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate string, variant JobVariant) ([]sippyJob, error) { + reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs") + if err != nil { + panic(fmt.Sprintf("couldn't parse sippy jobs url: %v", err)) + } + + filter := &sippy.SippyQueryStruct{ + Items: []sippy.SippyQueryItem{ + { + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("Capability:%s", featureGate), + }, + { + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("Platform:%s", variant.Cloud), + }, + { + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("Architecture:%s", variant.Architecture), + }, + { + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("Topology:%s", variant.Topology), + }, + }, + } + + if variant.NetworkStack != "" { + filter.Items = append(filter.Items, sippy.SippyQueryItem{ + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("NetworkStack:%s", variant.NetworkStack), + }) + } + + if variant.OS != "" { + filter.Items = append(filter.Items, sippy.SippyQueryItem{ + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("OS:%s", variant.OS), + }) + } + + filterBytes, err := json.Marshal(filter) + if err != nil { + panic(fmt.Sprintf("couldn't marshal jobs filter: %v", err)) + } + + queryValues := url.Values{ + "release": []string{release}, + "filter": []string{string(filterBytes)}, + } + + reqURL.RawQuery = queryValues.Encode() + + resp, err := client.Get(reqURL.String()) + if err != nil { + return nil, fmt.Errorf("getting job info: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + + defer resp.Body.Close() + + jobs := []sippyJob{} + err = json.Unmarshal(body, jobs) + if err != nil { + return nil, fmt.Errorf("unmarshalling response body: %w", err) + } + + return jobs, nil +} + type sippyJobRun struct { Variants []string `json:"variants,omitempty"` Failed bool `json:"failed"` @@ -1068,24 +1174,14 @@ type sippyJobRun struct { OverallResult string `json:"overall_result,omitempty"` } -type sippyFilter struct { - Items []sippyFilterQuery `json:"items,omitempty"` -} - -type sippyFilterQuery struct { - ColumnField string `json:"columnField,omitempty"` - OperatorValue string `json:"operatorValue,omitempty"` - Value string `json:"value,omitempty"` -} - func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJobRun, error) { reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs/runs") if err != nil { - panic(fmt.Sprintf("couldn't parse sippy jobs url: %v", err)) + panic(fmt.Sprintf("couldn't parse sippy jobs runs url: %v", err)) } - filter := &sippyFilter{ - Items: []sippyFilterQuery{ + filter := &sippy.SippyQueryStruct{ + Items: []sippy.SippyQueryItem{ { ColumnField: "name", OperatorValue: "equals", @@ -1139,6 +1235,60 @@ func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJ return nil, fmt.Errorf("unmarshalling response body: %w", err) } - // should always have exactly one job return jobs.Rows, nil } + +type sippyTriageItem struct { + Regressions []sippyRegression `json:"regressions,omitempty"` +} + +type sippyRegression struct { + Release string `json:"release,omitempty"` + TestName string `json:"test_name,omitempty"` + Variants []string `json:"variants,omitempty"` +} + +func getTriagedTestFailuresFromSippy(client *http.Client, release string) (sets.Set[string], error) { + reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/component_readiness/triages") + if err != nil { + panic(fmt.Sprintf("couldn't parse sippy triages url: %v", err)) + } + + resp, err := client.Get(reqURL.String()) + if err != nil { + return nil, fmt.Errorf("getting sippy triages: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + + defer resp.Body.Close() + + triageItems := []sippyTriageItem{} + err = json.Unmarshal(body, triageItems) + if err != nil { + return nil, fmt.Errorf("unmarshalling response body: %w", err) + } + + regressedTests := sets.New[string]() + + for _, triageItem := range triageItems { + for _, regression := range triageItem.Regressions { + if regression.Release != release { + continue + } + + // TODO: handle variant matching here? + + regressedTests.Insert(regression.TestName) + } + } + + return regressedTests, nil +} From 034a89fb583c53315c161c2fbc9707ba34c8c5f3 Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Thu, 25 Jun 2026 11:22:46 -0400 Subject: [PATCH 4/6] drop: promote AWSDualStackInstall Signed-off-by: Bryce Palmer --- features.md | 2 +- features/features.go | 2 +- .../featuregates/featureGate-4-10-Hypershift-Default.yaml | 6 +++--- .../featuregates/featureGate-4-10-Hypershift-OKD.yaml | 6 +++--- .../featureGate-4-10-SelfManagedHA-Default.yaml | 6 +++--- .../featuregates/featureGate-4-10-SelfManagedHA-OKD.yaml | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/features.md b/features.md index 0b16169cfe9..c0f7c8eff47 100644 --- a/features.md +++ b/features.md @@ -30,7 +30,6 @@ | ProvisioningRequestAvailable| | | Enabled | Enabled | | | | | | AWSClusterHostedDNS| | | Enabled | Enabled | | | Enabled | Enabled | | AWSDedicatedHosts| | | Enabled | Enabled | | | Enabled | Enabled | -| AWSDualStackInstall| | | Enabled | Enabled | | | Enabled | Enabled | | AWSEuropeanSovereignCloudInstall| | | Enabled | Enabled | | | Enabled | Enabled | | AdditionalStorageConfig| | | Enabled | Enabled | | | Enabled | Enabled | | AutomatedEtcdBackup| | | Enabled | Enabled | | | Enabled | Enabled | @@ -93,6 +92,7 @@ | AWSServiceLBNetworkSecurityGroup| | Enabled | Enabled | Enabled | | Enabled | Enabled | Enabled | | OSStreams| | Enabled | Enabled | Enabled | | Enabled | Enabled | Enabled | | AWSClusterHostedDNSInstall| Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | +| AWSDualStackInstall| Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | | AzureClusterHostedDNSInstall| Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | | AzureWorkloadIdentity| Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | | BootImageSkewEnforcement| Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | Enabled | diff --git a/features/features.go b/features/features.go index 1d0f9bcce43..d359f947ab4 100644 --- a/features/features.go +++ b/features/features.go @@ -854,7 +854,7 @@ var ( contactPerson("sadasu"). productScope(ocpSpecific). enhancementPR("https://github.com/openshift/enhancements/pull/1806"). - enable(inTechPreviewNoUpgrade(), inDevPreviewNoUpgrade()). + enable(inDefault(), inOKD(), inTechPreviewNoUpgrade(), inDevPreviewNoUpgrade()). mustRegister() FeatureGateAzureDualStackInstall = newFeatureGate("AzureDualStackInstall"). diff --git a/payload-manifests/featuregates/featureGate-4-10-Hypershift-Default.yaml b/payload-manifests/featuregates/featureGate-4-10-Hypershift-Default.yaml index 389432b924e..b4cf444145f 100644 --- a/payload-manifests/featuregates/featureGate-4-10-Hypershift-Default.yaml +++ b/payload-manifests/featuregates/featureGate-4-10-Hypershift-Default.yaml @@ -20,9 +20,6 @@ { "name": "AWSDedicatedHosts" }, - { - "name": "AWSDualStackInstall" - }, { "name": "AWSEuropeanSovereignCloudInstall" }, @@ -292,6 +289,9 @@ { "name": "AWSClusterHostedDNSInstall" }, + { + "name": "AWSDualStackInstall" + }, { "name": "AzureClusterHostedDNSInstall" }, diff --git a/payload-manifests/featuregates/featureGate-4-10-Hypershift-OKD.yaml b/payload-manifests/featuregates/featureGate-4-10-Hypershift-OKD.yaml index 52a878dbfdd..94e4d0e6fb4 100644 --- a/payload-manifests/featuregates/featureGate-4-10-Hypershift-OKD.yaml +++ b/payload-manifests/featuregates/featureGate-4-10-Hypershift-OKD.yaml @@ -22,9 +22,6 @@ { "name": "AWSDedicatedHosts" }, - { - "name": "AWSDualStackInstall" - }, { "name": "AWSEuropeanSovereignCloudInstall" }, @@ -294,6 +291,9 @@ { "name": "AWSClusterHostedDNSInstall" }, + { + "name": "AWSDualStackInstall" + }, { "name": "AzureClusterHostedDNSInstall" }, diff --git a/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-Default.yaml b/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-Default.yaml index f0f1078fd29..e2f05430d1e 100644 --- a/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-Default.yaml +++ b/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-Default.yaml @@ -20,9 +20,6 @@ { "name": "AWSDedicatedHosts" }, - { - "name": "AWSDualStackInstall" - }, { "name": "AWSEuropeanSovereignCloudInstall" }, @@ -283,6 +280,9 @@ { "name": "AWSClusterHostedDNSInstall" }, + { + "name": "AWSDualStackInstall" + }, { "name": "AWSServiceLBNetworkSecurityGroup" }, diff --git a/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-OKD.yaml b/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-OKD.yaml index faa2bc84479..fc899ff19e5 100644 --- a/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-OKD.yaml +++ b/payload-manifests/featuregates/featureGate-4-10-SelfManagedHA-OKD.yaml @@ -22,9 +22,6 @@ { "name": "AWSDedicatedHosts" }, - { - "name": "AWSDualStackInstall" - }, { "name": "AWSEuropeanSovereignCloudInstall" }, @@ -285,6 +282,9 @@ { "name": "AWSClusterHostedDNSInstall" }, + { + "name": "AWSDualStackInstall" + }, { "name": "AWSServiceLBNetworkSecurityGroup" }, From c07aa2fbcd002f7548b338b117bf36f56bb11d91 Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Thu, 25 Jun 2026 11:39:29 -0400 Subject: [PATCH 5/6] wip: example Signed-off-by: Bryce Palmer --- .../codegen/cmd/featuregate-test-analyzer.go | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 9136fe65e51..80cb0215e6e 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -1061,6 +1061,13 @@ func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*Test } if len(untriagedTestFailures) > 0 { + var writer strings.Builder + writer.WriteString(fmt.Sprintf("job run %s has untriaged test failures:\n", jobRun.TestGridURL)) + for _, testFailure := range untriagedTestFailures { + writer.WriteString(fmt.Sprintf("\t- %s\n", testFailure)) + } + + fmt.Println(writer.String()) testResults.FailedRuns++ } @@ -1083,6 +1090,19 @@ type sippyJob struct { } func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate string, variant JobVariant) ([]sippyJob, error) { + // TODO: Remove this once done testing the other logic. + + out := []sippyJob{ + { + Name: "periodic-ci-openshift-release-main-nightly-5.0-e2e-aws-ovn-installer-dualstack-ipv4-primary-techpreview", + }, + { + Name: "periodic-ci-openshift-release-main-nightly-5.0-e2e-aws-ovn-installer-dualstack-ipv6-primary-techpreview", + }, + } + + return out, nil + reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs") if err != nil { panic(fmt.Sprintf("couldn't parse sippy jobs url: %v", err)) @@ -1158,7 +1178,7 @@ func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate st defer resp.Body.Close() jobs := []sippyJob{} - err = json.Unmarshal(body, jobs) + err = json.Unmarshal(body, &jobs) if err != nil { return nil, fmt.Errorf("unmarshalling response body: %w", err) } @@ -1172,6 +1192,7 @@ type sippyJobRun struct { FailedTestNames []string `json:"failed_test_names,omitempty"` KnownFailure bool `json:"known_failure"` OverallResult string `json:"overall_result,omitempty"` + TestGridURL string `json:"test_grid_url,omitempty"` } func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJobRun, error) { @@ -1271,7 +1292,7 @@ func getTriagedTestFailuresFromSippy(client *http.Client, release string) (sets. defer resp.Body.Close() triageItems := []sippyTriageItem{} - err = json.Unmarshal(body, triageItems) + err = json.Unmarshal(body, &triageItems) if err != nil { return nil, fmt.Errorf("unmarshalling response body: %w", err) } From dbd3ddd5195e60ab2c46b018f79dfb0e3350f990 Mon Sep 17 00:00:00 2001 From: Bryce Palmer Date: Thu, 25 Jun 2026 14:57:02 -0400 Subject: [PATCH 6/6] fixup! some adjustments Signed-off-by: Bryce Palmer --- .../codegen/cmd/featuregate-test-analyzer.go | 240 +++++++----------- tools/codegen/pkg/sippy/json_types.go | 148 +++++++++++ 2 files changed, 244 insertions(+), 144 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 80cb0215e6e..d05b2e1eb60 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -868,7 +868,7 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi // Feature gates used by the installer don't need separate tests, use the overall install tests if strings.Contains(featureGate, "Install") { - return verifyInstallFeatureGatePromotion(featureGate, jobVariant) + return verifyJobBasedFeatureGatePromotion(featureGate, jobVariant) } fmt.Printf("Query sippy for all test run results for pattern %q on variant %#v\n", testPattern, jobVariant) @@ -990,7 +990,7 @@ func matchTwoNodeFeatureGates(featureGate string, topology string) bool { return false } -func verifyInstallFeatureGatePromotion(featureGate string, jobVariant JobVariant) (*TestingResults, error) { +func verifyJobBasedFeatureGatePromotion(featureGate string, jobVariant JobVariant) (*TestingResults, error) { ocpRelease, err := getRelease() if err != nil { return nil, fmt.Errorf("getting release version: %w", err) @@ -1021,7 +1021,7 @@ func verifyInstallFeatureGatePromotion(featureGate string, jobVariant JobVariant testResults := []TestResults{} for _, job := range jobs { - results, err := verifyJobPassRate(sippyClient, ocpRelease, job) + results, err := verifyJobPassRate(sippyClient, ocpRelease, job, jobVariant) if err != nil { return nil, fmt.Errorf("verifying job pass rate for job %q: %w", job.Name, err) } @@ -1035,8 +1035,57 @@ func verifyInstallFeatureGatePromotion(featureGate string, jobVariant JobVariant }, nil } -func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*TestResults, error) { - // TODO: use other job fields for early-return calculations +func verifyJobPassRate(client *http.Client, release string, job sippy.SippyJob, variant JobVariant) (*TestResults, error) { + // Do an early check for 95% pass rate with at least 14 runs + runs := job.CurrentRuns + passes := job.CurrentPasses + + if runs < 14 { + runs += job.PreviousRuns + passes += job.PreviousPasses + + fmt.Println("current + previous runs", runs) + } + + // If we have less than 14 runs, return the current set of results as-is + // because it doesn't meet promotion criteria. + // + // This saves us from unnecessarily making calls out to Sippy to perform a more nuanced + // failures analysis of the job runs to see if failed runs are true failures or known regressions. + if runs < 14 { + return &TestResults{ + TestName: job.Name, + TotalRuns: runs, + SuccessfulRuns: passes, + FailedRuns: runs - passes, + }, nil + } + + // If we have greater than or equal to 14 runs AND they are passing at a rate of at least 95%, + // we can return early because this job has passed the promotion requirements. + // + // This saves us from unnecessarily making calls out to Sippy to perform a more nuanced + // failures analysis of the job runs to see if failed runs are true failures or known regressions. + if (passes * 100) / runs >= 95 { + return &TestResults{ + TestName: job.Name, + TotalRuns: runs, + SuccessfulRuns: passes, + FailedRuns: runs - passes, + }, nil + } + + // We haven't passed promotion requirements with this job, but jobs might be impacted + // by known regressed tests. While important to get fixed, many regressions are either + // release blockers or require an exception to not be a release blocker. + // + // We can be reasonably confident in promoting a feature if the tests that are failing + // on failed runs are only ones with known regressions for the platform being tested. + // + // From here on, we fetch the 14 most recent job runs for the job in question from Sippy, + // fetch the known regressions for the release + platform variant, and compare failing + // job runs failed tests with the known regressions - only counting failures that have + // unknown test failures as a true failure. jobRuns, err := getJobRunsFromSippy(client, release, job.Name) if err != nil { @@ -1048,7 +1097,10 @@ func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*Test TotalRuns: len(jobRuns), } - triagedTestFailures, err := getTriagedTestFailuresFromSippy(client, release) + triagedTestFailures, err := getTriagedTestFailuresFromSippy(client, release, variant) + if err != nil { + return nil, fmt.Errorf("getting triaged test failures from sippy: %q", err) + } for _, jobRun := range jobRuns { if jobRun.OverallResult == "F" && !jobRun.KnownFailure { @@ -1069,9 +1121,9 @@ func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*Test fmt.Println(writer.String()) testResults.FailedRuns++ - } - continue + continue + } } testResults.SuccessfulRuns++ @@ -1080,88 +1132,25 @@ func verifyJobPassRate(client *http.Client, release string, job sippyJob) (*Test return testResults, nil } -type sippyJob struct { - Name string `json:"name,omitempty"` - Variants []string `json:"variants,omitempty"` - CurrentRuns int `json:"current_runs,omitempty"` - CurrentPasses int `json:"current_passes,omitempty"` - PreviousRuns int `json:"previous_runs,omitempty"` - PreviousPasses int `json:"previous_passes,omitempty"` -} - -func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate string, variant JobVariant) ([]sippyJob, error) { +func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate string, variant JobVariant) ([]sippy.SippyJob, error) { // TODO: Remove this once done testing the other logic. - out := []sippyJob{ + out := []sippy.SippyJob{ { Name: "periodic-ci-openshift-release-main-nightly-5.0-e2e-aws-ovn-installer-dualstack-ipv4-primary-techpreview", + CurrentRuns: 25, + CurrentPasses: 3, }, { Name: "periodic-ci-openshift-release-main-nightly-5.0-e2e-aws-ovn-installer-dualstack-ipv6-primary-techpreview", + CurrentRuns: 25, + CurrentPasses: 3, }, } return out, nil - reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs") - if err != nil { - panic(fmt.Sprintf("couldn't parse sippy jobs url: %v", err)) - } - - filter := &sippy.SippyQueryStruct{ - Items: []sippy.SippyQueryItem{ - { - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("Capability:%s", featureGate), - }, - { - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("Platform:%s", variant.Cloud), - }, - { - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("Architecture:%s", variant.Architecture), - }, - { - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("Topology:%s", variant.Topology), - }, - }, - } - - if variant.NetworkStack != "" { - filter.Items = append(filter.Items, sippy.SippyQueryItem{ - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("NetworkStack:%s", variant.NetworkStack), - }) - } - - if variant.OS != "" { - filter.Items = append(filter.Items, sippy.SippyQueryItem{ - ColumnField: "variants", - OperatorValue: "contains", - Value: fmt.Sprintf("OS:%s", variant.OS), - }) - } - - filterBytes, err := json.Marshal(filter) - if err != nil { - panic(fmt.Sprintf("couldn't marshal jobs filter: %v", err)) - } - - queryValues := url.Values{ - "release": []string{release}, - "filter": []string{string(filterBytes)}, - } - - reqURL.RawQuery = queryValues.Encode() - - resp, err := client.Get(reqURL.String()) + resp, err := client.Get(sippy.BuildSippyJobsForFeatureGateURL(featureGate, release, variant.Topology, variant.Cloud, variant.Architecture, variant.NetworkStack, variant.OS)) if err != nil { return nil, fmt.Errorf("getting job info: %w", err) } @@ -1177,7 +1166,7 @@ func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate st defer resp.Body.Close() - jobs := []sippyJob{} + jobs := []sippy.SippyJob{} err = json.Unmarshal(body, &jobs) if err != nil { return nil, fmt.Errorf("unmarshalling response body: %w", err) @@ -1186,53 +1175,8 @@ func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate st return jobs, nil } -type sippyJobRun struct { - Variants []string `json:"variants,omitempty"` - Failed bool `json:"failed"` - FailedTestNames []string `json:"failed_test_names,omitempty"` - KnownFailure bool `json:"known_failure"` - OverallResult string `json:"overall_result,omitempty"` - TestGridURL string `json:"test_grid_url,omitempty"` -} - -func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJobRun, error) { - reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/jobs/runs") - if err != nil { - panic(fmt.Sprintf("couldn't parse sippy jobs runs url: %v", err)) - } - - filter := &sippy.SippyQueryStruct{ - Items: []sippy.SippyQueryItem{ - { - ColumnField: "name", - OperatorValue: "equals", - Value: jobName, - }, - { - ColumnField: "timestamp", - OperatorValue: ">=", - Value: fmt.Sprintf("%d", time.Now().Add(-1*(14*(24*time.Hour))).Unix()), - }, - }, - } - - filterBytes, err := json.Marshal(filter) - if err != nil { - panic(fmt.Sprintf("couldn't marshal jobs filter: %v", err)) - } - - queryValues := url.Values{ - "release": []string{release}, - "filter": []string{string(filterBytes)}, - "sortField": []string{"timestamp"}, - "sort": []string{"desc"}, - "perPage": []string{"14"}, // Limit to the most recent 14 runs within the last 2 weeks. - "page": []string{"0"}, - } - - reqURL.RawQuery = queryValues.Encode() - - resp, err := client.Get(reqURL.String()) +func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippy.SippyJobRun, error) { + resp, err := client.Get(sippy.BuildSippyJobRunsForJobURL(release, jobName, time.Now().Add(-1 * 14 * 24 * time.Hour))) if err != nil { return nil, fmt.Errorf("getting job info: %w", err) } @@ -1248,28 +1192,16 @@ func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippyJ defer resp.Body.Close() - jobs := &struct { - Rows []sippyJobRun `json:"rows,omitempty"` - }{} - err = json.Unmarshal(body, jobs) + runResults := &sippy.SippyJobRunsResult{} + err = json.Unmarshal(body, runResults) if err != nil { return nil, fmt.Errorf("unmarshalling response body: %w", err) } - return jobs.Rows, nil -} - -type sippyTriageItem struct { - Regressions []sippyRegression `json:"regressions,omitempty"` + return runResults.Rows, nil } -type sippyRegression struct { - Release string `json:"release,omitempty"` - TestName string `json:"test_name,omitempty"` - Variants []string `json:"variants,omitempty"` -} - -func getTriagedTestFailuresFromSippy(client *http.Client, release string) (sets.Set[string], error) { +func getTriagedTestFailuresFromSippy(client *http.Client, release string, variant JobVariant) (sets.Set[string], error) { reqURL, err := url.Parse("https://sippy.dptools.openshift.org/api/component_readiness/triages") if err != nil { panic(fmt.Sprintf("couldn't parse sippy triages url: %v", err)) @@ -1291,7 +1223,7 @@ func getTriagedTestFailuresFromSippy(client *http.Client, release string) (sets. defer resp.Body.Close() - triageItems := []sippyTriageItem{} + triageItems := []sippy.SippyTriageItem{} err = json.Unmarshal(body, &triageItems) if err != nil { return nil, fmt.Errorf("unmarshalling response body: %w", err) @@ -1305,7 +1237,27 @@ func getTriagedTestFailuresFromSippy(client *http.Client, release string) (sets. continue } - // TODO: handle variant matching here? + regressionVariants := sets.New(regression.Variants...) + + if !regressionVariants.Has(fmt.Sprintf("Platform:%s", variant.Cloud)) { + continue + } + + if !regressionVariants.Has(fmt.Sprintf("Topology:%s", variant.Topology)) { + continue + } + + if !regressionVariants.Has(fmt.Sprintf("Architecture:%s", variant.Architecture)) { + continue + } + + if variant.NetworkStack != "" && !regressionVariants.Has(fmt.Sprintf("NetworkStack:%s", variant.NetworkStack)) { + continue + } + + if variant.OS != "" && !regressionVariants.Has(fmt.Sprintf("OS:%s", variant.OS)) { + continue + } regressedTests.Insert(regression.TestName) } diff --git a/tools/codegen/pkg/sippy/json_types.go b/tools/codegen/pkg/sippy/json_types.go index 0795c2519a4..992cd141a30 100644 --- a/tools/codegen/pkg/sippy/json_types.go +++ b/tools/codegen/pkg/sippy/json_types.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "strings" + "time" "k8s.io/apimachinery/pkg/util/sets" ) @@ -54,6 +55,38 @@ type SippyTestInfo struct { OpenBugs int `json:"open_bugs"` } +type SippyJob struct { + Name string `json:"name"` + Variants []string `json:"variants"` + CurrentRuns int `json:"current_runs"` + CurrentPasses int `json:"current_passes"` + PreviousRuns int `json:"previous_runs"` + PreviousPasses int `json:"previous_passes"` +} + +type SippyJobRun struct { + Variants []string `json:"variants"` + Failed bool `json:"failed"` + FailedTestNames []string `json:"failed_test_names"` + KnownFailure bool `json:"known_failure"` + OverallResult string `json:"overall_result"` + TestGridURL string `json:"test_grid_url"` +} + +type SippyJobRunsResult struct { + Rows []SippyJobRun `json:"rows"` +} + +type SippyTriageItem struct { + Regressions []SippyRegression `json:"regressions,omitempty"` +} + +type SippyRegression struct { + Release string `json:"release,omitempty"` + TestName string `json:"test_name,omitempty"` + Variants []string `json:"variants,omitempty"` +} + func QueriesFor(cloud, architecture, topology, networkStack, os, jobTiers, testPattern string) []*SippyQueryStruct { // Build base query items that are common to all JobTier queries baseItems := []SippyQueryItem{ @@ -219,3 +252,118 @@ func BuildSippyTestAnalysisURL(release, testName, topology, cloud, architecture, return u.String() } + +func BuildSippyJobsForFeatureGateURL(featureGate, release, topology, cloud, architecture, networkStack, os string) string { + filterItems := []SippyQueryItem{ + { + ColumnField: "variants", + Not: true, + OperatorValue: "has entry", + Value: "never-stable", + }, + { + ColumnField: "variants", + Not: true, + OperatorValue: "has entry", + Value: "aggregated", + }, + { + ColumnField: "variants", + OperatorValue: "contains", + Value: fmt.Sprintf("Capability:%s", featureGate), + }, + { + ColumnField: "variants", + OperatorValue: "has entry", + Value: fmt.Sprintf("Topology:%s", topology), + }, + { + ColumnField: "variants", + OperatorValue: "has entry", + Value: fmt.Sprintf("Platform:%s", cloud), + }, + { + ColumnField: "variants", + OperatorValue: "has entry", + Value: fmt.Sprintf("Architecture:%s", architecture), + }, + } + if networkStack != "" { + filterItems = append(filterItems, SippyQueryItem{ + ColumnField: "variants", + OperatorValue: "has entry", + Value: fmt.Sprintf("NetworkStack:%s", networkStack), + }) + } + if os != "" { + filterItems = append(filterItems, SippyQueryItem{ + ColumnField: "variants", + OperatorValue: "has entry", + Value: fmt.Sprintf("OS:%s", os), + }) + } + // Note: We don't filter by JobTier in the URL so the link shows all tiers + // The actual queries filter by each tier separately and merge results + + filterObj := SippyQueryStruct{ + Items: filterItems, + LinkOperator: "and", + } + filterJSON, err := json.Marshal(filterObj) + if err != nil { + return "" + } + + u := &url.URL{ + Scheme: "https", + Host: "sippy.dptools.openshift.org", + Path: "/api/jobs", + } + q := u.Query() + q.Set("filters", string(filterJSON)) + q.Set("release", release) + u.RawQuery = q.Encode() + + return u.String() +} + +func BuildSippyJobRunsForJobURL(release, jobName string, timestamp time.Time) string { + filterItems := []SippyQueryItem{ + { + ColumnField: "name", + OperatorValue: "equals", + Value: jobName, + }, + { + ColumnField: "timestamp", + OperatorValue: ">=", + Value: fmt.Sprintf("%d", timestamp.Unix()), + }, + } + + filterObj := SippyQueryStruct{ + Items: filterItems, + LinkOperator: "and", + } + filterJSON, err := json.Marshal(filterObj) + if err != nil { + return "" + } + + u := &url.URL{ + Scheme: "https", + Host: "sippy.dptools.openshift.org", + Path: "/api/jobs/runs", + } + q := u.Query() + q.Set("filters", string(filterJSON)) + q.Set("release", release) + q.Set("sortField", "timestamp") + q.Set("sort", "desc") + q.Set("perPage", "14") // hardcoded to 14 most recent job runs to prevent excessive "expensive" calculations of job results. + q.Set("page", "0") + u.RawQuery = q.Encode() + + return u.String() +} +