diff --git a/README.md b/README.md index 11e65ae..b1fd767 100644 --- a/README.md +++ b/README.md @@ -32,23 +32,23 @@ The python scrip `vcast_exec.py` is the main driver for build/execute VectorCAST The api for vcast_exec.py follows: ``` - usage: vcast_exec.py [VectorCAST Project] - [-h/--help] - [--build-execute] [--setup SETUP] - [--use_imported_result IMPORTEDRESULTS] - [--build | --incremental] - [--output_dir OUTPUT_DIR] [--source_root SOURCE_ROOT] - [--html_base_dir HTML_BASE_DIR] - [--cobertura] [--cobertura_extended] [--lcov] [--junit] - [--export_rgw] [--sonarqube] [--pclp_input PCLP_INPUT] - [--pclp_output_html PCLP_OUTPUT_HTML] - [--exit_with_failed_count] [EXIT_WITH_FAILED_COUNT]] - [--exit_with_failed_comp [EXIT_WITH_FAILED_COMP] - [--check_build_log] [--aggregate] [--metrics] - [--fullstatus] [--utfull] [--tcmr] [--noindex] - [--jobs JOBS] [--ci] [-l LEVEL] [-e ENVIRONMENT] - [--gitlab | --azure] [--print_exc] [--timing] - [-v/--verbose] [--version] + usage: vcast_exec.py [-h] [--build-execute] [--setup SETUP] + [--use_imported_result IMPORTEDRESULTS] + [--build | --incremental] [--output_dir OUTPUT_DIR] + [--source_root SOURCE_ROOT] + [--html_base_dir HTML_BASE_DIR] [--cobertura] + [--cobertura_extended] [--lcov] [--junit] [--export_rgw] + [--sonarqube] [--pclp_input PCLP_INPUT] + [--pclp_output_html PCLP_OUTPUT_HTML] + [--exit_with_failed_count [EXIT_WITH_FAILED_COUNT]] + [--exit_with_failed_comp [EXIT_WITH_FAILED_COMP]] + [--check_build_log] + [--covToDisplay {statement,branch,mcdc,function,functioncall}] + [--aggregate] [--metrics] [--fullstatus] [--utfull] + [--tcmr] [--noindex] [--jobs JOBS] [--ci] [-l LEVEL] + [-e ENVIRONMENT] [--gitlab | --azure] [--print_exc] + [--timing] [-v] [--version] + [ManageProject] positional arguments: ManageProject VectorCAST Project Name @@ -62,62 +62,62 @@ The api for vcast_exec.py follows: --build-execute Builds and exeuctes the VectorCAST Project --setup SETUP Path to setup_env.bat/.sh (optional) --use_imported_result IMPORTEDRESULTS - Use existing VCR file from repository for CBT via - Imported Results + Use existing VCR file from repository for CBT via + Imported Results --build Only builds the VectorCAST Project - --incremental Use Change Based Testing - (Cannot be used with --build) + --incremental Use Change Based Testing (Cannot be used with --build) Metrics Options: Options generating metrics --output_dir OUTPUT_DIR - Set the base directory of the xml_data directory. - Default is the workspace directory + Set the base directory of the xml_data directory. + Default is the workspace directory --source_root SOURCE_ROOT - Set the absolute path for the source file - in coverage reporting + Set the absolute path for the source file in coverage + reporting --html_base_dir HTML_BASE_DIR - Set the base directory of the html_reports directory. - The default is the workspace directory + Set the base directory of the html_reports directory. + The default is the workspace directory --cobertura Generate coverage results in Cobertura xml format - --cobertura_extended Generate coverage results in extended - Cobertura xml format + --cobertura_extended Generate coverage results in extended Cobertura xml + format --lcov Generate coverage results in an LCOV format --junit Generate test results in Junit xml format --export_rgw Export RGW data - --sonarqube Generate test results in SonarQube Generic - test execution report format (CppUnit) + --sonarqube Generate test results in SonarQube Generic test + execution report format (CppUnit) --pclp_input PCLP_INPUT - Generate static analysis results from - PC-lint Plus XML file to generic - static analysis format (codequality) + Generate static analysis results from PC-lint Plus XML + file to generic static analysis format (codequality) --pclp_output_html PCLP_OUTPUT_HTML - Generate static analysis results from - PC-lint Plus XML file to an HTML output + Generate static analysis results from PC-lint Plus XML + file to an HTML output --exit_with_failed_count [EXIT_WITH_FAILED_COUNT] - Returns failed test case count as script exit. - Set a value to indicate a percentage above which - the job will be marked as failed + Returns failed test case count as script exit. Set a + value to indicate a percentage above which the job + will be marked as failed --exit_with_failed_comp [EXIT_WITH_FAILED_COMP] - Returns failed if any of the functions have a - Complexity (Vg) > value. - --check_build_log Checks build log for a list of error phrases. - Returns failure if any are found. + Returns failed if any of the functions have a + Complexity (Vg) > value. + --check_build_log Checks build log for a list of error phrases. Returns + failure if any are found. + --covToDisplay {statement,branch,mcdc,function,functioncall} + Selects which coverage to display for coverage print. + Default is "statement". Report Selection: VectorCAST Manage reports that can be generated - --aggregate Generate aggregate coverage report for VectorCAST Project + --aggregate Generate aggregate coverage report VectorCAST Project --metrics Generate metrics reports for VectorCAST Project --fullstatus Generate full status reports for VectorCAST Project - --utfull Generate Full Reports for each VectorCAST - environment in the VectorCAST Project - --tcmr Generate Test Cases Management Reports for each - VectorCAST environment in the VectorCAST Project - --noindex Stops index.html report that ties all the other - HTML reports together from being created - for VectorCAST Project + --utfull Generate Full Reports for each VectorCAST environment + in project + --tcmr Generate Test Cases Management Reports for each + VectorCAST environment in project + --noindex Stops index.html report that ties all the other HTML + reports together from being created Build/Execution Options: Options that effect build/execute operation @@ -125,7 +125,7 @@ The api for vcast_exec.py follows: --jobs JOBS Number of concurrent jobs (default = 1) --ci Use Continuous Integration Licenses -l LEVEL, --level LEVEL - Environment Name if only doing single environment. + Environment Name if only doing single environment. Should be in the form of compiler/testsuite -e ENVIRONMENT, --environment ENVIRONMENT Environment Name if only doing single environment. @@ -139,10 +139,18 @@ The api for vcast_exec.py follows: --timing Prints timing information for metrics generation -v, --verbose Enable verbose output --version Displays the version information + ``` # Change log +07/2026 + 06/2026 +* Added --covToDisplay +* Added additional failure phrases + * Harness Link Failure + * Environment built but not Linked +* Modified failure return codes * Fixed Full Reports always getting generated * Fixed --output_dir and --html_base_dir so that * xml_data gets copied to --output_dir diff --git a/VERSION.txt b/VERSION.txt index 5cc3896..ea77f4a 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.06.16-1329-0400-gdb82f6b +v2026.07.09-1544-0400-gb6e4569 diff --git a/check_build_log.py b/check_build_log.py index cebf63b..394a6f3 100644 --- a/check_build_log.py +++ b/check_build_log.py @@ -28,7 +28,9 @@ "not permitted in continuous integration mode", "has been opened by a newer version of VectorCAST", "Environment built but not Compiled", - "ERROR: Compile failed for unit" + "Harness Link Failure", + "ERROR: Compile failed for unit", + "Environment built but not Linked" ] VC_unstablePhrases = [ diff --git a/cobertura.py b/cobertura.py index c0ab9bc..4117863 100644 --- a/cobertura.py +++ b/cobertura.py @@ -757,7 +757,8 @@ def runCoberturaResults(packages, api, verbose = False, extended = False, source return total_st, cov_st, total_lines, cov_lines, total_br, cov_br, total_func, cov_func, total_fc, cov_fc, total_mcdc, cov_mcdc, branch_rate, statement_rate, line_rate, func_rate, FC_rate, MCDC_rate, vg -def generateCoverageResults(inFile, azure = False, xml_data_dir = "xml_data", verbose = False, extended = False, source_root = "" ): +def generateCoverageResults(inFile, azure = False, xml_data_dir = "xml_data", + verbose = False, extended = False, source_root = "", covToDisplay="statement"): # cwd = os.getcwd() # xml_data_dir = os.path.join(cwd,xml_data_dir) @@ -816,7 +817,38 @@ def generateCoverageResults(inFile, azure = False, xml_data_dir = "xml_data", ve if FC_rate != -1.0: print ("function calls: {:.2f}% ({:d} out of {:d})".format(FC_rate*100.0, cov_fc, total_fc)) if MCDC_rate != -1.0: print ("mcdc pairs: {:.2f}% ({:d} out of {:d})".format(MCDC_rate*100.0, cov_mcdc, total_mcdc)) - if statement_rate != -1.0: print ("coverage: {:.2f}% of statements".format(statement_rate*100.0)) + # use selected coverage from --covToDisplay option + match covToDisplay: + case "statement": + if statement_rate != -1.0: + print ("coverage: {:.2f}% of statements".format(statement_rate*100.0)) + else: + print (f"[ERROR] selected coverage {covToDisplay} has no coverage metrics") + + case "branch": + if branch_rate != -1.0: + print ("coverage: {:.2f}% of branch".format(branch_rate*100.0)) + else: + print (f"[ERROR] selected coverage {covToDisplay} has no coverage metrics") + + case "mcdc": + if MCDC_rate != -1.0: + print ("coverage: {:.2f}% of mcdc pairs".format(MCDC_rate*100.0)) + else: + print (f"[ERROR] selected coverage {covToDisplay} has no coverage metrics") + + case "function": + if func_rate != -1.0: + print ("coverage: {:.2f}% of functions".format(func_rate*100.0)) + else: + print (f"[ERROR] selected coverage {covToDisplay} has no coverage metrics") + + case "functioncall": + if FC_rate != -1.0: + print ("coverage: {:.2f}% of function calls".format(FC_rate*100.0)) + else: + print (f"[ERROR] selected coverage {covToDisplay} has no coverage metrics") + if complexity != -1.0: print ("complexity: {:d}".format(complexity)) source = etree.SubElement(sources, "source") source.text = "./" diff --git a/generate_xml.py b/generate_xml.py index a2e3996..51271ac 100644 --- a/generate_xml.py +++ b/generate_xml.py @@ -929,6 +929,8 @@ def generate_local_results(self, results, key): localXML.topLevelAPI = self.api localXML.noResults = self.noResults localXML.generate_unit() + self.failed_count += localXML.failed_count + self.passed_count += localXML.passed_count ##need_fixup if not self.no_full_reports: @@ -1062,6 +1064,13 @@ def generate_testresults(self): self.fh_data = "" self.localDataOnly = True self.noResults = False + # CI exit code uses self.failed_count / self.passed_count (see vcast_exec --exit_with_failed_count). + # get_full_status ALL aggregate can disagree with per-environment JUnit (e.g. EXEC_NO_EXPECTED_VALUES + # / "No expected values" in testcase XML). Merge counts from GenerateXml.generate_unit() per env. + self.failed_count = 0 + self.passed_count = 0 + all_errors = 0 + all_success = 0 if results['ALL']['testcase_results'] == {}: print("** No results in project") self.noResults = True @@ -1070,14 +1079,13 @@ def generate_testresults(self): success = results['ALL']['testcase_results']['success_count'] errors = total - success failed = errors + all_errors = errors + all_success = success self.fh_data += ("\n") self.fh_data += ("\n") self.fh_data += (" \n" % (errors,total,failed,escape(self.manageProjectName, quote=False))) - self.failed_count = errors - self.passed_count = success - for result in results: if result in all_envs: if len(result.split("/")) != 3: @@ -1110,6 +1118,27 @@ def generate_testresults(self): self.fh_data += (testcaseString % (tc_name_full, classname, extraStatus)) self.failed_count += 1 + if self.failed_count == 0 and self.passed_count == 0 and not self.noResults: + self.failed_count = all_errors + self.passed_count = all_success + elif self.noResults and self.failed_count == 0 and self.passed_count == 0: + # The environment(s) were processed during build-execute but produced + # ZERO testcase results (results['ALL']['testcase_results'] == {} and no + # imported results contributed counts above). + # This is what happens when the VectorCAST report engine drops results + # - e.g. a non-UTF-8 byte in the execution data ("Replacing unknown + # characters in TC-000023.XML" / "Error generating report for + # VCAST_AUTOMATIC_COMPOUND"), or a report-generation error. + # Without this branch failed_count stays 0, so vcast_exec + # --exit_with_failed_count exits 0 and the CI job passes *vacuously* + # while reporting no tests at all - masking the real state (and any + # genuine regression). Record a synthetic failure so the job fails. + print("** ERROR: No testcase results reported for %s - report " + "generation produced no results; counting as a failure so the " + "CI job does not pass vacuously." % self.manageProjectName) + self.failed_count = 1 + self.passed_count = 0 + self.fh_data += (" \n") self.fh_data += ("\n") if not self.localDataOnly: diff --git a/merge_vcr.py b/merge_vcr.py index a7ad202..08321c7 100644 --- a/merge_vcr.py +++ b/merge_vcr.py @@ -65,7 +65,7 @@ def run(origVcrFile, newVcrFile, verbose): except: pass - tempNewVcrFile = os.path.join("newVcr",newVcrFile) + tempNewVcrFile = os.path.join("newVcr",os.path.basename(newVcrFile)) tempOrigVcrFile = os.path.join("origVcr",os.path.basename(origVcrFile)) shutil.copyfile(newVcrFile, tempNewVcrFile) @@ -76,12 +76,16 @@ def run(origVcrFile, newVcrFile, verbose): for file in glob.glob("newVcr/*.*"): if file.endswith(".db") and "_cover" not in file: newDbName = file + elif file.endswith(".db") and "_cover" in file: + newCoverDbName = file with zipfile.ZipFile(tempOrigVcrFile, 'r') as zip_ref: zip_ref.extractall("origVcr") for file in glob.glob("origVcr/*.*"): if file.endswith(".db") and "_cover" not in file: origDbName = file + elif file.endswith(".db") and "_cover" in file: + origCoverDbName = file os.remove(tempNewVcrFile) os.remove(tempOrigVcrFile) @@ -102,6 +106,8 @@ def run(origVcrFile, newVcrFile, verbose): new_db.close() orig_db.close() + + shutil.copyfile(newCoverDbName, origCoverDbName) os.remove(newVcrFile) os.remove(origVcrFile) diff --git a/vcast_exec.py b/vcast_exec.py index 58f81b6..d449ec2 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -155,6 +155,7 @@ def __init__(self, args): self.useJunitFailCountPct = True self.junit_percent_to_fail = int(args.exit_with_failed_count) self.failed_count = 0 + self.passed_count = 0 if args.output_dir: self.output_dir = args.output_dir @@ -166,6 +167,8 @@ def __init__(self, args): if not os.path.exists(self.xml_data_dir): os.makedirs(self.xml_data_dir) + self.covToDisplay = args.covToDisplay + if args.build and not args.build_execute: self.build_execute = "--build" self.vcast_action = "--vcast_action build" @@ -370,8 +373,11 @@ def generateIndexHtml(self): prj_dir = os.getcwd().replace("\\","/") + "/" tempHtmlReportList = glob.glob("*.html") - tempHtmlReportList += glob.glob(os.path.join('xml_data', "*.html")) - tempHtmlReportList += glob.glob(os.path.join('html_reports', "*.html")) + #tempHtmlReportList += glob.glob(os.path.join('xml_data', "*.html")) + #tempHtmlReportList += glob.glob(os.path.join('html_reports', "*.html")) + tempHtmlReportList += glob.glob(os.path.join(self.xml_data_dir, "*.html")) + tempHtmlReportList += glob.glob(os.path.join(self.html_base_dir, "*.html")) + htmlReportList = [] for report in tempHtmlReportList: @@ -442,7 +448,8 @@ def runCoberturaMetrics(self): print("Creating Cobertura Metrics") cobertura.generateCoverageResults(self.FullMP, self.azure, self.xml_data_dir, verbose = self.verbose, - extended=self.cobertura_extended, source_root = self.source_root) + extended=self.cobertura_extended, source_root = self.source_root, + covToDisplay = self.covToDisplay) def runSonarQubeMetrics(self): if not checkVectorCASTVersion(21): @@ -625,8 +632,35 @@ def runExec(self): with open(self.build_log_name,"wb") as fd: fd.write(build_log.encode(self.encFmt, "replace")) + + def getReturnCode(self): + + msgs = [] + + complexityFailureCount = 0 + if self.complexityCheck: + for key in cobertura.vgByFunction: + if cobertura.vgByFunction[key] > self.complexityThreshold: + file, func = key.split("::") + print (f"[ERROR] \n File : {file}\n Function: {func}\n Message : COMPLEXITY is greater than {self.complexityThreshold}") + complexityFailureCount += 1 + + if complexityFailureCount > 0: + msgs.append(f"{complexityFailureCount} complexity failures") + if args.check_build_log: + if check_build_log(self.build_log_name) == 2: + msgs.append(f"Build log error. See information above...") + if self.useJunitFailCountPct: + print(f"[ERROR] exit_with_failed_count={args.exit_with_failed_count} specified. Fail Percent = {round(self.failed_pct,0)}% Return code: {self.failed_count}") + msgs.append(f"Tests case failues greater than {args.exit_with_failed_count} specified") + + if msgs: + return " ; ".join(msgs) + else: + return 0 + if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -659,6 +693,10 @@ def runExec(self): nargs='?', default=None) metricsGroup.add_argument('--check_build_log', help='Checks build log for a list of error phrases. Returns failure if any are found.', action="store_true", default = False) + + metricsGroup.add_argument("--covToDisplay", type=str.lower, choices=["statement", "branch", "mcdc", "function", "functioncall"], + default="statement",help='Selects which coverage to display for coverage print. Default is "statement".',) + reportGroup = parser.add_argument_group('Report Selection', 'VectorCAST Manage reports that can be generated') reportGroup.add_argument('--aggregate', help='Generate aggregate coverage report VectorCAST Project', action="store_true", default = False) reportGroup.add_argument('--metrics', help='Generate metrics reports for VectorCAST Project', action="store_true", default = False) @@ -768,22 +806,7 @@ def runExec(self): if args.output_dir: vcExec.copyXmlData() - complexityFailureCount = 0 - if vcExec.complexityCheck: - for key in cobertura.vgByFunction: - if cobertura.vgByFunction[key] > vcExec.complexityThreshold: - file, func = key.split("::") - print (f"[ERROR] \n File : {file}\n Function: {func}\n Message : COMPLEXITY is greater than {vcExec.complexityThreshold}") - complexityFailureCount += 1 - - if complexityFailureCount > 0: - sys.exit(complexityFailureCount) - - if vcExec.useJunitFailCountPct: - print(f"[ERROR] exit_with_failed_count={args.exit_with_failed_count} specified. Fail Percent = {round(vcExec.failed_pct,0)}% Return code: {vcExec.failed_count}") - sys.exit(vcExec.failed_count) - - if args.check_build_log: - if check_build_log(vcExec.build_log_name) == 2: - sys.exit(2) - + returnCode = vcExec.getReturnCode() + + sys.exit(returnCode) +