From ee2b372066c4b161a0b8eacdba3421633d4c08e8 Mon Sep 17 00:00:00 2001 From: TimSVector Date: Thu, 23 Apr 2026 16:04:32 -0400 Subject: [PATCH 01/12] Should work with imported results. Added gate for function complexity greater that X --- cobertura.py | 8 + generate_results.py | 569 ++++++++++++------- generate_xml.py | 1312 +++++++++++++++++++++++++++++-------------- vcast_exec.py | 30 +- 4 files changed, 1272 insertions(+), 647 deletions(-) diff --git a/cobertura.py b/cobertura.py index b64c9de..a456431 100644 --- a/cobertura.py +++ b/cobertura.py @@ -56,6 +56,8 @@ encFmt = getVectorCASTEncoding() +vgByFunction = {} + def write_xml(x, name, verbose = False): if verbose: @@ -370,6 +372,10 @@ def processStatementBranchMCDC(fileApi, lines, extended = False): return linesCovered, linesTotal +def updateVgByFunction(coverXML, coverApi): + for func in coverApi.functions: + fullName = func.source_file.path + "::" + func.parameterized_name + vgByFunction[fullName] = func.metrics.complexity def procesCoverage(coverXML, coverApi, extended = False, source_root = ""): @@ -634,6 +640,8 @@ def runCoberturaResults(packages, api, verbose = False, extended = False, source vg += file.metrics.complexity pkg_vg += file.metrics.complexity + + updateVgByFunction(classes, file) if extended: total_fc += file.metrics.function_calls diff --git a/generate_results.py b/generate_results.py index 7763040..0172c63 100644 --- a/generate_results.py +++ b/generate_results.py @@ -34,21 +34,25 @@ import subprocess import time import traceback - -import getjobs - from managewait import ManageWait +import generate_qa_results_xml +using_new_reports = False +legacy = False try: from vector.apps.ReportBuilder.custom_report import CustomReport try: from vector.apps.DataAPI.unit_test_api import UnitTestApi except: from vector.apps.DataAPI.api import Api as UnitTestApi + using_new_reports = True except: pass -from vector.enums import ENVIRONMENT_STATUS_TYPE_T + -from vcast_utils import dump, getVectorCASTEncoding +try: + from vector.apps.DataAPI.vcproject_api import VCProjectApi +except: + pass encFmt = getVectorCASTEncoding() @@ -57,11 +61,44 @@ global print_exc global wait_time global wait_loops +verbose = False +print_exc = False +need_fixup = False wait_time = 30 wait_loops = 1 -verbose = False -print_exc = True +import getjobs + +def skipReporting(build_dir, skipReportsForSkippedEnvs, cbtDict): + + import hashlib + + ## use hash code instead of final directory name as regression scripts can have overlapping final directory names + + build_dir_4hash = build_dir.upper() + build_dir_4hash = "/".join(build_dir_4hash.split("/")[-2:]) + + # Unicode-objects must be encoded before hashing in Python 3 + if sys.version_info[0] >= 3: + build_dir_4hash = build_dir_4hash.encode('utf-8') + + hashCode = hashlib.md5(build_dir_4hash).hexdigest() + + # skip report gen for skipped environments + if skipReportsForSkippedEnvs and cbtDict: + if hashCode not in cbtDict.keys(): + if verbose: + print("skipping report because hash not round in cbtdict") + + return True + else: + c,i,s = cbtDict[hashCode] + if len(c)==0 and len(i)==0 and len(s)==0: + if verbose: + print("skipping report because c,i,s are all 0 size") + return True + return False + enabledEnvironmentArray = [] def getEnabledEnvironments(MPname): @@ -70,7 +107,11 @@ def getEnabledEnvironments(MPname): for line in output.split("\n"): if line.strip(): # type being system or unit test - compiler, testsuite, environment = line.split() + try: + compiler, testsuite, environment = line.split() + except: + compiler, testsuite, environment, source, machine = line.split() + enabledEnvironmentArray.append([compiler, testsuite, environment]) def environmentEnabled(comp,ts,env): @@ -91,26 +132,17 @@ def runManageWithWait(command_line, silent=False): # Determine if this version of VectorCAST supports new-style reporting/Data API def checkUseNewReportsAndAPI(): if os.environ.get("VCAST_REPORT_ENGINE", "") == "LEGACY": - # Using legacy reporting with new reports - fall back to parsing html report + # The execution plugin will ignore this value, but warn user. if verbose: print("VectorCAST/Execution ignoring LEGACY VCAST_REPORT_ENGINE.") - # Look for existence of file that only exists in distribution with the new reports - check_file = os.path.join(os.environ.get('VECTORCAST_DIR'), - "python", - "vector", - "apps", - "ReportBuilder", - "reports", - "full_report.pyc") - if os.path.isfile(check_file): - if verbose: - print("Using VectorCAST with new style reporting. Use Data API for Jenkins reports.") - return True - else: - if verbose: - print("Using VectorCAST without new style reporting. Use VectorCAST reports for Jenkins reports.") - return False + if verbose: + if using_new_reports: + print("Using VectorCAST with new style reporting. Use Data API for CI reports.") + else: + print("Using VectorCAST without new style reporting. Use VectorCAST reports for CI reports.") + + return using_new_reports # Read the Manage project file to determine its version # File has already been checked for existence @@ -119,46 +151,24 @@ def readManageVersion(ManageFile): if os.path.isfile(ManageFile + ".vcm"): ManageFile = ManageFile + '.vcm' - py2 = sys.version_info[0] < 3 - - with open(ManageFile, "rb") as projFile: - for raw_line in projFile: - # --- Normalize line to text (unicode in Py3, unicode or str in Py2) --- - if py2: - # In Py2, raw_line is a str (byte string) - try: - line = raw_line.decode(encFmt, "replace") - except Exception: - line = raw_line.decode("utf-8", "replace") - else: - # In Py3, raw_line is bytes - if isinstance(raw_line, bytes): - try: - line = raw_line.decode(encFmt, "replace") - except Exception: - line = raw_line.decode("utf-8", "replace") - else: - line = raw_line - - # --- Look for version/project keywords --- - if "version" in line and "project" in line: - match = re.search(r"\d+", line) - if match: - version = int(match.group()) - break - + with open(ManageFile, 'rb') as projFile: + for raw_line in projFile: # iterates lazily, line by line + line = raw_line.decode(encFmt, "replace") # decode each line + if 'version' in line and 'project' in line: + version = int(re.findall(r'\d+', line)[0]) + break + if verbose: - print("Version of Manage project file = %d" % version) + print("Version of VectorCAST project file = %d" % version) print("(Levels change in version 17 (*maybe) and above)") - return version # Call manage to get the mapping of Envs to Directory etc. -def getManageEnvs(FullManageProjectName, use_ci = ""): +def getManageEnvs(FullManageProjectName): manageEnvs = {} cmd_prefix = os.environ.get('VECTORCAST_DIR') + os.sep - callStr = cmd_prefix + "manage --project " + FullManageProjectName + " " + use_ci + " --build-directory-name" + callStr = cmd_prefix + "manage --project " + FullManageProjectName + " --build-directory-name" out_mgt = runManageWithWait(callStr, silent=True) if verbose: print(out_mgt) @@ -203,11 +213,8 @@ def delete_file(filename): if os.path.exists(filename): os.remove(filename) -def genDataApiReports(FullManageProjectName, entry, use_ci, xml_data_dir): - - global print_exc - - xml_file = None +def genDataApiReports(FullManageProjectName, entry, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte): + xml_file = "" try: from generate_xml import GenerateXml @@ -230,21 +237,39 @@ def genDataApiReports(FullManageProjectName, entry, use_ci, xml_data_dir): xmlUnitReportName, jenkins_link, jobNameDotted, - verbose, - use_ci) - + verbose, + cbtDict, + generate_exec_rpt_each_testcase, + use_archive_extract, + report_only_failures, + print_exc, + useStartLine, + teePrint, + use_cte) + if xml_file.api != None: if verbose: - print(" Generate Jenkins testcase report: {}".format(xmlUnitReportName)) + print(" Generate CI testcase report: {}".format(xmlUnitReportName)) xml_file.generate_unit() + if verbose: + print(" Generate Jenkins coverage report: {}".format(xmlCoverReportName)) + xml_file.generate_cover() + else: + print(" Skipping environment: " + jobNameDotted) + print("\n\n") + print ("******************************************************") + print ("** Environment's that only use imported results **") + print ("** will not properly generate metrics with this **") + print ("** version of VectorCAST. **") + print ("******************************************************") + print("\n\n") + except Exception as e: print("ERROR: failed to generate XML reports using vpython and the Data API for ", entry["compiler"] + "_" + entry["testsuite"] + "_" + entry["env"], "in directory", entry["build_dir"]) - if True: - traceback.print_exc() - try: + try: failed_count = xml_file.failed_count passed_count = xml_file.passed_count del xml_file @@ -252,64 +277,87 @@ def genDataApiReports(FullManageProjectName, entry, use_ci, xml_data_dir): except: traceback.print_exc() return 0, 0 + + +def fixup_css(report_name): + global need_fixup + # Needed for VC19 and VC19 SP1. + # From VC19 SP2 onwards a new option VCAST_RPTS_SELF_CONTAINED is used instead + + if not need_fixup: + return + + with open(report_name,"rb") as fd: + data = fd.read().decode('utf-8','replace') + + #fix up inline CSS because of Content Security Policy violation + newData = data[: data.index("")+8:] + + #fix up style directive because of Content Security Policy violation + newData = newData.replace("
","
") + + #fixup the inline VectorCAST image because of Content Security Policy violation + regex_str = r"\"Vector\".*"",newData) + + with open(report_name, "wb") as fd: + fd.write(newData.encode('utf-8','replace')) + + workspace = os.getenv("WORKSPACE") + if workspace is None: + workspace = os.getcwd() + + vc_scripts = os.path.join(workspace,"vc_scripts") + + shutil.copy(os.path.join(vc_scripts,"vector_style.css"), "management/vector_style.css") + shutil.copy(os.path.join(vc_scripts,"vectorcast.png"), "management/vectorcast.png") def generateCoverReport(path, env, level ): - from vector.apps.ReportBuilder.custom_report import CustomReport + def _dummy(*args, **kwargs): + return True + from vector.apps.DataAPI.cover_api import CoverApi - try: - api=CoverApi(path) - except: - print("CR: Skipping environment: "+ env) - print("CR: *" + env + " DataAPI is invalid") - return - - try: - if api.environment.status != ENVIRONMENT_STATUS_TYPE_T.NORMAL: - print("CR: Skipping environment: "+ env) - print("CR: *" + env + " status is not NORMAL") - return - except: - pass - - report_name = "html_reports/" + level + "_" + env + ".html" + api=CoverApi(path) + report_name = "management/" + level + "_" + env + ".html" + try: - CustomReport.report_from_api(api, report_type="Demo", formats=["HTML"], output_file=report_name, sections=["CUSTOM_HEADER", "REPORT_TITLE", "TABLE_OF_CONTENTS", "CONFIG_DATA", "METRICS", "MCDC_TABLES", "AGGREGATE_COVERAGE", "CUSTOM_FOOTER"]) + try: + api.commit = _dummy + api.report(report_type="AGGREGATE_REPORT", formats=["HTML"], output_file=report_name) + except: + CustomReport.report_from_api(api, report_type="Demo", formats=["HTML"], output_file=report_name, sections=["CUSTOM_HEADER", "REPORT_TITLE", "TABLE_OF_CONTENTS", "CONFIG_DATA", "METRICS", "MCDC_TABLES", "AGGREGATE_COVERAGE", "CUSTOM_FOOTER"]) + fixup_css(report_name) + except Exception as e: - print("CR: *Problem generating custom report for " + env + ": ") - if print_exc: - traceback.print_exc() + build_dir = path.replace("\\","/") + build_dir = build_dir.rsplit("/",1)[0] + + print(traceback.format_exc(), print_exc, level.split("_")[0] , level.split("_")[2], env, build_dir) def generateUTReport(path, env, level): global verbose def _dummy(*args, **kwargs): return True + report_name = "management/" + level + "_" + env + ".html" - try: - api=UnitTestApi(path) - except: - print("UTR: Skipping environment: "+ env) - print("UTR: *" + env + "'s DataAPI is invalid") - return - - if api.environment.status != ENVIRONMENT_STATUS_TYPE_T.NORMAL: - print("UTR: Skipping environment: "+ env) - print("UTR: *" + env + " status is not NORMAL") - return - - report_name = "html_reports/" + level + "_" + env + ".html" + api=UnitTestApi(path) try: api.commit = _dummy api.report(report_type="FULL_REPORT", formats=["HTML"], output_file=report_name) + fixup_css(report_name) except Exception as e: - print("UTR: *Problem generating custom report for " + env + ".") - if print_exc: - traceback.print_exc() + build_dir = path.replace("\\","/") + build_dir = build_dir.rsplit("/",1)[0] + print(traceback.format_exc(), print_exc, level.split("_")[0] , level.split("_")[2], env, build_dir) + def generateIndividualReports(entry, envName): global verbose @@ -327,63 +375,126 @@ def generateIndividualReports(entry, envName): elif os.path.exists(unit_path): generateUTReport(unit_path , env, level) +def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, teePrint, use_cte): + global verbose -def useNewAPI(FullManageProjectName, manageEnvs, level, envName, use_ci, xml_data_dir = "xml_data"): - failed_count = 0 - passed_count = 0 + print("Using VCProjectApi") + + xml_file = "" + + try: + from generate_xml import GenerateManageXml + + xml_file = GenerateManageXml(FullManageProjectName, + verbose, + cbtDict, + generate_exec_rpt_each_testcase, + use_archive_extract, + report_only_failures, + no_full_report, + print_exc, + useStartLine, teePrint, use_cte) + + if xml_file.api != None: + xml_file.generate_testresults() + xml_file.generate_cover() + else: + print(" Skipping environment: " + jobNameDotted) + print("\n\n") + print ("******************************************************") + print ("** Environment's that only use imported results **") + print ("** will not properly generate metrics with this **") + print ("** version of VectorCAST. **") + print ("******************************************************") + print("\n\n") + + except Exception as e: + parse_traceback.parse(traceback.format_exc(), print_exc) + #traceback.print_exc() + + + try: + return xml_file.passed_count, xml_file.failed_count + except: + return 0, 0 + + +def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, teePrint, use_cte): + failed_count = 0 + passed_count = 0 + + print("Using DataAPI per environment") + for currentEnv in manageEnvs: + if skipReporting(manageEnvs[currentEnv]["build_dir"], use_archive_extract, cbtDict): + print(" No Change for " + currentEnv + ". Skipping reporting.") + continue if envName == None: - fc, pc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], use_ci, xml_data_dir) - failed_count += fc + pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte) passed_count += pc + failed_count += fc + if no_full_report: + continue + generateIndividualReports(manageEnvs[currentEnv], envName) elif manageEnvs[currentEnv]["env"].upper() == envName.upper(): env_level = manageEnvs[currentEnv]["compiler"] + "/" + manageEnvs[currentEnv]["testsuite"] - if level is None: - level = env_level - - if env_level.upper() == level.upper(): - fc, pc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], use_ci, xml_data_dir) - failed_count += fc + if level == None or env_level.upper() == level.upper(): + pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte) passed_count += pc + failed_count += fc + + if no_full_report: + continue + generateIndividualReports(manageEnvs[currentEnv], envName) - - with open("unit_test_fail_count.txt","wb") as fd: - fd.write(str(failed_count).encode(encFmt,'replace')) - - return failed_count, passed_count + + return passed_count, failed_count +def cleanupDirectory(path, teePrint): + + # if the path exists, try to delete all file in it + if os.path.isdir(path): + shutil.rmtree(path) + os.mkdir(path) + +def cleanupOldBuilds(teePrint): + for path in ["xml_data","management","execution"]: + cleanupDirectory(path, teePrint) # build the Test Case Management Report for Manage Project # envName and level only supplied when doing reports for a sub-project # of a multi-job -def buildReports(FullManageProjectName = None, - level = None, - envName = None, - generate_individual_reports = True, - timing = False, - cbtDict = None, - use_archive_extract = False, - report_only_failures = False, - no_full_report = False, - use_ci = "", - xml_data_dir = "xml_data", - useStartLine = False): - +def buildReports(FullManageProjectName = None, + level = None, + envName = None, + generate_individual_reports = True, + timing = False, + cbtDict = None, + use_archive_extract = False, + report_only_failures = False, + no_full_report = False, + use_ci = "", + xml_data_dir = "xml_data", + useStartLine = False, + use_cte = False): + if timing: - print("Start: " + str(time.time())) + print("Start report generation: " + str(time.time())) saved_level = level saved_envName = envName + + getEnabledEnvironments(FullManageProjectName) - # make sure the project exists + # make sure the project exists if not os.path.isfile(FullManageProjectName) and not os.path.isfile(FullManageProjectName + ".vcm"): - raise IOError(FullManageProjectName + ' does not exist') + raise FileNotFoundError(FullManageProjectName + ' does not exist') return manageProjectName = os.path.splitext(os.path.basename(FullManageProjectName))[0] @@ -392,66 +503,65 @@ def buildReports(FullManageProjectName = None, useNewReport = checkUseNewReportsAndAPI() manageEnvs = {} - getEnabledEnvironments(FullManageProjectName) - if timing: print("Version Check: " + str(time.time())) - # cleaning up old builds - for path in [os.path.join(xml_data_dir,"junit"),"html_reports"]: - # if the path exists, try to delete it - if os.path.isdir(path): - try: - shutil.rmtree(path) - except: - # if there was an error removing the directory...delete all the files - print("Error removing directory: " + path) - for file in glob.glob(path + "/*.*"): - try: - os.remove(file); - except: - print("Error removing file after failed to remove directory: " + path + "/" + file) - pass - - # we should either have an empty directory or no directory - if not os.path.isdir(path): - try: - os.makedirs(path) - except: - print("Error creating directory: " + path) + cleanupOldBuilds(teePrint) + for file in glob.glob("*.csv"): try: os.remove(file); if verbose: print("Removing file: " + file) except Exception as e: - print("Error removing " + file) - print(e) - - - ### Using new data API - 2019 and beyond - + print(" *INFO: File System Error removing " + file + ". Check console for environment build/execution errors") + if print_exc: traceback.print_exc() + failed_count = 0 passed_count = 0 + + ### Using new data API - 2019 and beyond if timing: print("Cleanup: " + str(time.time())) - if useNewReport: - + if useNewReport and not legacy: try: - shutil.rmtree("execution") + vcproj = VCProjectApi(FullManageProjectName) + tool_version = vcproj.tool_version + if tool_version.startswith("20"): + use_manage_api = False + else: + use_manage_api = True + vcproj.close() except: - pass - manageEnvs = getManageEnvs(FullManageProjectName, use_ci) - if timing: - print("Using DataAPI for reporting") - print("Get Info: " + str(time.time())) - fc, pc = useNewAPI(FullManageProjectName, manageEnvs, level, envName, use_ci = use_ci, xml_data_dir=xml_data_dir) - failed_count += fc - passed_count += pc + use_manage_api = False + + if use_manage_api: + passed_count, failed_count = useManageAPI(FullManageProjectName, cbtDict, generate_individual_reports, + use_archive_extract, report_only_failures, no_full_report, + useStartLine, teePrint, use_cte) + + else: + + manageEnvs = getManageEnvs(FullManageProjectName) + if timing: + print("Using DataAPI for reporting") + print("Get Info: " + str(time.time())) + passed_count, failed_count = useNewAPI(FullManageProjectName, + manageEnvs, level, envName, cbtDict, generate_individual_reports, + use_archive_extract, report_only_failures, no_full_report, + useStartLine, teePrint, use_cte) + if timing: print("XML and Individual reports: " + str(time.time())) + with open("unit_test_fail_count.txt", "wb") as fd: + fd.write(str(failed_count).encode(encFmt, "replace")) + + with open("unit_test_passfail_count.txt", "wb") as fd: + text = "{} {}".format(passed_count, failed_count) + fd.write(text.encode(encFmt, "replace")) + ### NOT Using new data API else: raise IOError('VectorCAST 2020 or later required') @@ -462,71 +572,94 @@ def buildReports(FullManageProjectName = None, if timing: - print("Complete: " + str(time.time())) + print("Complete report generate: " + str(time.time())) return failed_count, passed_count + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('ManageProject', help='Manager Project Name') - parser.add_argument('-v', '--verbose', help='Enable verbose output', action="store_true") - parser.add_argument('-l', '--level', help='Environment Name if only doing single environment. Should be in the form of level/env') - parser.add_argument('-e', '--environment', help='Environment Name if only doing single environment. Should be in the form of level/env') - parser.add_argument('-g', '--dont-generate-individual-reports', help='Don\'t Generated Individual Reports (below 2019 - this just controls execution report generate, 2019 and later - no individual reports will be generated', action="store_true") - parser.add_argument('--wait_time', help='Time (in seconds) to wait between execution attempts', type=int, default=30) - parser.add_argument('--wait_loops', help='Number of times to retry execution', type=int, default=1) - parser.add_argument('--timing', help='Display timing information for report generation', action="store_true") - parser.add_argument('--cobertura', help='Output coverage results in Cobertura format', action="store_true", default=False) - parser.add_argument('--api', help='Unused', type=int) - parser.add_argument('--final', help='Write Final JUnit Test Results file', action="store_true") - parser.add_argument('--gitlab', help='Generate Cobertura in a format GitLab can use', action="store_true", default=True) - parser.add_argument('--ci', help='Use continuous integration licenses', action="store_true", default=False) - parser.add_argument('--output_dir', help='Set the base directory of the xml_data directory. Default is the workspace directory', default = "xml_data") - parser.add_argument('--azure', help='Build using Azure DevOps', action="store_true", default = False) + parser.add_argument('ManageProject', help='Manager Project Name') + parser.add_argument('-v', '--verbose', help='Enable verbose output', action="store_true") + parser.add_argument('-l', '--level', help='Level for doing single environment. Should be in the form of compiler/testsuite') + parser.add_argument('-e', '--environment', help='Environment Name if only doing single environment') + parser.add_argument('-g', '--dont-generate-individual-reports', + help='Don\'t Generated Individual Reports. Below VC2019 - this just controls execution report generate. VC2019 and later - execution reports for each testcase won\'t be generated', action="store_true", default=False) + parser.add_argument('--wait_time', help='Time (in seconds) to wait between execution attempts', type=int, default=30) + parser.add_argument('--wait_loops', help='Number of times to retry execution', type=int, default=1) + parser.add_argument('--timing', help='Display timing information for report generation', action="store_true", default = False) + parser.add_argument('--buildlog', help='Build Log for CBT Statitics', default = None) + + ## Hidden because they are specific to customer need or testing + parser.add_argument('--junit', help=argparse.SUPPRESS, action="store_true") + parser.add_argument('--junit_use_cte_for_classname', help=argparse.SUPPRESS, action="store_true", dest="use_cte") + parser.add_argument('--print_exc', help=argparse.SUPPRESS, action="store_true") + parser.add_argument('--api', help=argparse.SUPPRESS, type=int) + parser.add_argument('--use_archive_extract', help=argparse.SUPPRESS, action="store_true", default = False) + parser.add_argument('--report_only_failures', help=argparse.SUPPRESS, action="store_true", default = False) + parser.add_argument('--no_full_report', help=argparse.SUPPRESS, action="store_true", default = False) + parser.add_argument('--legacy', help=argparse.SUPPRESS, action="store_true", default = False) args = parser.parse_args() - try: - if "19.sp1" in open(os.path.join(os.environ['VECTORCAST_DIR'],"DATA/tools_version.txt").read()): - # custom report patch for SP1 problem - should be fixed in future release - old_init = CustomReport._post_init - def new_init(self): - old_init(self) - self.context['report']['use_all_testcases'] = True - CustomReport._post_init = new_init - except: - pass + if args.use_archive_extract and (not args.buildlog or not os.path.exists(args.buildlog)): + print("Must have a valid --buildlog file to use --use_archive_extract") + print("The option use_archive_extract is disabled") + args.use_archive_extract = False + + legacy = args.legacy + timing = args.timing + if timing: + print("Start: " + str(time.time())) + + if legacy and sys.version_info[0] >= 3: + print ("Legacy mode testing not support with Python3 (VectorCAST 2021 and above)") + sys.exit(-1) + + + + + generate_individual_reports = not args.dont_generate_individual_reports + if args.verbose: verbose = True + + if args.print_exc or verbose: + print_exc = True + wait_time = args.wait_time wait_loops = args.wait_loops - if args.dont_generate_individual_reports: - dont_generate_individual_reports = False - else: - dont_generate_individual_reports = True - + junit = True + cbtDict = None if args.timing: timing = True else: timing = False - + # Used for pre VC19 os.environ['VCAST_RPTS_PRETTY_PRINT_HTML'] = 'FALSE' # Used for VC19 SP2 onwards os.environ['VCAST_RPTS_SELF_CONTAINED'] = 'FALSE' + os.environ['VCAST_MANAGE_PROJECT_DIRECTORY'] = os.path.abspath(args.ManageProject).rsplit(".",1)[0] - if args.ci: - os.environ['VCAST_USE_CI_LICENSES'] = '1' - use_ci = " --ci " - else: - use_ci = "" - xml_data_dir = args.output_dir - failed_count, passed_count = buildReports(args.ManageProject,args.level,args.environment,dont_generate_individual_reports, timing, use_ci, xml_data_dir) + failed_count, passed_count = buildReports(args.ManageProject, + args.level, + args.environment, + generate_individual_reports, + timing, + cbtDict, + args.use_archive_extract, + args.report_only_failures, + args.no_full_report, + use_ci = "", + xml_data_dir = xml_data_dir, + useStartLine = False, + use_cte = args.use_cte) if args.cobertura: for file in glob.glob(os.path.join(xml_data_dir,"cobertura","coverage_results_*.*")): diff --git a/generate_xml.py b/generate_xml.py index d6a1f17..e92a3ee 100644 --- a/generate_xml.py +++ b/generate_xml.py @@ -1,7 +1,7 @@ # # The MIT License # -# Copyright 2024 Vector Informatik, GmbH. +# Copyright 2026 Vector Informatik, GmbH. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -59,9 +59,13 @@ def fmt_percent(x,y): from operator import attrgetter from vector.enums import COVERAGE_TYPE_TYPE_T -from vector.enums import ENVIRONMENT_STATUS_TYPE_T from vcast_utils import dump, getVectorCASTEncoding import hashlib +import traceback + +from pprint import pprint + +import re def dummy(*args, **kwargs): return None @@ -71,43 +75,44 @@ def dummy(*args, **kwargs): # (Emma based) report for Coverage # class BaseGenerateXml(object): - def __init__(self, cover_report_name, verbose, use_ci): - self.cover_report_name = cover_report_name + def __init__(self, FullManageProjectName, verbose, use_cte): + projectName = os.path.splitext(os.path.basename(FullManageProjectName))[0] + self.manageProjectName = projectName + self.cover_report_name = os.path.join("xml_data","coverage_results_"+ self.manageProjectName + ".xml") + self.unit_report_name = os.path.join("xml_data","test_results_"+ self.manageProjectName + ".xml") self.verbose = verbose - self.using_cover = False + self.has_sfp_enabled = False + self.print_exc = False + + self.use_cte = use_cte # get the VC langaguge and encoding self.encFmt = getVectorCASTEncoding() - if use_ci: - self.use_ci = " --ci " - else: - self.use_ci = "" + self.fh_data = "" + self.compiler = "" + self.testsuite = "" + self.env = "" + self.build_dir = "" + self.system_tests_status_report_generated = False def generate_system_test_status_report(self): if self.system_tests_status_report_generated: return - report_name = os.path.basename(self.FullManageProjectName)[:-4] + "_system_tests_status.html" - - print(" Creating System Test Status " + self.FullManageProjectName) - callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage -p " + self.FullManageProjectName + " --system-tests-status=" + report_name + print(" Creating System Test Status " + self.FullManageProjectName) + for report_name_ext in [".txt", ".html"]: + report_name = os.path.basename(self.FullManageProjectName)[:-4] + "_system_tests_status" + report_name_ext + callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage --project " + self.FullManageProjectName + " --system-tests-status=" + report_name + import subprocess + p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = p.communicate() - import subprocess + if err: + print("Cannot create system test status report{} {}".format(out, err)) - print(callStr) - p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - out, err = p.communicate() + self.system_tests_status_report_generated = True - if os.path.exists(report_name): - print("File exists: " + report_name) - else: - print("File not exists: " + report_name) - - if err: - print("Cannot create system test status report{} {}".format(out, err)) - - self.system_tests_status_report_generated = True # # BaseGenerateXml - calculate coverage value # @@ -118,38 +123,83 @@ def calc_cov_values(self, x, y): else: column = '%s%% (%d / %d)' % (fmt_percent(x, y), x, y) return column + + def convertTestHistory (self,status): + convertDict = {'TEST_HISTORY_FAILURE_REASON_DATA_SKEW_UNDERFLOW':"Harness Data Underflow", + 'TEST_HISTORY_FAILURE_REASON_DATA_SKEW_OVERFLOW':"Harness Data Overflow", + 'TEST_HISTORY_FAILURE_REASON_HARNESS_FAILURE':"Harness Error", + 'TEST_HISTORY_FAILURE_REASON_THISTORY_FILE_DOES_NOT_EXIST':"Event History Missing", + 'TEST_HISTORY_FAILURE_REASON_THISTORY_LINE_INVALID':"Event Data Invalid", + 'TEST_HISTORY_FAILURE_REASON_THISTORY_ENDED_PREMATURELY':"Event History Processing Failed", + 'TEST_HISTORY_FAILURE_REASON_EXPECTED_ENDED_PREMATURELY':"Event History Processing Failed", + 'TEST_HISTORY_FAILURE_REASON_HARNESS_COMMNAD_INVALID':"Harness Command Invalid", + 'TEST_HISTORY_FAILURE_REASON_TEST_HISTORY_OUTPUT_FILES_CONTAIN_ERROR':"Test History Output Files contain errors", + 'TEST_HISTORY_FAILURE_REASON_STRICT_IMPORT_FAILED':"Strict Import Failure - See Scripting Log under Test=>View ", + 'TEST_HISTORY_FAILURE_REASON_MACRO_NOT_FOUND':"Symbolic constant not found", + 'TEST_HISTORY_FAILURE_REASON_SYMBOL_OR_MACRO_NOT_FOUND':"Symbolic constant not found", + 'TEST_HISTORY_FAILURE_REASON_SYMBOL_OR_MACRO_TYPE_MISMATCH':"Symbolic constant has incorrect type", + 'TEST_HISTORY_FAILURE_REASON_EMPTY_TESTCASES':"Empty Test Case", + 'TEST_HISTORY_FAILURE_REASON_NO_EXPECTED_VALUES':"No expected values", + 'TEST_HISTORY_FAILURE_REASON_NO_EXPECTED_RETURN':"No expected return", + 'TEST_HISTORY_FAILURE_REASON_EXECUTABLE_MISSING':"Executable Missing", + 'TEST_HISTORY_FAILURE_REASON_MAX_VARY_EXCEEDED':"Max Vary Failure - too many Range/List input values ", + 'TEST_HISTORY_FAILURE_REASON_INSUFFICIENT_HEAP_SIZE':"VCAST_malloc failed - insufficient heap.", + 'TEST_HISTORY_FAILURE_REASON_LIBRARY_MALLOC_FAILED':"malloc failed - memory was exhausted", + 'TEST_HISTORY_FAILURE_REASON_TRUNCATED_HARNESS_DATA':"Truncated Harness Data", + 'TEST_HISTORY_FAILURE_REASON_HARNESS_STDOUT_DATA_UNDERFLOW':"Harness Standard Out Data Underflow", + 'TEST_HISTORY_FAILURE_REASON_MAX_STRING_LENGTH_EXCEEDED':"Harness Maximum String Length Exceeded", + 'TEST_HISTORY_FAILURE_REASON_TIMEOUT_EXCEEDED':"Timed Out"} + return convertDict[str(status)] + + def convertTcStatus(self, status): + convertDict = { 'TCR_STATUS_OK' : 'Testcase can run', + 'TCR_STRICT_IMPORT_FAILED' : 'Strict Testcase Import Failure', + 'TCR_MAXIMUM_VARY_EXCEEDED' : 'Maximum varied parameters exceeded', + 'TCR_EMPTY_TEST_CASES' : 'Empty testcase', + 'TCR_NO_EXPECTED_VALUES' : 'No expected values', + 'TCR_NO_EXPECTED_RETURN' : 'No expected return value', + 'TCR_NO_SLOTS' : 'Compound with no slot', + 'TCR_ZERO_ITERATIONS' : 'Compound with zero slot', + 'TCR_RECURSIVE_COMPOUND' : 'Recursive Compound Test', + 'TCR_COMMON_COMPOUND_CONTAINING_SPECIALIZED' : 'Non-specialized compound containing specialized testcases', + 'TCR_HIDING_EXPECTED_RESULTS' : 'Hiding expected results', + 'TCR_MAX_STRING_LENGTH_EXCEEDED' : 'Maximum string length exceeded', + 'TCR_MAX_FILE_COUNT_EXCEEDED' : 'Maximum file count exceeded', + 'TCR_TIMEOUT_EXCEEDED' : 'Testcase timeout', + 'TCR_INTERNAL_ERROR' : 'Internal VectorCAST Error' + } + return convertDict[str(status)] def convertExecStatus(self, status): - convertDict = { 'EXEC_SUCCESS_PASS':['Testcase passed','passed'], - 'EXEC_SUCCESS_FAIL':['Testcase failed','failed'], - 'EXEC_SUCCESS_NONE':['No expected results','run'], - 'EXEC_EXECUTION_FAILED':['Testcase failed to run to completion (possible testcase timeout)','failed'], - 'EXEC_ABORTED':['User aborted testcase','cancelled'], - 'EXEC_TIMEOUT_EXCEEDED':['Testcase timeout','failed'], - 'EXEC_VXWORKS_LOAD_ERROR':['VxWorks load error','notrun'], - 'EXEC_USER_CODE_COMPILE_FAILED':['User code failed to compile','notrun'], - 'EXEC_COMPOUND_ONLY':['Compound only test case','notrun'], - 'EXEC_STRICT_IMPORT_FAILED':['Strict Testcase Import Failure','failed'], - 'EXEC_MACRO_NOT_FOUND':['Macro not found','notrun'], - 'EXEC_SYMBOL_OR_MACRO_NOT_FOUND':['Symbol or macro not found','notrun'], - 'EXEC_SYMBOL_OR_MACRO_TYPE_MISMATCH':['Symbol or macro type mismatch','notrun'], - 'EXEC_MAX_VARY_EXCEEDED':['Maximum varied parameters exceeded','notrun'], - 'EXEC_COMPOUND_WITH_NO_SLOTS':['Compound with no slot','notrun'], - 'EXEC_COMPOUND_WITH_ZERO_ITERATIONS':['Compound with zero slot','notrun'], - 'EXEC_STRING_LENGTH_EXCEEDED':['Maximum string length exceeded','notrun'], - 'EXEC_FILE_COUNT_EXCEEDED':['Maximum file count exceeded','notrun'], - 'EXEC_EMPTY_TESTCASE':['Empty testcase','notrun'], - 'EXEC_NO_EXPECTED_RETURN':['No expected return value','failed'], - 'EXEC_NO_EXPECTED_VALUES':['No expected values','failed'], - 'EXEC_CSV_MAP':['CSV Map','notrun'], - 'EXEC_DRIVER_DATA_COMPILE_FAILED':['Driver data failed to compile','notrun'], - 'EXEC_RECURSIVE_COMPOUND':['Recursive Compound Test','failed'], - 'EXEC_SPECIALIZED_COMPOUND_CONTAINING_COMMON':['Specialized compound containing non-specialized testcases','failed'], - 'EXEC_COMMON_COMPOUND_CONTAINING_SPECIALIZED':['Non-specialized compound containing specialized testcases','failed'], - 'EXEC_HIDING_EXPECTED_RESULTS':['Hiding expected results','run'], - 'INVALID_TEST_CASE':['Invalid Test Case','failed'] - } - + convertDict = { 'EXEC_SUCCESS_PASS':'Testcase passed', + 'EXEC_SUCCESS_FAIL':'Testcase failed', + 'EXEC_SUCCESS_NONE':'No expected results', + 'EXEC_EXECUTION_FAILED':'Testcase failed to run to completion (possible testcase timeout)', + 'EXEC_ABORTED':'User aborted testcase', + 'EXEC_TIMEOUT_EXCEEDED':'Testcase timeout', + 'EXEC_VXWORKS_LOAD_ERROR':'VxWorks load error', + 'EXEC_USER_CODE_COMPILE_FAILED':'User code failed to compile', + 'EXEC_COMPOUND_ONLY':'Compound only test case', + 'EXEC_STRICT_IMPORT_FAILED':'Strict Testcase Import Failure', + 'EXEC_MACRO_NOT_FOUND':'Macro not found', + 'EXEC_SYMBOL_OR_MACRO_NOT_FOUND':'Symbol or macro not found', + 'EXEC_SYMBOL_OR_MACRO_TYPE_MISMATCH':'Symbol or macro type mismatch', + 'EXEC_MAX_VARY_EXCEEDED':'Maximum varied parameters exceeded', + 'EXEC_COMPOUND_WITH_NO_SLOTS':'Compound with no slot', + 'EXEC_COMPOUND_WITH_ZERO_ITERATIONS':'Compound with zero slot', + 'EXEC_STRING_LENGTH_EXCEEDED':'Maximum string length exceeded', + 'EXEC_FILE_COUNT_EXCEEDED':'Maximum file count exceeded', + 'EXEC_EMPTY_TESTCASE':'Empty testcase', + 'EXEC_NO_EXPECTED_RETURN':'No expected return value', + 'EXEC_NO_EXPECTED_VALUES':'No expected values', + 'EXEC_CSV_MAP':'CSV Map', + 'EXEC_DRIVER_DATA_COMPILE_FAILED':'Driver data failed to compile', + 'EXEC_RECURSIVE_COMPOUND':'Recursive Compound Test', + 'EXEC_SPECIALIZED_COMPOUND_CONTAINING_COMMON':'Specialized compound containing non-specialized testcases', + 'EXEC_COMMON_COMPOUND_CONTAINING_SPECIALIZED':'Non-specialized compound containing specialized testcases', + 'EXEC_HIDING_EXPECTED_RESULTS':'Hiding expected results', + 'INVALID_TEST_CASE':'Invalid Test Case' + } try: s = convertDict[str(status)] except: @@ -201,31 +251,28 @@ def add_coverage(self, is_unit, unit_or_func, metrics, cov_type): if self.has_call_coverage: entry["functioncall"] = self.calc_cov_values( metrics.max_covered_function_calls + - metrics.max_annotated_function_calls, + metrics.max_annotations_function_calls, metrics.function_calls ) - if self.verbose: - print("Coverage Type:", cov_type) - if 'NONE' in cov_type_str: return entry if "MCDC" in cov_type_str: entry["mcdc"] = self.calc_cov_values( metrics.max_covered_mcdc_branches + - metrics.max_annotated_mcdc_branches, + metrics.max_annotations_mcdc_branches, metrics.mcdc_branches ) if not self.simplified_mcdc: entry["mcdc"] = self.calc_cov_values( metrics.max_covered_mcdc_pairs + - metrics.max_annotated_mcdc_pairs, + metrics.max_annotations_mcdc_pairs, metrics.mcdc_pairs ) entry["branch"] = self.calc_cov_values( metrics.max_covered_mcdc_branches + - metrics.max_annotated_mcdc_branches, + metrics.max_annotations_mcdc_branches, metrics.mcdc_branches ) if "BASIS_PATH" in cov_type_str: @@ -234,31 +281,87 @@ def add_coverage(self, is_unit, unit_or_func, metrics, cov_type): if "STATEMENT" in cov_type_str: entry["statement"] = self.calc_cov_values( metrics.max_covered_statements + - metrics.max_annotated_statements, + metrics.max_annotations_statements, metrics.statements ) if "BRANCH" in cov_type_str: entry["branch"] = self.calc_cov_values( metrics.max_covered_branches + - metrics.max_annotated_branches, + metrics.max_annotations_branches, metrics.branches ) if "FUNCTION_FUNCTION_CALL" in cov_type_str: entry["functioncall"] = self.calc_cov_values( metrics.max_covered_function_calls + - metrics.max_annotated_function_calls, + metrics.max_annotations_function_calls, metrics.function_calls ) entry["function"] = self.calc_cov_values( metrics.max_covered_functions + - metrics.max_annotated_functions, + metrics.max_annotations_functions, metrics.functions ) return entry +# +# BaseGenerateXml - write the units to the coverage file +# + def write_cov_units(self): + + self.reported_units = {} + + for unit in self.our_units: + unit_name = unit["unit"].name + if unit_name in self.reported_units.keys(): + self.reported_units[unit_name] += 1 + unit_name = unit_name + "'%d" % self.reported_units[unit_name] + else: + self.reported_units[unit_name] = 0 + + self.fh_data += (' \n' % escape(unit_name, quote=False)) + if unit["coverage"]["statement"]: + self.fh_data += (' \n' % unit["coverage"]["statement"]) + if unit["coverage"]["branch"]: + self.fh_data += (' \n' % unit["coverage"]["branch"]) + if unit["coverage"]["mcdc"]: + self.fh_data += (' \n' % unit["coverage"]["mcdc"]) + if unit["coverage"]["basispath"]: + self.fh_data += (' \n' % unit["coverage"]["basispath"]) + if unit["coverage"]["function"]: + self.fh_data += (' \n' % unit["coverage"]["function"]) + if unit["coverage"]["functioncall"]: + self.fh_data += (' \n' % unit["coverage"]["functioncall"]) + self.fh_data += (' \n' % unit["complexity"]) + + for func in unit["functions"]: + + # if isinstance(self.api, CoverApi) or isinstance(self.api, VCProjectApi): + # func_name = escape(func["func"].name, quote=True) + # else: + # func_name = escape(func["func"].display_name, quote=True) + + func_name = escape(func["func"].name, quote=True) + self.fh_data += (' \n' % func_name) + + if func["coverage"]["statement"]: + self.fh_data += (' \n' % func["coverage"]["statement"]) + if func["coverage"]["branch"]: + self.fh_data += (' \n' % func["coverage"]["branch"]) + if func["coverage"]["mcdc"]: + self.fh_data += (' \n' % func["coverage"]["mcdc"]) + if func["coverage"]["basispath"]: + self.fh_data += (' \n' % func["coverage"]["basispath"]) + if func["coverage"]["function"]: + self.fh_data += (' \n' % func["coverage"]["function"]) + if func["coverage"]["functioncall"]: + self.fh_data += (' \n' % func["coverage"]["functioncall"]) + self.fh_data += (' \n' % func["complexity"]) + + self.fh_data += (' \n') + self.fh_data += (' \n') # -# Internal - calculate 'grand total' coverage values for coverage report +# BaseGenerateXml - calculate 'grand total' coverage values for coverage report # def grand_total_coverage(self, cov_type): @@ -272,55 +375,55 @@ def grand_total_coverage(self, cov_type): entry["function"] = None entry["functioncall"] = None - if self.has_function_coverage: + if self.toplevel_has_function_coverage: entry["function"] = self.calc_cov_values( self.grand_total_max_covered_functions, self.grand_total_max_coverable_functions ) - if self.has_call_coverage: + if self.toplevel_has_call_coverage: entry["functioncall"] = self.calc_cov_values( self.grand_total_max_covered_function_calls, self.grand_total_function_calls ) - if cov_type == None: - return entry + if "MCDC" in cov_type_str: entry["mcdc"] = self.calc_cov_values( - self.grand_total_max_mcdc_covered_branches, + self.grand_total_max_mcdc_covered_branches, self.grand_total_mcdc_branches ) if not self.simplified_mcdc: entry["mcdc"] = self.calc_cov_values( - self.grand_total_max_covered_mcdc_pairs, + self.grand_total_max_covered_mcdc_pairs, self.grand_total_mcdc_pairs ) entry["branch"] = self.calc_cov_values( - self.grand_total_max_mcdc_covered_branches, + self.grand_total_max_mcdc_covered_branches, self.grand_total_mcdc_branches ) if "BASIS_PATH" in cov_type_str: entry["basis_path"] = self.calc_cov_values( - self.grand_total_cov_basis_path, + self.grand_total_cov_basis_path, self.grand_total_total_basis_path ) if "STATEMENT" in cov_type_str: entry["statement"] = self.calc_cov_values( - self.grand_total_max_covered_statements, + self.grand_total_max_covered_statements, self.grand_total_statements ) if "BRANCH" in cov_type_str: entry["branch"] = self.calc_cov_values( - self.grand_total_max_covered_branches, + self.grand_total_max_covered_branches, self.grand_total_branches ) if "FUNCTION_FUNCTION_CALL" in cov_type_str: entry["functioncall"] = self.calc_cov_values( - self.grand_total_max_covered_function_calls, + self.grand_total_max_covered_function_calls, self.grand_total_function_calls ) return entry + # -# Internal - generate the formatted timestamp to write to the coverage file +# BaseGenerateXml - generate the formatted timestamp to write to the coverage file # def get_timestamp(self): dt = datetime.now() @@ -333,20 +436,28 @@ def get_timestamp(self): # BaseGenerateXml - start writing to the coverage file # def start_cov_file(self): - if self.verbose: - print(" Writing coverage xml file: {}".format(self.cover_report_name)) - self.fh = open(self.cover_report_name, "wb") - data = "\n".format(self.get_timestamp()) - data += "\n" - data += " \n" - self.fh.write(data.encode(self.encFmt,"replace")) + + self.fh_data = "" + self.fh_data += ('\n' % self.get_timestamp()) + self.fh_data += ('\n') + self.fh_data += (' \n') # # BaseGenerateXml - write the end of the coverage file and close it # def end_cov_file(self): - self.fh.write('') - self.fh.close() + self.fh_data += ('') + with open(self.cover_report_name,"wb") as fd: + fd.write(self.fh_data.encode(self.encFmt, "replace")) + +# +# BaseGenerateXml - write the end of the coverage file and close it +# + def end_cov_file_environment(self, useEnvs = True): + self.fh_data += (' \n') + self.fh_data += (' \n') + self.fh_data += (' \n') + self.end_cov_file() # # BaseGenerateXml the XML Modified 'Emma' coverage data # @@ -404,12 +515,15 @@ def hasAnyCov(self, srcFile): # BaseGenerateXml the XML Modified 'Emma' coverage data # def _generate_cover(self, cov_type): + self.num_functions = 0 self.simplified_mcdc = self.api.environment.get_option("VCAST_SIMPLIFIED_CONDITION_COVERAGE") self.our_units = [] self.has_call_coverage = False self.has_function_coverage = False + self.toplevel_has_function_coverage = False + self.toplevel_has_call_coverage = False self.grand_total_complexity = 0 self.grand_total_max_covered_branches = 0 @@ -505,44 +619,40 @@ def _generate_cover(self, cov_type): if functions_added: self.our_units.append(entry) - self.grand_total_branches += ( - metrics.branches + metrics.mcdc_branches - ) - self.grand_total_statements += metrics.statements - self.grand_total_mcdc_branches += metrics.mcdc_branches - self.grand_total_mcdc_pairs += metrics.mcdc_pairs - self.grand_total_function_calls += metrics.function_calls - - self.grand_total_max_covered_statements += ( - metrics.max_covered_statements + - metrics.max_annotated_statements - ) self.grand_total_max_covered_branches += ( - metrics.max_covered_branches + + metrics.max_covered_branches + metrics.max_covered_mcdc_branches + - metrics.max_annotated_branches + - metrics.max_annotated_mcdc_branches + metrics.max_annotations_branches + + metrics.max_annotations_mcdc_branches ) - + self.grand_total_branches += metrics.branches + metrics.mcdc_branches + self.grand_total_max_covered_statements += ( + metrics.max_covered_statements + metrics.max_annotations_statements + ) + self.grand_total_statements += metrics.statements self.grand_total_max_mcdc_covered_branches += ( metrics.max_covered_mcdc_branches + - metrics.max_annotated_mcdc_branches + metrics.max_annotations_mcdc_branches ) - + self.grand_total_mcdc_branches += metrics.mcdc_branches self.grand_total_max_covered_mcdc_pairs += ( metrics.max_covered_mcdc_pairs + - metrics.max_annotated_mcdc_pairs + metrics.max_annotations_mcdc_pairs ) + + self.grand_total_mcdc_pairs += metrics.mcdc_pairs self.grand_total_max_covered_function_calls += ( - metrics.max_covered_function_calls + - metrics.max_annotated_function_calls + metrics.max_covered_function_calls + + metrics.max_annotations_function_calls ) + self.grand_total_function_calls += metrics.function_calls + try: if self.has_function_coverage: self.grand_total_max_covered_functions += ( metrics.max_covered_functions + - metrics.max_annotated_functions + metrics.max_annotations_functions ) self.grand_total_max_coverable_functions += ( metrics.functions @@ -562,45 +672,447 @@ def _generate_cover(self, cov_type): # # BaseGenerateXml - Generate the XML Modified 'Emma' coverage data # -class GenerateManageXml (BaseGenerateXml): - def __init__(self, cover_report_name, verbose, manage_path, use_ci): - super(GenerateManageXml, self).__init__(cover_report_name, verbose, use_ci) - self.using_cover = True - from vector.apps.DataAPI.manage_api import VCProjectApi + def generate_cover(self): + self.units = [] + if isinstance(self.api, CoverApi): + self.units = self.api.File.all() + self.units.sort(key=lambda x: (x.coverage_type, x.unit_index)) + else: + self.units = self.api.Unit.all() + + # unbuilt (re: Error) Ada environments causing a crash + try: + cov_type = self.api.environment.coverage_type_text + except Exception as e: + parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) + return + + self._generate_cover(cov_type) - self.api = VCProjectApi(manage_path) + self.start_cov_file_environment() + self.write_cov_units() + self.end_cov_file_environment() - def write_coverage_data(self): - data = " \n".format(self.grand_total_complexity) +# +# BaseGenerateXml - write the start of the coverage file for and environment +# + def start_cov_file_environment(self): + self.start_cov_file() + self.fh_data += (' \n') + self.fh_data += (' \n') + self.fh_data += (' \n' % self.num_units) + self.fh_data += (' \n' % self.num_functions) + self.fh_data += (' \n') + self.fh_data += (' \n') + + self.fh_data += (' \n') if self.coverage["statement"]: - data += " \n".format(self.coverage["statement"]) + self.fh_data += (' \n' % self.coverage["statement"]) if self.coverage["branch"]: - data += " \n".format(self.coverage["branch"]) + self.fh_data += (' \n' % self.coverage["branch"]) if self.coverage["mcdc"]: - data += " \n".format(self.coverage["mcdc"]) + self.fh_data += (' \n' % self.coverage["mcdc"]) if self.coverage["basispath"]: - data += " \n".format(self.coverage["basispath"]) + self.fh_data += (' \n' % self.coverage["basispath"]) if self.coverage["function"]: - data += " \n".format(self.coverage["function"]) + self.fh_data += (' \n' % self.coverage["function"]) if self.coverage["functioncall"]: - data += " \n".format(self.coverage["functioncall"]) - self.fh.write(data.encode(self.encFmt, "replace")) + self.fh_data += (' \n' % self.coverage["functioncall"]) + self.fh_data += (' \n' % self.grand_total_complexity) + self.fh_data += ('\n') + + if isinstance(self, GenerateManageXml): + self.fh_data += (' \n' % escape(self.manageProjectName, quote=False)) + else: + self.fh_data += (' \n' % escape(self.jenkins_name, quote=False)) + if self.coverage["statement"]: + self.fh_data += (' \n' % self.coverage["statement"]) + if self.coverage["branch"]: + self.fh_data += (' \n' % self.coverage["branch"]) + if self.coverage["mcdc"]: + self.fh_data += (' \n' % self.coverage["mcdc"]) + if self.coverage["basispath"]: + self.fh_data += (' \n' % self.coverage["basispath"]) + if self.coverage["function"]: + self.fh_data += (' \n' % self.coverage["function"]) + if self.coverage["functioncall"]: + self.fh_data += (' \n' % self.coverage["functioncall"]) + self.fh_data += (' \n' % self.grand_total_complexity) + self.fh_data += ('\n') + +########################################################################## +# This class generates the XML (JUnit based) report for the overall +# (Emma based) report for Coverage +# +class GenerateManageXml (BaseGenerateXml): + +# GenerateManageXml + + def __init__(self, FullManageProjectName, verbose = False, + cbtDict = None, + generate_exec_rpt_each_testcase = True, + use_archive_extract = False, + report_failed_only = False, + no_full_reports = False, + print_exc = False, + useStartLine = False, + use_cte = False): + + super(GenerateManageXml, self).__init__(FullManageProjectName, verbose, use_cte) + + self.FullManageProjectName = FullManageProjectName + self.generate_exec_rpt_each_testcase = generate_exec_rpt_each_testcase + self.use_archive_extract = use_archive_extract + self.report_failed_only = report_failed_only + self.cbtDict = cbtDict + self.no_full_reports = no_full_reports + self.failed_count = 0 + self.passed_count = 0 + self.print_exc = print_exc + + self.units = [] + + self.useStartLine = useStartLine + + self.cleanupXmlDataDir() + + vcproj = VCProjectApi(FullManageProjectName) + + try: + self.has_sfp_enabled = vcproj.environment.get_option("VCAST_COVERAGE_SOURCE_FILE_PERSPECTIVE") + except: + self.has_sfp_enabled = False + + hasCover = any(isinstance(env.api, CoverApi) for env in vcproj.Environment.all()) + vcproj.close() + + if hasCover: + self.generate_system_test_status_report() + + self.api = VCProjectApi(FullManageProjectName) + + def cleanupXmlDataDir(self): + path="xml_data" + import glob + # if the path exists, try to delete all file in it + if os.path.isdir(path): + for file in glob.glob(path + "/*.*", recursive=False): + try: + os.remove(file); + except: + print(" *INFO: File System Error removing file after failed to remove directory: " + path + "/" + file + ". Check console for environment build/execution errors") + if print_exc: traceback.print_exc() + + # we should either have an empty directory or no directory + else: + try: + os.mkdir(path) + except: + print("failed making path: " + path) + print(" *INFO: File System Error creating directory: " + path + ". Check console for environment build/execution errors") + if print_exc: traceback.print_exc() def __del__(self): try: self.api.close() except: + print("[DEBUG] Exception closing in self.api generate_xml::GenerateManageXml::__del__") pass # GenerateManageXml def generate_cover(self): - self.units = self.api.project.cover_api.File.all() + + environments = self.api.Environment.all() + + localDisplayPaths = [] + for env in environments: + if not env.is_active: + continue + try: + n = len(env.api.SourceFile.all()) + except: + continue + for srcFile in env.api.SourceFile.all(): + display_path = srcFile.display_path + if display_path not in localDisplayPaths: + localDisplayPaths.append(display_path) + + + localUnits = self.api.project.cover_api.SourceFile.all() ##self.api.project.cover_api.File.all() + localUnits.sort(key=lambda x: (x.name)) + for unit in localUnits: + if unit.display_path in localDisplayPaths: + self.units.append(unit) + self._generate_cover(None) - self.start_cov_file() - self.write_coverage_data() - self.end_cov_file() - self.api.close() + self.start_cov_file_environment() + self.write_cov_units() + self.end_cov_file_environment() + + def fixupReport(self, report_name): + + fixup = False + if self.api.tool_version.startswith("19 "): + fixup = True + elif self.api.tool_version.startswith("19sp1"): + fixup = True + + if not fixup: + return + + with open(report_name,"rb") as fd: + data = fd.read().decode('utf-8','replace') + + #fix up inline CSS because of Content Security Policy violation + newData = data[: data.index("")+8:] + + #fix up style directive because of Content Security Policy violation + newData = newData.replace("
","
") + + #fixup the inline VectorCAST image because of Content Security Policy violation + regex_str = r"\"Vector\".*"",newData) + + with open(report_name, "wb") as fd: + fd.write(newData.encode('utf-8','replace')) + + workspace = os.getenv("WORKSPACE") + if workspace is None: + workspace = os.getcwd() + + vc_scripts = os.path.join(workspace,"vc_scripts") + + shutil.copy(os.path.join(vc_scripts,"vector_style.css"), "management/vector_style.css") + shutil.copy(os.path.join(vc_scripts,"vectorcast.png"), "management/vectorcast.png") + + def generate_local_results(self, results, key): + # get the level from the name + + if len(key.split("/")) != 3: + comp, ts, group, env_name = key.split("/") + else: + comp, ts, env_name = key.split("/") + + env_key = comp + "/" + ts + "/" + env_name + + env = self.api.project.environments[env_key] + env_def = self.api.project.environments[env_key].definition + + build_dir = env.build_directory + vceFile = os.path.join(build_dir, env.name+".vce") + vcpFile = os.path.join(build_dir, env.name+".vcp") + if not os.path.exists(vceFile) and not os.path.exists(vcpFile): + print("Error: Could not determine environment location for {}/{}".format(build_dir, env.name)) + print(" {}/{}/{}".format(comp, ts, env_name)) + return + + xmlUnitReportName = os.getcwd() + os.sep + "xml_data" + os.sep + "test_results_" + "_".join([comp, ts, env_name]) + ".xml" + + localXML = None + + localXML = GenerateXml(self.FullManageProjectName, build_dir, env_name, comp, ts, + None, key, xmlUnitReportName, None, None, self.verbose, + self.cbtDict, + self.generate_exec_rpt_each_testcase, + self.use_archive_extract, + self.report_failed_only, + self.print_exc, + self.useStartLine, + self.use_cte, + self.system_tests_status_report_generated) + + localXML.topLevelAPI = self.api + localXML.noResults = self.noResults + localXML.generate_unit() + + ##need_fixup + if not self.no_full_reports: + try: + unit_test_models.clear_caches(localXML.api) + except: + pass + report_name = os.path.join("management", comp + "_" + ts + "_" + env_name + ".html") + try: + if isinstance(localXML.api, CoverApi): + try: + localXML.api.report(report_type="AGGREGATE_REPORT", formats=["HTML"], output_file=report_name) + except: + if self.verbose: + print("Failed to create " + report_name + " by CustomReport API. Using clicast directly") + self.runAggregateReport(comp, ts, env_name, report_name) + else: + try: + localXML.api.report(report_type="FULL_REPORT", formats=["HTML"], output_file=report_name) + except: + if self.verbose: + print("Failed to create " + report_name + " by CustomReport API. Using clicast directly") + self.runFullReport(comp, ts, env_name, report_name) + self.fixupReport(report_name) + except: + print("Error creating report " + report_name + ". Contact Vector Support") + parse_traceback.parse(traceback.format_exc(), self.verbose, self.compiler, self.testsuite, self.env, self.build_dir) + + def runFullReport(self,comp,ts,env_name,report_name): + try: + from managewait import ManageWait + callStr = "--project " + self.FullManageProjectName + " --level " + comp + "/" + ts + " --environment " + env_name + " --clicast-args report custom full" + + manageWait = ManageWait(False, callStr, 1, 1) + out = manageWait.exec_manage(True) + fname = None + for line in out.split("\n"): + if "The HTML report was saved to" in line: + fname = line.split("\"")[1] + + if fname: + import shutil + shutil.copyfile(fname, report_name) + else: + print("Error creating report " + report_name + ". Contact Vector Support") + except: + traceback.print_exc() + + def runAggregateReport(self,comp,ts,env_name,report_name): + try: + from managewait import ManageWait + callStr = "--project " + self.FullManageProjectName + " --level " + comp + "/" + ts + " --environment " + env_name + " --clicast-args cover report aggregate" + + manageWait = ManageWait(False, callStr, 1, 1) + out = manageWait.exec_manage(True) + for line in out.split("\n"): + if "The HTML report was saved to" in line: + fname = line.split("\"")[1] + if fname: + import shutil + shutil.copyfile(fname, report_name) + else: + print("Error creating report " + report_name + ". Contact Vector Support") + except: + traceback.print_exc() + + def skipReporting(self, env): + + build_dir = "" + + if self.use_archive_extract and self.cbtDict: + try: + prj_dir = os.environ['WORKSPACE'].replace("\\","/") + "/" + except: + prj_dir = os.getcwd().replace("\\","/") + "/" + + try: + build_dir = os.path.relpath(env.build_directory,prj_dir).replace("\\","/") + except: + build_dir = env.build_directory.replace("\\","/") + + try: + build_dir = "build/" + build_dir.rsplit("build/",1)[-1] + + except: + traceback.print_exc() + print("exception converting directory into relative path: {} {}".format(env.build_directory, build_dir)) + + ## use hash code instead of final directory name as regression scripts can have overlapping final directory names + + build_dir_4hash = build_dir.upper() + build_dir_4hash = "/".join(build_dir_4hash.split("/")[-2:]) + + # Unicode-objects must be encoded before hashing in Python 3 + if sys.version_info[0] >= 3: + build_dir_4hash = build_dir_4hash.encode('utf-8') + + hashCode = hashlib.md5(build_dir_4hash).hexdigest() + + if hashCode not in self.cbtDict.keys(): + if self.verbose: + print("Skipping report because hashCode (" + hashCode + ") for build dir (" + build_dir + ") not found in cbtdict") + + return True + else: + c,i,s = self.cbtDict[hashCode] + if len(c)==0 and len(i)==0 and len(s)==0: + if self.verbose: + print("skipping report because c,i,s are all 0 size") + return True + + return False + +# GenerateManageXml + def generate_testresults(self): + testcaseString = """ + + %s + +""" + results = self.api.project.repository.get_full_status([]) + all_envs = [] + for env in self.api.Environment.all(): + + if self.skipReporting(env): + continue + + if env.is_active: + all_envs.append(env.level._full_path) + + self.fh_data = "" + self.localDataOnly = True + self.noResults = False + if results['ALL']['testcase_results'] == {}: + print("** No results in project") + self.noResults = True + else: + total = results['ALL']['testcase_results']['total_count'] + success = results['ALL']['testcase_results']['success_count'] + errors = total - success + failed = errors + self.fh_data += ("\n") + self.fh_data += ("\n") + self.fh_data += (" \n" % + (errors,total,failed,escape(self.manageProjectName, quote=False))) + + self.failed_count = errors + self.passed_count = success + + for result in results: + if result in all_envs: + if len(result.split("/")) != 3: + comp, ts, group, env_name = result.split("/") + else: + comp, ts, env_name = result.split("/") + + if results[result]['local'] != {}: + self.generate_local_results(results,result) + else: + for key in results[result]['imported'].keys(): + self.localDataOnly = False + importedResult = results[result]['imported'][key] + total = importedResult['testcase_results']['total_count'] + success = importedResult['testcase_results']['success_count'] + errors = total - success + failed = errors + importName = importedResult['name'] + classname = "ImportedResults." + importName + "." + comp + "." + ts + "." + env_name + classname = comp + "." + ts + "." + env_name + for idx in range(1,success+1): + tc_name_full = "ImportedResults." + importName + ".TestCase.PASS.%03d" % idx + extraStatus = "\n \n" + self.fh_data += (testcaseString % (tc_name_full, classname, extraStatus)) + self.passed_count += 1 + + for idx in range(1,failed+1): + tc_name_full = "ImportedResults." + importName + ".TestCase.FAIL.%03d" % idx + extraStatus = "\n \n" + self.fh_data += (testcaseString % (tc_name_full, classname, extraStatus)) + self.failed_count += 1 + + self.fh_data += (" \n") + self.fh_data += ("\n") + if not self.localDataOnly: + with open(self.unit_report_name, "wb") as fd: + fd.write(self.fh_data.encode(self.encFmt, "replace")) ########################################################################## # This class generates the XML (Junit based) report for dynamic tests and @@ -610,10 +1122,21 @@ def generate_cover(self): # class GenerateXml(BaseGenerateXml): - def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, cover_report_name, jenkins_name, unit_report_name, jenkins_link, jobNameDotted, verbose = False, cbtDict= None, use_ci = False): - super(GenerateXml, self).__init__(cover_report_name, verbose, use_ci) + def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, cover_report_name, jenkins_name, unit_report_name, jenkins_link, jobNameDotted, verbose = False, cbtDict= None, generate_exec_rpt_each_testcase = True, + use_archive_extract = False, report_failed_only = False, print_exc = False, useStartLine = False, useCI = None, use_cte = False, system_tests_status_report_generated = False): + + super(GenerateXml, self).__init__(FullManageProjectName, verbose, use_cte) + self.cbtDict = cbtDict self.FullManageProjectName = FullManageProjectName + self.generate_exec_rpt_each_testcase = generate_exec_rpt_each_testcase + self.use_archive_extract = use_archive_extract + self.report_failed_only = report_failed_only + self.print_exc = print_exc + self.topLevelAPI = None + self.noResults = False + self.useStartLine = useStartLine + self.system_tests_status_report_generated = system_tests_status_report_generated ## use hash code instead of final directory name as regression scripts can have overlapping final directory names build_dir = build_dir.replace("\\","/") @@ -630,6 +1153,7 @@ def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, c if verbose: print ("HashCode: " + self.hashCode + "for build dir: " + build_dir) + print(env) self.build_dir = build_dir self.env = env @@ -640,39 +1164,14 @@ def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, c self.unit_report_name = unit_report_name self.jenkins_link = jenkins_link self.jobNameDotted = jobNameDotted - self.using_cover = False cov_path = os.path.join(build_dir,env + '.vcp') unit_path = os.path.join(build_dir,env + '.vce') - self.failed_count = 0 - self.passed_count = 0 - self.useStartLine = False - self.noResults = False - self.report_failed_only = False - self.cbtDict = None - - if os.path.exists(cov_path) and os.path.exists(cov_path[:-4]): - self.using_cover = True + if os.path.exists(cov_path): self.generate_system_test_status_report() - try: - self.api = CoverApi(cov_path) - except: - self.api = None - return - - elif os.path.exists(unit_path) and os.path.exists(unit_path[:-4]): - self.using_cover = False - try: - self.api = UnitTestApi(unit_path) - except: - self.api = None - - return - - if self.api.environment.status != ENVIRONMENT_STATUS_TYPE_T.NORMAL: - self.api.close() - self.api = None - return + self.api = CoverApi(cov_path) + elif os.path.exists(unit_path): + self.api = UnitTestApi(unit_path) else: self.api = None if verbose: @@ -708,13 +1207,15 @@ def add_init_tests(self): def generate_unit(self): if isinstance(self.api, CoverApi): - try: - from vector.apps.DataAPI.vcproject_api import VCProjectApi self.start_system_test_file() - api = VCProjectApi(self.FullManageProjectName) - for env in api.Environment.all(): + if self.topLevelAPI == None: + vcproj = VCProjectApi(self.FullManageProjectName) + else: + vcproj = self.topLevelAPI + + for env in vcproj.Environment.all(): if env.compiler.name == self.compiler and env.testsuite.name == self.testsuite and env.name == self.env and env.system_tests: for st in env.system_tests: pass_fail_rerun = "" @@ -728,19 +1229,17 @@ def generate_unit(self): pass_fail_rerun = ": Failed" level = env.compiler.name + "/" + env.testsuite.name + "/" + env.name - if self.verbose: - print (level, st.name, pass_fail_rerun) - self.write_testcase(st, level, st.name) - api.close() + self.write_testcase(st, level, st.name, env.definition.is_monitored) - except ImportError as e: - pass - - from generate_qa_results_xml import genQATestResults - pc,fc = genQATestResults(self.FullManageProjectName, self.compiler + "/" + self.testsuite, self.env, True, self.encFmt) - self.failed_count += fc - self.passed_count += pc + if self.topLevelAPI == None: + vcproj.close() + except ImportError as e: + from generate_qa_results_xml import genQATestResults + pc,fc = genQATestResults(self.FullManageProjectName, self.compiler + "/" + self.testsuite, self.env, True, self.encFmt) + self.failed_count += fc + self.passed_count += pc + return else: try: @@ -790,21 +1289,31 @@ def isTcPlaceHolder(self, tc): return placeHolder # -# Internal - start the JUnit XML file +# GenerateXml - write the end of the jUnit XML file and close it +# + def end_test_results_file(self): + self.fh_data += (" \n") + self.fh_data += ("\n") + with open(self.unit_report_name, "wb") as fd: + fd.write(self.fh_data.encode(self.encFmt,"replace")) + +# +# GenerateXml - start the JUnit XML file # - def start_system_test_file(self): - if self.verbose: - print(" Writing testcase xml file: {}".format(self.unit_report_name)) - self.fh = open(self.unit_report_name, "wb") + def start_system_test_file(self): errors = 0 failed = 0 success = 0 from vector.apps.DataAPI.vcproject_api import VCProjectApi - api = VCProjectApi(self.FullManageProjectName) - for env in api.Environment.all(): + if self.topLevelAPI == None: + vcproj = VCProjectApi(self.FullManageProjectName) + else: + vcproj = self.topLevelAPI + + for env in vcproj.Environment.all(): if env.compiler.name == self.compiler and env.testsuite.name == self.testsuite and env.name == self.env and env.system_tests: for st in env.system_tests: if st.passed == st.total: @@ -814,45 +1323,93 @@ def start_system_test_file(self): failed += 1 errors += 1 self.failed_count += 1 - api.close() - data = "\n".format(self.encFmt) - data += "\n" - data += " \n".format(errors, success+failed+errors, failed, escape(self.env, quote=False)) + if self.topLevelAPI == None: + vcproj.close() - self.fh.write(data.encode(self.encFmt, "replace")) + self.fh_data = "" + self.fh_data += ("\n") + self.fh_data += ("\n") + self.fh_data += (" \n" % + (errors,success+failed+errors, failed, escape(self.env, quote=False))) def start_unit_test_file(self): - if self.verbose: - print(" Writing testcase xml file: {}".format(self.unit_report_name)) - self.fh = open(self.unit_report_name, "wb") errors = 0 failed = 0 success = 0 for tc in self.api.TestCase.all(): - if (not tc.for_compound_only or tc.testcase_status == "TCR_STRICT_IMPORT_FAILED") and not self.isTcPlaceHolder(tc): + if not self.noResults and (not tc.for_compound_only or tc.testcase_status == "TCR_STRICT_IMPORT_FAILED") and not self.isTcPlaceHolder(tc): if not tc.passed: self.failed_count += 1 - failed += 1 if tc.execution_status != "EXEC_SUCCESS_FAIL ": errors += 1 + else: + failed += 1 else: success += 1 self.passed_count += 1 + self.fh_data = "" + self.fh_data += ("\n") + self.fh_data += ("\n") + self.fh_data += (" \n" % + (errors,success+failed+errors, failed, escape(self.env, quote=False))) + + def testcase_failed(self, tc): + + try: + from vector.apps.DataAPI.manage_models import SystemTest + if (isinstance(tc, SystemTest)): + if tc.run_needed and tc.type == 2: + return False + elif tc.run_needed: + return False + elif tc.passed == tc.total: + return False + else: + return True + except: + pass + + if not tc.passed: + return True + + return False + + def get_xml_string(self, fpath = None): - data = "\n" - data += "\n" - data += " \n".format( - errors, - success+failed+errors, - failed, - escape(self.env, quote=False) - ) + if False: #fpath: + testcaseStringExtraStatus=""" + + %s + +%s + + +""" - self.fh.write(data.encode(self.encFmt, "replace")) + testcaseString =""" + + %s + +""" + else: + testcaseStringExtraStatus=""" + + %s + +%s + + +""" + testcaseString =""" + + %s + +""" + return testcaseString, testcaseStringExtraStatus # # GenerateXml - write a testcase to the jUnit XML file # @@ -861,16 +1418,22 @@ def write_testcase(self, tc, unit_name, func_name, st_is_monitored = False, unit fpath = "" startLine = "" unitName = "" + + failureReasons = "" + didntRunReason = "" + + if unit: + if tc.status == "TC_EXECUTION_PASSED": + pass - unitName = unit_name - - if self.noResults: - return + if tc.status == "TC_EXECUTION_FAILED": + for reason in tc.failure_reasons: + failureReasons += self.convertTestHistory(reason) + ' | ' + failureReasons = failureReasons[:-3] - if self.report_failed_only and not self.testcase_failed(tc): - return + if tc.status == "TC_EXECUTION_NONE": + didntRunReason = self.convertTcStatus(tc.testcase_status) - if unit: try: filePath = unit.sourcefile.normalized_path(normcase=False) except: @@ -894,12 +1457,20 @@ def write_testcase(self, tc, unit_name, func_name, st_is_monitored = False, unit startLine = list(tc.cover_data.covered_statements)[0].start_line except: startLine = "0" - print("failed to access any start_line ", self.env, func_name, tc.name) + print("failed to access any start_line {} {} {}".format(self.env, func_name, tc.name)) else: startLine = "0" unitName = unit.name + if self.noResults: + return + + failure_message = "" + + if self.report_failed_only and not self.testcase_failed(tc): + return + isSystemTest = False try: @@ -941,34 +1512,22 @@ def write_testcase(self, tc, unit_name, func_name, st_is_monitored = False, unit envName = escape(self.env, quote=False).replace(".","") classname = compiler + "." + testsuite + "." + envName - extra_message = "" - status = "" - control_flow_fail = False - exception_fail = False - signal_fail = False if isSystemTest: - if fpath == "": - fpath = tc_name - tc_name_full = classname + "." + tc_name exp_total = tc.total exp_pass = tc.passed - extra_message = "System Test Build Status: " + tc.build_status + ". System Test: " + tc.name + ". " + result = " System Test Build Status: " + tc.build_status + ". \n System Test: " + tc.name + " \n Execution Status: " if tc.run_needed and tc.type == 2: #SystemTestType.MANUAL: - status = "notrun" - extra_message += "Manual system tests can't be run in CI tools" + result += "Manual system tests can't be run in Jenkins" tc.passed = 1 elif tc.run_needed: - status = "notrun" - extra_message += "Needs to be executed" + result += "Needs to be executed" tc.passed = 1 elif tc.passed > 0 and tc.passed == tc.total: - status = "passed" - extra_message += "Passed" + result += "Passed" else: - status = "failed" - extra_message += "Failed {} / {} ".format(tc.passed, tc.total) + result += "Failed {} / {} ".format(tc.passed, tc.total) tc.passed = 0 else: @@ -976,91 +1535,70 @@ def write_testcase(self, tc, unit_name, func_name, st_is_monitored = False, unit summary = tc.history.summary exp_total = summary.expected_total exp_pass = exp_total - summary.expected_fail + if self.api.environment.get_option("VCAST_OLD_STYLE_MANAGEMENT_REPORT"): + exp_pass += summary.control_flow_total - summary.control_flow_fail + exp_total += summary.control_flow_total + summary.signals + summary.unexpected_exceptions - if summary.control_flow_fail > 0: - control_flow_fail = True - - if summary.unexpected_exceptions > 0: - exception_fail = True - - if summary.signals > 0: - signal_fail = True + result = self.__get_testcase_execution_results( + tc, + classname, + tc_name_full) exp_pass += summary.control_flow_total - summary.control_flow_fail exp_total += summary.control_flow_total + summary.signals + summary.unexpected_exceptions if tc.testcase_status == "TCR_STRICT_IMPORT_FAILED": - status = "failed" - extra_message = "Strict Test Import Failure." + result += "\nStrict Test Import Failure." + # Failure takes priority - elif tc.status != "TC_EXECUTION_NONE": - extra_message, status = self.convertExecStatus(tc.execution_status) + if tc.status != "TC_EXECUTION_NONE": + failure_message = failureReasons else: - status = "notrun" - extra_message = "Test was not executed" - - extra_message = escape(extra_message, quote=False) - extra_message = extra_message.replace("\"","") - extra_message = extra_message.replace("\n"," ") - extra_message = extra_message.replace("\r","") + failure_message = didntRunReason + msg = "" + status = "" if tc.passed == None: - status = "skipped" - extraStatus = "" + extraStatus = "\n \n" + status = "Testcase may have been skipped by VectorCAST Change Based Testing. Last execution data shown.\n\nFAIL" + msg = "{} {} / {} \n\nExecution Report:\n {}".format(status, exp_pass, exp_total, result) elif not tc.passed: - whyFail = "" - expectedResultsFailure = "" - - if exception_fail: - whyFail += "Unexpected exception failure. " - - if signal_fail: - whyFail += "Signal failure. " - - if control_flow_fail: - whyFail += "Control flow failure. " - expectedResultsFailure = "Control flow values" - - if tc.history.summary.expected_total: - whyFail += "Expected values failure. " - if len(expectedResultsFailure) > 0: - expectedResultsFailure += " and " - expectedResultsFailure += "Expected values totals" - - if whyFail == "Signal failure. ": - extraStatus = '' - - elif extra_message == "Strict Test Import Failure": - extraStatus = '' - - elif tcSkipped: - status = "skipped" - extraStatus = ''.format(extra_message, whyFail, expectedResultsFailure, exp_pass, exp_total) + if tcSkipped: + status = "Testcase may have been skipped by VectorCAST Change Based Testing. Last execution data shown.\n\nFAIL" else: - extraStatus = ''.format(extra_message, whyFail, expectedResultsFailure, exp_pass, exp_total) + status = "FAIL" + extraStatus = "\n \n" + msg = "{} {} / {} \n\nExecution Report:\n {}".format(status, exp_pass, exp_total, result) elif tcSkipped: - extraStatus = "" + extraStatus = "\n \n" + status = "Skipped by VectorCAST Change Based Testing. Last execution data shown.\n\nPASS" + msg = "{} {} / {} \n\nExecution Report:\n {}".format(status, exp_pass, exp_total, result) else: + status = "PASS" extraStatus = "" + testcaseString, testcaseStringExtraStatus = self.get_xml_string(fpath) - testcaseString =' \n" - else: - extraXmlTag = "/>\n" + if self.use_cte or unitName == "": + unitName = classname - data = testcaseString % (tc_name_full, classname, deltaTimeStr, fpath, status, extraXmlTag) + if status != "": + msg = "{} {} / {} \n\nExecution Report:\n {}".format(status, exp_pass, exp_total, result) + msg = escape(msg, quote=False) + msg = msg.replace("\"","") + msg = msg.replace("\n"," ") + msg = msg.replace("\r","") - self.fh.write(data.encode(self.encFmt, "replace")) + testcaseString = testcaseStringExtraStatus + self.fh_data += (testcaseString % (tc_name_full, unitName, deltaTimeStr, fpath, startLine, extraStatus, msg)) + else: + self.fh_data += (testcaseString % (tc_name_full, unitName, deltaTimeStr, fpath, startLine, extraStatus)) + +## GenerateXml -# -# Internal - no support for skipped test cases yet -# def was_test_case_skipped(self, tc, searchName, isSystemTest): - return False try: if isSystemTest: compoundTests, initTests, simpleTestcases = self.cbtDict[self.hashCode] @@ -1094,155 +1632,79 @@ def was_test_case_skipped(self, tc, searchName, isSystemTest): except Exception as e: parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) if self.print_exc: - print ("CBT Dictionary:" + self.cbtDict, width = 132) - pprint(self.cbtDict, width = 132) + import json + print ("CBT Dictionary:\n{}".format(json.dumps(self.cbtDict, indent=2))) -# -# Internal - write the end of the jUnit XML file and close it -# - def end_test_results_file(self): - self.fh.write(" \n".encode(self.encFmt, "replace")) - self.fh.write("\n".encode(self.encFmt, "replace")) - self.fh.close() +## GenerateXml -# -# Internal - write the start of the coverage file for and environment -# - def start_cov_file_environment(self): - self.start_cov_file() - data = "" - data += " \n" - data += " \n" - data += " \n".format(self.num_units) - data += " \n".format(self.num_functions) - data += " \n" - data += " \n" - data += " \n" - if self.coverage["statement"]: - data += " \n".format(self.coverage["statement"]) - if self.coverage["branch"]: - data += " \n".format(self.coverage["branch"]) - if self.coverage["mcdc"]: - data += " \n".format(self.coverage["mcdc"]) - if self.coverage["basispath"]: - data += " \n".format(self.coverage["basispath"]) - if self.coverage["function"]: - data += " \n".format(self.coverage["function"]) - if self.coverage["functioncall"]: - data += " \n".format(self.coverage["functioncall"]) - data += " \n".format(self.grand_total_complexiy) - data += "\n" + def __get_testcase_execution_results(self, tc, classname, tc_name): - data += " \n".format(escape(self.jenkins_name, quote=False)) - if self.coverage["statement"]: - data += " \n".format(self.coverage["statement"]) - if self.coverage["branch"]: - data += " \n".format(self.coverage["branch"]) - if self.coverage["mcdc"]: - data += " \n".format(self.coverage["mcdc"]) - if self.coverage["basispath"]: - data += " \n".format(self.coverage["basispath"]) - if self.coverage["function"]: - data += " \n".format(self.coverage["function"]) - if self.coverage["functioncall"]: - data += " \n".format(self.coverage["functioncall"]) - data += " \n".format(self.grand_total_complexity) - self.fh.write(data.encode(self.encFmt, "replace")) + if not self.testcase_failed(tc): + return "" + + if not self.generate_exec_rpt_each_testcase: + return "Execution Report disabled by using --dont-generate-individual-reports" -# -# Internal - write the end of the coverage file and close it -# - def end_cov_file_environment(self): - self.fh.write(' \n'.encode(self.encFmt, "replace")) - self.fh.write(' \n'.encode(self.encFmt, "replace")) - self.fh.write(' \n'.encode(self.encFmt, "replace")) - self.end_cov_file() + report_name_hash = '.'.join( + ["execution_results", classname, tc_name]) + # Unicode-objects must be encoded before hashing in Python 3 + if sys.version_info[0] >= 3: + report_name_hash = report_name_hash.encode(self.encFmt) -# -# Internal - write the units to the coverage file -# - def write_cov_units(self): - for unit in self.our_units: - data = "" - data += " \n".format(escape(unit["unit"].name, quote=False)) - if unit["coverage"]["statement"]: - data += " \n".format(unit["coverage"]["statement"]) - if unit["coverage"]["branch"]: - data += " \n".format(unit["coverage"]["branch"]) - if unit["coverage"]["mcdc"]: - data += " \n".format(unit["coverage"]["mcdc"]) - if unit["coverage"]["basispath"]: - data += " \n".format(unit["coverage"]["basispath"]) - if unit["coverage"]["function"]: - data += " \n".format(unit["coverage"]["function"]) - if unit["coverage"]["functioncall"]: - data += " \n".format(unit["coverage"]["functioncall"]) - data += " \n".format(unit["complexity"]) + report_name = hashlib.md5(report_name_hash).hexdigest() - for func in unit["functions"]: - if self.using_cover: - func_name = escape(func["func"].name, quote=True) - data += " \n".format(func_name) - else: - func_name = escape(func["func"].display_name, quote=True) - data += " \n".format(func_name) - if func["coverage"]["statement"]: - data += " \n".format(func["coverage"]["statement"]) - if func["coverage"]["branch"]: - data += " \n".format(func["coverage"]["branch"]) - if func["coverage"]["mcdc"]: - data += " \n".format(func["coverage"]["mcdc"]) - if func["coverage"]["basispath"]: - data += " \n".format(func["coverage"]["basispath"]) - if func["coverage"]["function"]: - data += " \n".format(func["coverage"]["function"]) - if func["coverage"]["functioncall"]: - data += " \n".format(func["coverage"]["functioncall"]) - data += " \n".format(func["complexity"]) + import time - data += " \n" - data += " \n" - self.fh_write(data.encode(self.encFmt, "replace")) + try: + try: + unit_test_models.clear_caches(self.api.connection) + except: + pass + self.api.report( + testcases=[tc], + single_testcase=True, + report_type="Demo", + formats=["TEXT"], + output_file=report_name, + sections=[ "TESTCASE_SECTIONS"], + testcase_sections=["EXECUTION_RESULTS"]) + + with open(report_name, "rb") as fd: + out = fd.read() -# -# Generate the XML Modified 'Emma' coverage data -# - def generate_cover(self): - self.units = [] - if self.using_cover: - self.units = self.api.File.all() - self.units.sort(key=lambda x: (x.coverage_type, x.unit_index)) - else: - self.units = self.api.Unit.all() + try: + # Prefer UTF-8 if possible + out = out.decode("utf-8") + except UnicodeDecodeError: + # Fallback to system/default encoding (e.g. cp936 in CN) with replace + out = out.decode(self.encFmt, errors="replace") - # unbuilt (re: Error) Ada environments causing a crash - try: - cov_type = self.api.environment.coverage_type_text - except Exception as e: - print("Couldn't access coverage information...skipping. Check console for environment build/execution errors") - return + os.remove(report_name) + except: + out = "No execution results found" + parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) - self._generate_cover(cov_type) + return out - self.start_cov_file_environment() - self.write_cov_units() - self.end_cov_file_environment() +## GenerateXml def __print_test_case_was_skipped(self, searchName, passed): if self.verbose: - print("skipping ", self.hashCode, searchName, passed) + print("skipping {} {} {}".format(self.hashCode, searchName, passed)) -def __generate_xml(xml_file, envPath, env, xmlCoverReportName, xmlTestingReportName, teePrint): +def __generate_xml(xml_file, envPath, env, xmlCoverReportName, xmlTestingReportName): if xml_file.api == None: - teePrint.teePrint ("\nCannot find project file (.vcp or .vce): " + envPath + os.sep + env) + print ("\nCannot find project file (.vcp or .vce): " + envPath + os.sep + env) elif isinstance(xml_file, CoverApi): xml_file.generate_cover() - teePrint.teePrint ("\nvectorcast-coverage plugin for Jenkins compatible file generated: " + xmlCoverReportName) + print ("\nvectorcast-coverage plugin for Jenkins compatible file generated: " + xmlCoverReportName) else: xml_file.generate_unit() - teePrint.teePrint ("\nJunit plugin for Jenkins compatible file generated: " + xmlTestingReportName) + print ("\nJunit plugin for Jenkins compatible file generated: " + xmlTestingReportName) + xml_file.generate_cover() + print ("\nVCC plugin for Jenkins compatible file generated: " + xmlTestingReportName) if __name__ == '__main__': @@ -1280,13 +1742,9 @@ def __generate_xml(xml_file, envPath, env, xmlCoverReportName, xmlTestingReportN None, args.ci) - if xml_file.api == None: - print ("\nCannot find project file (.vcp or .vce): " + envPath + os.sep + env) - - elif xml_file.using_cover: - xml_file.generate_cover() - print ("\nvectorcast-coverage plugin for Jenkins compatible file generated: " + xmlCoverReportName) - - else: - xml_file.generate_unit() - print ("\nJunit plugin for Jenkins compatible file generated: " + xmlTestingReportName) + __generate_xml( + xml_file, + envPath, + env, + xmlCoverReportName, + xmlTestingReportName) diff --git a/vcast_exec.py b/vcast_exec.py index 3cc0205..c164cb9 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -54,6 +54,7 @@ import shlex, platform from pathlib import Path +import cobertura from enum import Enum @@ -134,6 +135,13 @@ def __init__(self, args): self.html_base_dir = args.html_base_dir self.use_cte = args.use_cte self.noIndex = args.noindex + + try: + self.complexityThreshold = int(args.exit_with_failed_comp) + self.complexityCheck = True + except: + self.complexityCheck = False + self.complexityThreshold = 100000 if args.exit_with_failed_count == 'not present': self.useJunitFailCountPct = False @@ -364,7 +372,6 @@ def runCoberturaMetrics(self): if not checkVectorCASTVersion(21): print("Cannot create Cobertura metrics. Please upgrade VectorCAST") else: - import cobertura if self.cobertura_extended: print("Creating Extended Cobertura Metrics") @@ -580,9 +587,17 @@ def runExec(self): metricsGroup.add_argument('--pclp_output_html', help='Generate static analysis results from PC-lint Plus XML file to an HTML output', action="store", default = "pclp_findings.html") metricsGroup.add_argument('--exit_with_failed_count', help='Returns failed test case count as script exit. Set a value to indicate a percentage above which the job will be marked as failed', nargs='?', default='not present', const='(default 0)') + metricsGroup.add_argument('--exit_with_failed_comp', help='Returns failed if any of the functions have a Complexity (Vg) > value.', + nargs='?', default=10) metricsGroup.add_argument('--check_build_log', help='Checks build log for a list of error phrases. Returns failure if any are found.', action="store_true", default = False) + importedResultsGroup = parser.add_argument_group('Imported Results Selection', 'Options for Using Change Based Testing from Imported Results') + resultsSpecifics = importedResultsGroup.add_mutually_exclusive_group() + + resultsSpecifics.add_argument('--use_local_imported_results', help='Use artifacts from last non-failing build for CBT Imported Results', action="store_true", dest="intResults", default = False) + resultsSpecifics.add_argument('--use_external_imported_results', help='Use artifacts from repository for CBT Imported Results', dest="extResults", default = None) + reportGroup = parser.add_argument_group('Report Selection', 'VectorCAST Manage reports that can be generated') reportGroup.add_argument('--aggregate', help='Generate aggregate coverage report VectorCAST Project', action="store_true", default = False) reportGroup.add_argument('--metrics', help='Generate metrics reports for VectorCAST Project', action="store_true", default = False) @@ -678,9 +693,20 @@ def runExec(self): if args.export_rgw: vcExec.exportRgw() + + complexityFailureCount = 0 + if vcExec.complexityCheck: + for key in cobertura.vgByFunction: + if cobertura.vgByFunction[key] > vcExec.complexityThreshold: + file, func = key.split("::") + print (f"[ERROR] \n File : {file}\n Function: {func}\n Message : COMPLEXITY is greater than {vcExec.complexityThreshold}") + complexityFailureCount += 1 + + if complexityFailureCount > 0: + sys.exit(complexityFailureCount) if vcExec.useJunitFailCountPct: - print("--exit_with_failed_count=" + args.exit_with_failed_count + " specified. Fail Percent = " + str(round(vcExec.failed_pct,0)) + "% Return code: " + str(vcExec.failed_count)) + print(f"[ERROR] exit_with_failed_count={args.exit_with_failed_count} specified. Fail Percent = {round(vcExec.failed_pct,0)}% Return code: {vcExec.failed_count}") sys.exit(vcExec.failed_count) if args.check_build_log: From a1fa46933bd0e56d5323cd9e77b94ea8217b923c Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 23 Apr 2026 20:15:16 +0000 Subject: [PATCH 02/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 7d0967b..d8d94c5 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.03.09-1138-0400-gff5d994 +v2026.04.23-1615-0400-gee2b372 From cfeaec0300eb59e770aec6dbf76d8c1bfd8a534c Mon Sep 17 00:00:00 2001 From: TimSVector Date: Thu, 23 Apr 2026 17:26:12 -0400 Subject: [PATCH 03/12] removing jenkins specific calls --- generate_results.py | 31 +++++++++++++++---------------- generate_xml.py | 13 ++++++++----- getjobs.py | 13 +++++-------- vcast_exec.py | 23 +++++++++++++---------- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/generate_results.py b/generate_results.py index 0172c63..6a587ed 100644 --- a/generate_results.py +++ b/generate_results.py @@ -53,6 +53,8 @@ from vector.apps.DataAPI.vcproject_api import VCProjectApi except: pass + +from vcast_utils import dump, getVectorCASTEncoding encFmt = getVectorCASTEncoding() @@ -213,7 +215,7 @@ def delete_file(filename): if os.path.exists(filename): os.remove(filename) -def genDataApiReports(FullManageProjectName, entry, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte): +def genDataApiReports(FullManageProjectName, entry, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, useStartLine, use_cte): xml_file = "" try: @@ -244,7 +246,6 @@ def genDataApiReports(FullManageProjectName, entry, cbtDict, generate_exec_rpt_e report_only_failures, print_exc, useStartLine, - teePrint, use_cte) if xml_file.api != None: @@ -375,7 +376,7 @@ def generateIndividualReports(entry, envName): elif os.path.exists(unit_path): generateUTReport(unit_path , env, level) -def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, teePrint, use_cte): +def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, use_cte): global verbose print("Using VCProjectApi") @@ -393,7 +394,7 @@ def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase report_only_failures, no_full_report, print_exc, - useStartLine, teePrint, use_cte) + useStartLine, use_cte) if xml_file.api != None: xml_file.generate_testresults() @@ -409,9 +410,7 @@ def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase print("\n\n") except Exception as e: - parse_traceback.parse(traceback.format_exc(), print_exc) - #traceback.print_exc() - + print(traceback.format_exc(), print_exc) try: return xml_file.passed_count, xml_file.failed_count @@ -419,7 +418,7 @@ def useManageAPI(FullManageProjectName, cbtDict, generate_exec_rpt_each_testcase return 0, 0 -def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, teePrint, use_cte): +def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, generate_exec_rpt_each_testcase, use_archive_extract, report_only_failures, no_full_report, useStartLine, use_cte): failed_count = 0 passed_count = 0 @@ -432,7 +431,7 @@ def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, genera continue if envName == None: - pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte) + pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, use_cte) passed_count += pc failed_count += fc @@ -445,7 +444,7 @@ def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, genera env_level = manageEnvs[currentEnv]["compiler"] + "/" + manageEnvs[currentEnv]["testsuite"] if level == None or env_level.upper() == level.upper(): - pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, teePrint, use_cte) + pc, fc = genDataApiReports(FullManageProjectName, manageEnvs[currentEnv], cbtDict, generate_exec_rpt_each_testcase,use_archive_extract, report_only_failures, useStartLine, use_cte) passed_count += pc failed_count += fc @@ -456,16 +455,16 @@ def useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, genera return passed_count, failed_count -def cleanupDirectory(path, teePrint): +def cleanupDirectory(path): # if the path exists, try to delete all file in it if os.path.isdir(path): shutil.rmtree(path) os.mkdir(path) -def cleanupOldBuilds(teePrint): +def cleanupOldBuilds(): for path in ["xml_data","management","execution"]: - cleanupDirectory(path, teePrint) + cleanupDirectory(path) # build the Test Case Management Report for Manage Project # envName and level only supplied when doing reports for a sub-project @@ -507,7 +506,7 @@ def buildReports(FullManageProjectName = None, print("Version Check: " + str(time.time())) - cleanupOldBuilds(teePrint) + cleanupOldBuilds() for file in glob.glob("*.csv"): try: @@ -539,7 +538,7 @@ def buildReports(FullManageProjectName = None, if use_manage_api: passed_count, failed_count = useManageAPI(FullManageProjectName, cbtDict, generate_individual_reports, use_archive_extract, report_only_failures, no_full_report, - useStartLine, teePrint, use_cte) + useStartLine, use_cte) else: @@ -550,7 +549,7 @@ def buildReports(FullManageProjectName = None, passed_count, failed_count = useNewAPI(FullManageProjectName, manageEnvs, level, envName, cbtDict, generate_individual_reports, use_archive_extract, report_only_failures, no_full_report, - useStartLine, teePrint, use_cte) + useStartLine, use_cte) if timing: print("XML and Individual reports: " + str(time.time())) diff --git a/generate_xml.py b/generate_xml.py index e92a3ee..a2e3996 100644 --- a/generate_xml.py +++ b/generate_xml.py @@ -83,6 +83,7 @@ def __init__(self, FullManageProjectName, verbose, use_cte): self.verbose = verbose self.has_sfp_enabled = False self.print_exc = False + self.using_cover = False self.use_cte = use_cte @@ -684,7 +685,7 @@ def generate_cover(self): try: cov_type = self.api.environment.coverage_type_text except Exception as e: - parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) + print(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) return self._generate_cover(cov_type) @@ -769,6 +770,7 @@ def __init__(self, FullManageProjectName, verbose = False, self.failed_count = 0 self.passed_count = 0 self.print_exc = print_exc + self.using_cover = True self.units = [] @@ -953,7 +955,7 @@ def generate_local_results(self, results, key): self.fixupReport(report_name) except: print("Error creating report " + report_name + ". Contact Vector Support") - parse_traceback.parse(traceback.format_exc(), self.verbose, self.compiler, self.testsuite, self.env, self.build_dir) + print(traceback.format_exc(), self.verbose, self.compiler, self.testsuite, self.env, self.build_dir) def runFullReport(self,comp,ts,env_name,report_name): try: @@ -1168,10 +1170,11 @@ def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, c unit_path = os.path.join(build_dir,env + '.vce') if os.path.exists(cov_path): self.generate_system_test_status_report() - + self.using_cover = True self.api = CoverApi(cov_path) elif os.path.exists(unit_path): self.api = UnitTestApi(unit_path) + self.using_cover = False else: self.api = None if verbose: @@ -1630,7 +1633,7 @@ def was_test_case_skipped(self, tc, searchName, isSystemTest): self.__print_test_case_was_skipped(tc.name, tc.passed) return [True, None, None] except Exception as e: - parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) + print(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) if self.print_exc: import json print ("CBT Dictionary:\n{}".format(json.dumps(self.cbtDict, indent=2))) @@ -1682,7 +1685,7 @@ def __get_testcase_execution_results(self, tc, classname, tc_name): os.remove(report_name) except: out = "No execution results found" - parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) + print(traceback.format_exc(), self.print_exc, self.compiler, self.testsuite, self.env, self.build_dir) return out diff --git a/getjobs.py b/getjobs.py index 2d0f470..012e42f 100644 --- a/getjobs.py +++ b/getjobs.py @@ -3,7 +3,6 @@ import os import re import sys -import tee_print import glob try: from vector.apps.DataAPI.vcproject_models import EnvironmentType @@ -12,11 +11,11 @@ manageCMD=os.environ['VECTORCAST_DIR'] + "/manage" -def printOutput(somethingPrinted, ManageProjectName, output, teePrint): +def printOutput(somethingPrinted, ManageProjectName, output): if not somethingPrinted: - teePrint.teePrint ("No environments found in " + ManageProjectName + ". Please check configuration") + print ("No environments found in " + ManageProjectName + ". Please check configuration") else: - teePrint.teePrint(output) + print(output) def getBuildDirectory(compiler , testsuite , env_name, buildDirInfo): for line in buildDirInfo: @@ -108,8 +107,7 @@ def printEnvInfoDataAPI(api, printData = True, printEnvType = False): output += "%s %s %s\n" % (env.compiler.name , env.testsuite.name , env.name) if printData: - with tee_print.TeePrint() as teePrint: - printOutput(somethingPrinted, api.vcm_file, output, teePrint) + printOutput(somethingPrinted, api.vcm_file, output) return output @@ -230,8 +228,7 @@ def printEnvInfoNoDataAPI(ManageProjectName, printData = True, printEnvType = Fa somethingPrinted = True; if printData: - with tee_print.TeePrint() as teePrint: - printOutput(somethingPrinted, ManageProjectName, output, teePrint) + printOutput(somethingPrinted, ManageProjectName, output) return output diff --git a/vcast_exec.py b/vcast_exec.py index c164cb9..7508df1 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -187,11 +187,13 @@ def __init__(self, args): self.useCI = "" self.ci = "" + self.importedResults = args.importedResults + if args.incremental: self.useCBT = "--incremental" else: self.useCBT = "" - + self.useLevelEnv = False self.environment = None self.level = None @@ -506,7 +508,10 @@ def runExec(self): self.manageWait.exec_manage_command ("--status") self.manageWait.exec_manage_command ("--force --release-locks") self.manageWait.exec_manage_command ("--config VCAST_CUSTOM_REPORT_FORMAT=HTML") - + if self.importedResults: + self.manageWait.exec_manage_command (f"--force --import-result={self.importedResults}") + self.manageWait.exec_manage_command ("--status") + if self.useLevelEnv: output = "--output " + self.mpName + self.reportsName + "_rebuild.html" else: @@ -568,6 +573,7 @@ def runExec(self): actionGroup = parser.add_argument_group('Script Actions', 'Options for the main tasks') actionGroup.add_argument('--build-execute', help='Builds and exeuctes the VectorCAST Project', action="store_true", default = False) actionGroup.add_argument("--setup", default="", help="Path to setup_env.bat/.sh (optional)") + actionGroup.add_argument('--use_imported_result', help='Use existing VCR file from repository for CBT via Imported Results', dest="importedResults", default = None) parser_specify = actionGroup.add_mutually_exclusive_group() parser_specify.add_argument('--build', help='Only builds the VectorCAST Project', action="store_true", default = False) @@ -590,14 +596,7 @@ def runExec(self): metricsGroup.add_argument('--exit_with_failed_comp', help='Returns failed if any of the functions have a Complexity (Vg) > value.', nargs='?', default=10) metricsGroup.add_argument('--check_build_log', help='Checks build log for a list of error phrases. Returns failure if any are found.', - action="store_true", default = False) - - importedResultsGroup = parser.add_argument_group('Imported Results Selection', 'Options for Using Change Based Testing from Imported Results') - resultsSpecifics = importedResultsGroup.add_mutually_exclusive_group() - - resultsSpecifics.add_argument('--use_local_imported_results', help='Use artifacts from last non-failing build for CBT Imported Results', action="store_true", dest="intResults", default = False) - resultsSpecifics.add_argument('--use_external_imported_results', help='Use artifacts from repository for CBT Imported Results', dest="extResults", default = None) - + action="store_true", default = False) reportGroup = parser.add_argument_group('Report Selection', 'VectorCAST Manage reports that can be generated') reportGroup.add_argument('--aggregate', help='Generate aggregate coverage report VectorCAST Project', action="store_true", default = False) reportGroup.add_argument('--metrics', help='Generate metrics reports for VectorCAST Project', action="store_true", default = False) @@ -625,6 +624,10 @@ def runExec(self): args = parser.parse_args() + if args.importedResults and not args.incremental: + print("[INFO] Calling conflict of --use_import_result and not --incremental") + print("[INFO] Calling it this way ignores --use_imported_result") + if args.verbose: import sys, shlex print("argv:", shlex.join(sys.argv)) # py3.8+ From 28795a98ae1e5874bca9e38a03069af08389938d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 23 Apr 2026 21:27:04 +0000 Subject: [PATCH 04/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index d8d94c5..c32d24d 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.04.23-1615-0400-gee2b372 +v2026.04.23-1727-0400-g5914da3 From 20b7ad46dfbc2350d0f3fae1812028fd89bc79d0 Mon Sep 17 00:00:00 2001 From: TimSVector Date: Fri, 24 Apr 2026 12:30:03 -0400 Subject: [PATCH 05/12] tying complexity check to cobertura --- vcast_exec.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vcast_exec.py b/vcast_exec.py index 7508df1..8c099d2 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -627,6 +627,9 @@ def runExec(self): if args.importedResults and not args.incremental: print("[INFO] Calling conflict of --use_import_result and not --incremental") print("[INFO] Calling it this way ignores --use_imported_result") + + if args.complexityCheck and not args.cobertura and not args.cobertura_extended): + args.cobertura = True if args.verbose: import sys, shlex From b9e1c8a4529ada20ae2e007c8582eca6b28f16db Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 24 Apr 2026 16:30:19 +0000 Subject: [PATCH 06/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index c32d24d..02cef4f 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.04.23-1727-0400-g5914da3 +v2026.04.24-1230-0400-g20b7ad4 From ffef80f09139a76dc5b42ca7f053bf8bfd275e45 Mon Sep 17 00:00:00 2001 From: TimSVector Date: Fri, 24 Apr 2026 14:05:06 -0400 Subject: [PATCH 07/12] typo --- vcast_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcast_exec.py b/vcast_exec.py index 8c099d2..284d9ff 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -628,7 +628,7 @@ def runExec(self): print("[INFO] Calling conflict of --use_import_result and not --incremental") print("[INFO] Calling it this way ignores --use_imported_result") - if args.complexityCheck and not args.cobertura and not args.cobertura_extended): + if args.complexityCheck and not args.cobertura and not args.cobertura_extended: args.cobertura = True if args.verbose: From 51ef54ccceabbb8b3f02943385c5d5f1c8b551be Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 24 Apr 2026 18:05:34 +0000 Subject: [PATCH 08/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 02cef4f..9b67fe7 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.04.24-1230-0400-g20b7ad4 +v2026.04.24-1405-0400-gfc026a2 From 5f62ce6976005d3fda002d4b66ab2315308fd646 Mon Sep 17 00:00:00 2001 From: TimSVector Date: Fri, 24 Apr 2026 14:15:59 -0400 Subject: [PATCH 09/12] fixing stuff --- vcast_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcast_exec.py b/vcast_exec.py index 284d9ff..e534a3a 100644 --- a/vcast_exec.py +++ b/vcast_exec.py @@ -628,7 +628,7 @@ def runExec(self): print("[INFO] Calling conflict of --use_import_result and not --incremental") print("[INFO] Calling it this way ignores --use_imported_result") - if args.complexityCheck and not args.cobertura and not args.cobertura_extended: + if args.exit_with_failed_comp and not args.cobertura and not args.cobertura_extended: args.cobertura = True if args.verbose: From 1305f89af1f014a6cff7662f107b0631ed05bc41 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 24 Apr 2026 18:16:16 +0000 Subject: [PATCH 10/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 9b67fe7..b873af0 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.04.24-1405-0400-gfc026a2 +v2026.04.24-1416-0400-g6f2d1eb From 69116b299bac951d6b18520628ed714baea5843f Mon Sep 17 00:00:00 2001 From: TimSVector Date: Fri, 24 Apr 2026 14:38:22 -0400 Subject: [PATCH 11/12] Updating docs --- README.md | 75 ++++++++++++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 1df8185..a65a09a 100644 --- a/README.md +++ b/README.md @@ -32,30 +32,30 @@ The python scrip `vcast_exec.py` is the main driver for build/execute VectorCAST The api for vcast_exec.py follows: ``` - usage: vcast_exec.py [-h] [--build-execute] [--build | --incremental] - [--output_dir OUTPUT_DIR] [--source_root SOURCE_ROOT] - [--html_base_dir HTML_BASE_DIR] [--cobertura] - [--cobertura_extended] [--lcov] [--junit] [--export_rgw] - [--sonarqube] [--pclp_input PCLP_INPUT] - [--pclp_output_html PCLP_OUTPUT_HTML] - [--exit_with_failed_count [EXIT_WITH_FAILED_COUNT]] - [--check_build_log] [--aggregate] [--metrics] - [--fullstatus] [--utfull] [--tcmr] [--noindex] - [--jobs JOBS] [--ci] [-l LEVEL] [-e ENVIRONMENT] - [--gitlab | --azure] [--print_exc] [--timing] [-v] - [--version] - [ManageProject] + usage: vcast_exec.py [VectorCAST Project] + [-h/--help] + [--build-execute] [--setup SETUP] [--use_imported_result IMPORTEDRESULTS] [--build | --incremental] + [--output_dir OUTPUT_DIR] [--source_root SOURCE_ROOT] [--html_base_dir HTML_BASE_DIR] + [--cobertura] [--cobertura_extended] [--lcov] [--junit] [--export_rgw] [--sonarqube] + [--pclp_input PCLP_INPUT] [--pclp_output_html PCLP_OUTPUT_HTML] + [--exit_with_failed_count] [EXIT_WITH_FAILED_COUNT]] [--exit_with_failed_comp [EXIT_WITH_FAILED_COMP] [--check_build_log] + [--aggregate] [--metrics] [--fullstatus] [--utfull] [--tcmr] [--noindex] + [--jobs JOBS] [--ci] [-l LEVEL] [-e ENVIRONMENT] [--gitlab | --azure] + [--print_exc] [--timing] [-v/--verbose] [--version] positional arguments: ManageProject VectorCAST Project Name - optional arguments: + options: -h, --help show this help message and exit Script Actions: Options for the main tasks --build-execute Builds and exeuctes the VectorCAST Project + --setup SETUP Path to setup_env.bat/.sh (optional) + --use_imported_result IMPORTEDRESULTS + Use existing VCR file from repository for CBT via Imported Results --build Only builds the VectorCAST Project --incremental Use Change Based Testing (Cannot be used with --build) @@ -63,34 +63,26 @@ The api for vcast_exec.py follows: Options generating metrics --output_dir OUTPUT_DIR - Set the base directory of the xml_data directory. - Default is the workspace directory + Set the base directory of the xml_data directory. Default is the workspace directory --source_root SOURCE_ROOT - Set the absolute path for the source file in coverage - reporting + Set the absolute path for the source file in coverage reporting --html_base_dir HTML_BASE_DIR - Set the base directory of the html_reports directory. - The default is the workspace directory + Set the base directory of the html_reports directory. The default is the workspace directory --cobertura Generate coverage results in Cobertura xml format - --cobertura_extended Generate coverage results in extended Cobertura xml - format + --cobertura_extended Generate coverage results in extended Cobertura xml format --lcov Generate coverage results in an LCOV format --junit Generate test results in Junit xml format --export_rgw Export RGW data - --sonarqube Generate test results in SonarQube Generic test - execution report format (CppUnit) + --sonarqube Generate test results in SonarQube Generic test execution report format (CppUnit) --pclp_input PCLP_INPUT - Generate static analysis results from PC-lint Plus XML - file to generic static analysis format (codequality) + Generate static analysis results from PC-lint Plus XML file to generic static analysis format (codequality) --pclp_output_html PCLP_OUTPUT_HTML - Generate static analysis results from PC-lint Plus XML - file to an HTML output + Generate static analysis results from PC-lint Plus XML file to an HTML output --exit_with_failed_count [EXIT_WITH_FAILED_COUNT] - Returns failed test case count as script exit. Set a - value to indicate a percentage above which the job - will be marked as failed - --check_build_log Checks build log for a list of error phrases. Returns - failure if any are found. + Returns failed test case count as script exit. Set a value to indicate a percentage above which the job will be marked as failed + --exit_with_failed_comp [EXIT_WITH_FAILED_COMP] + Returns failed if any of the functions have a Complexity (Vg) > value. + --check_build_log Checks build log for a list of error phrases. Returns failure if any are found. Report Selection: VectorCAST Manage reports that can be generated @@ -98,12 +90,9 @@ The api for vcast_exec.py follows: --aggregate Generate aggregate coverage report VectorCAST Project --metrics Generate metrics reports for VectorCAST Project --fullstatus Generate full status reports for VectorCAST Project - --utfull Generate Full Reports for each VectorCAST environment - in project - --tcmr Generate Test Cases Management Reports for each - VectorCAST environment in project - --noindex Stops index.html report that ties all the other HTML - reports together from being created + --utfull Generate Full Reports for each VectorCAST environment in project + --tcmr Generate Test Cases Management Reports for each VectorCAST environment in project + --noindex Stops index.html report that ties all the other HTML reports together from being created Build/Execution Options: Options that effect build/execute operation @@ -111,8 +100,7 @@ The api for vcast_exec.py follows: --jobs JOBS Number of concurrent jobs (default = 1) --ci Use Continuous Integration Licenses -l LEVEL, --level LEVEL - Environment Name if only doing single environment. - Should be in the form of compiler/testsuite + Environment Name if only doing single environment. Should be in the form of compiler/testsuite -e ENVIRONMENT, --environment ENVIRONMENT Environment Name if only doing single environment. --gitlab Build using GitLab CI (default) @@ -125,10 +113,13 @@ The api for vcast_exec.py follows: --timing Prints timing information for metrics generation -v, --verbose Enable verbose output --version Displays the version information - ``` # Change log +04/2025 +* Added a quality gate for function level complexity greater that X +* Added in support for Imported Results + 01/2025 * Added --check_build_log to examine the VectorCAST build log to see if any errors occurred during the build-execute * return codes: From ecab3bdd74e6de89b558f84ce7f38523c2542698 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 24 Apr 2026 18:38:39 +0000 Subject: [PATCH 12/12] Update VERSION.txt --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index b873af0..3f16d8f 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v2026.04.24-1416-0400-g6f2d1eb +v2026.04.24-1438-0400-g69116b2