diff --git a/chb/app/CHVersion.py b/chb/app/CHVersion.py index 11b5761f..a1638148 100644 --- a/chb/app/CHVersion.py +++ b/chb/app/CHVersion.py @@ -1,3 +1,3 @@ -chbversion: str = "0.3.20261001" +chbversion: str = "0.3.20261007" minimum_required_chb_version = "0.6.0_20260802" diff --git a/chb/arm/ARMCallOpcode.py b/chb/arm/ARMCallOpcode.py index 777bbdec..23893e03 100644 --- a/chb/arm/ARMCallOpcode.py +++ b/chb/arm/ARMCallOpcode.py @@ -49,7 +49,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -297,7 +297,8 @@ def ast_call_prov( if finfo.has_call_target(iaddr): calltarget = finfo.call_target(iaddr) if calltarget.is_unknown: - chklogger.logger.error( + chklogger.logger.error_id( + CHKLogID.UNSP_INDCALL_0001, "BL: Indirect call not yet handled at address %s", iaddr) if finfo.has_call_target_info(iaddr): @@ -509,8 +510,9 @@ def ast_call_prov( hl_arg = astree.mk_address_of( astree.mk_vinfo_lval(vinfo)) else: - chklogger.logger.error( - ("Unknown global address %s as call " + chklogger.logger.error_id( + CHKLogID.USER_GLBDECL_0001, + ("USER: Unknown global address %s as call " + "argument at address %s"), hexgaddr, iaddr) hl_arg = astree.mk_temp_lval_expression() diff --git a/chb/arm/ARMOpcode.py b/chb/arm/ARMOpcode.py index 8c6662dd..c5600858 100644 --- a/chb/arm/ARMOpcode.py +++ b/chb/arm/ARMOpcode.py @@ -52,7 +52,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -440,8 +440,9 @@ def ast_prov( List[AST.ASTInstruction], List[AST.ASTInstruction]]: """Return default; should be overridden by instruction opcodes.""" - chklogger.logger.error( - "no lifting support available for instruction %s at address %s", + chklogger.logger.error_id( + CHKLogID.UNSP_ASMINSTR_0001, + "UNSP: no lifting support available for instruction %s at address %s", self.mnemonic, iaddr) instrs = self.ast(astree, iaddr, bytestring, xdata) return (instrs, instrs) @@ -497,8 +498,9 @@ def ast_cc_condition_prov( return (hl_astcond, ll_astcond) else: - chklogger.logger.error( - "No condition found at address %s", iaddr) + chklogger.logger.error_id( + CHKLogID.RSLT_PRNOCC_0001, + "No instruction condition predicate found at address %s", iaddr) hl_astcond = astree.mk_temp_lval_expression() return (hl_astcond, ll_astcond) diff --git a/chb/arm/opcodes/ARMAdd.py b/chb/arm/opcodes/ARMAdd.py index b92a8d57..6822cca8 100644 --- a/chb/arm/opcodes/ARMAdd.py +++ b/chb/arm/opcodes/ARMAdd.py @@ -40,7 +40,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -260,8 +260,9 @@ def ast_prov( annotations: List[str] = [iaddr, "ADD"] if xdata.is_aggregate_jumptable: - chklogger.logger.warning( - "ADD: aggregate jumptable at address %s not yet handled", + chklogger.logger.error_id( + CHKLogID.UNSP_JMPTBL_0001, + "UNSP: ADD: aggregate jumptable at address %s not yet handled", iaddr) return ([], []) diff --git a/chb/arm/opcodes/ARMBranch.py b/chb/arm/opcodes/ARMBranch.py index ce275902..bc287770 100644 --- a/chb/arm/opcodes/ARMBranch.py +++ b/chb/arm/opcodes/ARMBranch.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -50,7 +50,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -374,7 +374,8 @@ def default(condition: XXpr) -> AST.ASTExpr: else: condition = xd.txpr else: - chklogger.logger.error( + chklogger.logger.error_id( + CHKLogID.RSLT_BRNOCC_0001, "Bcc: conditional branch without branch conditions " + "at address %s", iaddr) hl_astcond = astree.mk_temp_lval_expression() @@ -394,7 +395,7 @@ def default(condition: XXpr) -> AST.ASTExpr: elif len(ftconds) == 0: chklogger.logger.error( - "No branch condition found at address %s", iaddr) + "BR: No branch condition found at address %s", iaddr) hl_astcond = astree.mk_temp_lval_expression() return (hl_astcond, ll_astcond) diff --git a/chb/arm/opcodes/ARMLoadRegister.py b/chb/arm/opcodes/ARMLoadRegister.py index 965f8d3d..9d25b962 100644 --- a/chb/arm/opcodes/ARMLoadRegister.py +++ b/chb/arm/opcodes/ARMLoadRegister.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,7 +42,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -255,8 +255,9 @@ def has_cast() -> bool: lhs = xd.vrt if str(lhs) == "PC": - chklogger.logger.error( - "LDR: Indirect call via Load to PC not yet handled at %s", + chklogger.logger.error_id( + CHKLogID.UNSP_INDCALL_0002, + "UNSP: LDR: Indirect call via Load to PC not yet handled at %s", iaddr) return ([], (ll_pre + [ll_assign] + ll_post)) @@ -279,8 +280,9 @@ def has_cast() -> bool: hl_rhs = XU.xmemory_dereference_lval_expr( xaddr, xdata, iaddr, astree) - chklogger.logger.warning( - "LDR: Unable to use a C expression for rhs. Fall back to " + chklogger.logger.warning_id( + CHKLogID.RSLT_ERRCXPR_0002, + "RSLT: LDR: Unable to use a C expression for rhs. Fall back to " + "native byte-based address: %s to form rhs %s at address %s", str(xaddr), str(hl_rhs), iaddr) diff --git a/chb/arm/opcodes/ARMLoadRegisterByte.py b/chb/arm/opcodes/ARMLoadRegisterByte.py index ee10b223..100bf75d 100644 --- a/chb/arm/opcodes/ARMLoadRegisterByte.py +++ b/chb/arm/opcodes/ARMLoadRegisterByte.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,7 +42,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -142,7 +142,12 @@ def annotation(self) -> str: else: crhs = "None" cx = " (C: " + crhs + ")" - addr = str(self.xxaddr if self.is_xxaddr_ok else self.xaddr) + if self.is_xxaddr_ok: + addr = str(self.xxaddr) + elif self.is_xaddr_ok: + addr = str(self.xaddr) + else: + addr = "None" caddr = str(self.cxaddr if self.is_cxaddr_ok else "None") caddr = " (addr: " + addr + "; C: " + caddr + ")" if self.is_ok or self.is_xrmem_ok: @@ -278,13 +283,15 @@ def has_cast() -> bool: hl_rhs = XU.xmemory_dereference_lval_expr( xaddr, xdata, iaddr, astree) - chklogger.logger.warning( - "LDRB: Unable to use a C expression for rhs. Fall back to " + chklogger.logger.warning_id( + CHKLogID.RSLT_ERRCXPR_0003, + "RSLT: LDRB: Unable to use a C expression for rhs. Fall back to " + "native byte-based address: %s to form rhs %s at address %s", str(xaddr), str(hl_rhs), iaddr) else: - chklogger.logger.error( + chklogger.logger.error_id( + CHKLogID.RSLT_ERRVAL_0002, "LDRB: both memory value and address values are error values " + "at address %s: ", iaddr) return ([], (ll_pre + [ll_assign] + ll_post)) diff --git a/chb/arm/opcodes/ARMLoadRegisterHalfword.py b/chb/arm/opcodes/ARMLoadRegisterHalfword.py index a09991fc..ec80fb7a 100644 --- a/chb/arm/opcodes/ARMLoadRegisterHalfword.py +++ b/chb/arm/opcodes/ARMLoadRegisterHalfword.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,7 +42,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: from chb.arm.ARMDictionary import ARMDictionary @@ -271,8 +271,9 @@ def has_cast() -> bool: hl_rhs = XU.xmemory_dereference_lval_expr( xaddr, xdata, iaddr, astree) - chklogger.logger.warning( - "LDRH: Unable to use a C expression for rhs. Fall back to " + chklogger.logger.warning_id( + CHKLogID.RSLT_ERRCXPR_0001, + "RSLT: LDRH: Unable to use a C expression for rhs. Fall back to " + "native byte-based address: %s to form rhs %s at address %s", str(xaddr), str(hl_rhs), iaddr) diff --git a/chb/arm/opcodes/ARMPop.py b/chb/arm/opcodes/ARMPop.py index b6613ce8..736967d3 100644 --- a/chb/arm/opcodes/ARMPop.py +++ b/chb/arm/opcodes/ARMPop.py @@ -40,7 +40,7 @@ import chb.invariants.XXprUtil as XU import chb.util.fileutil as UF -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID from chb.util.IndexedTable import IndexedTableValue if TYPE_CHECKING: @@ -236,7 +236,8 @@ def ast_prov( xd = ARMPopXData(xdata) if not xd.is_ok: - chklogger.logger.error( + chklogger.logger.error_id( + CHKLogID.RSLT_ERRVAL_0001, "Encountered error value at address %s", iaddr) return ([], []) diff --git a/chb/arm/opcodes/ARMStoreMultipleIncrementAfter.py b/chb/arm/opcodes/ARMStoreMultipleIncrementAfter.py index 7d4fa6a1..e0fc8bf4 100644 --- a/chb/arm/opcodes/ARMStoreMultipleIncrementAfter.py +++ b/chb/arm/opcodes/ARMStoreMultipleIncrementAfter.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -511,8 +511,9 @@ def ast_prov( elif xd.are_memlhss_ok: memlhss = xd.memlhss else: - chklogger.logger.error( - "STM: Error value encountered in LHSs at address %s", iaddr) + chklogger.logger.error_id( + CHKLogID.RSLT_ERRVAL_0005, + "RSLT: STM: Error value encountered in LHSs at address %s", iaddr) return ([], []) if xd.are_crhss_ok: @@ -520,8 +521,9 @@ def ast_prov( elif xd.are_rrhss_ok: regrhss = xd.rhss else: - chklogger.logger.error( - "STM: Error value encountered in RHSs at address %s", iaddr) + chklogger.logger.error_id( + CHKLogID.RSLT_ERRVAL_0004, + "RSLT: STM: Error value encountered in RHSs at address %s", iaddr) return ([], []) annotations: List[str] = [iaddr, "STM"] diff --git a/chb/arm/opcodes/ARMStoreMultipleIncrementBefore.py b/chb/arm/opcodes/ARMStoreMultipleIncrementBefore.py index 3e0f4777..d9cf4b31 100644 --- a/chb/arm/opcodes/ARMStoreMultipleIncrementBefore.py +++ b/chb/arm/opcodes/ARMStoreMultipleIncrementBefore.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2025 Aarno Labs LLC +# Copyright (c) 2021-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ import chb.util.fileutil as UF from chb.util.IndexedTable import IndexedTableValue -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -135,8 +135,9 @@ def ast_prov( xd = ARMStoreMultipleIncrementBeforeXData(xdata) if not xd.is_ok: - chklogger.logger.error( - "STMIB: Error value encountered at address %s", iaddr) + chklogger.logger.error_id( + CHKLogID.RSLT_ERRVAL_0003, + "RSLT: STMIB: Error value encountered at address %s", iaddr) return ([], []) annotations: List[str] = [iaddr, "STMIB"] diff --git a/chb/astinterface/ASTIProvenance.py b/chb/astinterface/ASTIProvenance.py index 29a20a56..0ae892d8 100644 --- a/chb/astinterface/ASTIProvenance.py +++ b/chb/astinterface/ASTIProvenance.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2022-2025 Aarno Labs LLC +# Copyright (c) 2022-2026 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -32,7 +32,7 @@ from chb.ast.ASTProvenance import ASTProvenance import chb.util.fileutil as UF -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: from chb.invariants.VarInvariantFact import ( @@ -478,11 +478,18 @@ def resolve_reaching_defs(self) -> None: else: # temporarily silence warnings for payload addresses if not addr.startswith("F"): - chklogger.logger.warning( - "Reaching definition address %s for variable %s " - + " not found", - str(addr), str(v)) - + if "clobber" in str(addr): + chklogger.logger.warning_id( + CHKLogID.RDEF_CLOBBER_0001, + "RDEF: Reaching definition of address %s for clobbered " + + "value %s not found", + str(addr), str(v)) + else: + chklogger.logger.warning_id( + CHKLogID.RDEF_UNRESOLVED_0001, + "RDEF: Reaching definition address %s for variable %s " + + " not found", + str(addr), str(v)) def resolve_flag_reaching_defs(self) -> None: for (xid, frds) in self.flag_expr_rdefs.items(): diff --git a/chb/cmdline/astcmds.py b/chb/cmdline/astcmds.py index df4a1292..e77f51fa 100644 --- a/chb/cmdline/astcmds.py +++ b/chb/cmdline/astcmds.py @@ -66,7 +66,7 @@ from chb.util.DotGraph import DotGraph import chb.util.fileutil as UF import chb.util.graphutil as UG -from chb.util.loggingutil import chklogger, LogLevel +from chb.util.loggingutil import chklogger, LogLevel, CHKLogID, function_context if TYPE_CHECKING: @@ -158,6 +158,9 @@ def buildast(args: argparse.Namespace) -> NoReturn: loglevel: str = args.loglevel logfilename: Optional[str] = args.logfilename logfilemode: str = args.logfilemode + jlogfilename: Optional[str] = args.jlogfilename + jlogfilemode: str = args.jlogfilemode + jloglevel: str = args.jloglevel try: (path, xfile) = UC.get_path_filename(xname) @@ -171,6 +174,9 @@ def buildast(args: argparse.Namespace) -> NoReturn: path, logfilename=logfilename, mode=logfilemode, + jlogfilename=jlogfilename, + jmode=jlogfilemode, + jlevel=jloglevel, msg="results ast invoked") xinfo = XI.XInfo() @@ -289,144 +295,156 @@ def buildast(args: argparse.Namespace) -> NoReturn: for faddr in functions: if app.has_function(faddr): - f = app.function(faddr) - - if f is None: - UC.print_error("Unable to find function " + faddr) - continue + with function_context(faddr): - if app.has_function_name(faddr): - fname = app.function_name(faddr) - else: - fname = "sub_" + faddr[2:] + f = app.function(faddr) - fsummary = f.finfo.appsummary - - localsymboltable = ASTLocalSymbolTable(globalsymboltable) - returnsequences = ASTReturnSequences(codefragments) - astree = AbstractSyntaxTree( - faddr, - fname, - localsymboltable, - returnsequences=returnsequences, - registersizes=support.register_sizes, - flagnames=support.flagnames) - - srcprototype: Optional["BCVarInfo"] = None - astprototype: Optional[ASTVarInfo] = None - appsignature: Optional["AppFunctionSignature"] = None - if app.bcfiles.has_vardecl(fname): - srcprototype = app.bcfiles.vardecl(fname) - if fsummary is not None: - appsignature = fsummary.function_signature - elif fname == "main": - astprototype = astree.mk_vinfo_main_function(faddr) - - # Offsets are negated to make them consistent with their internal - # numerical value. - # - # For convenience local stack variable introductions are specified - # with positive offsets. Internally these are negative values (i.e., - # the difference between stackpointer value and stackpointer value - # at function entry, with a stack growing down). - fstackvarintros = { - -off: name - for (off, name) in stackvarintros.get(faddr, {}).items()} - functionannotation = userhints.function_annotation(faddr) - - astinterface = ASTInterface( - astree, - typconverter, - xinfo.architecture, - srcprototype=srcprototype, - astprototype=astprototype, - appsignature=appsignature, - varintros=varintros, - functionannotation=functionannotation, - stackvarintros=fstackvarintros, - patchevents=patchevents, - verbose=verbose) - - # Introduce ssa variables for all reaching definitions referenced in - # xdata records for all instructions in the function. Locations that - # have a common user are merged. Types are provided by lhs_types. - untyped = astinterface.introduce_ssa_variables( - f.rdef_location_partition(), f.register_lhs_types, f.lhs_names) - - # Introduce stack variables for all stack buffers with types - astinterface.introduce_stack_variables( - f.stackframe, f.stack_variable_types) - - regsuntyped: List[str] = [] - for (reg, varlocs) in untyped.items(): - for (var, locs) in varlocs.items(): - if len(locs) == 1 and locs[0] == "init": - continue - if "_spill" in var: - continue - if reg == "SP": - continue - # print(" Untyped: " + reg + ": " + var + " [" + ",".join(locs) + "]") - if not reg in regsuntyped: - regsuntyped.append(reg) + if f is None: + UC.print_error("Unable to find function " + faddr) + continue - astfunction = ASTInterfaceFunction( - faddr, fname, f, astinterface, patchevents=patchevents) + if app.has_function_name(faddr): + fname = app.function_name(faddr) + else: + fname = "sub_" + faddr[2:] - try: - asts = astfunction.mk_asts(support) - except UF.CHBError as e: - UC.print_error( - "Unable to create lifting for " - + faddr - + ":\n" - + ("-" * 80) - + "\n" - + str(e)) - functions_failed += 1 - # continue - raise + fsummary = f.finfo.appsummary - if len(asts) >= 2: - astapi.add_function_ast(astree, asts, verbose) + localsymboltable = ASTLocalSymbolTable(globalsymboltable) + returnsequences = ASTReturnSequences(codefragments) + astree = AbstractSyntaxTree( + faddr, + fname, + localsymboltable, + returnsequences=returnsequences, + registersizes=support.register_sizes, + flagnames=support.flagnames) + + srcprototype: Optional["BCVarInfo"] = None + astprototype: Optional[ASTVarInfo] = None + appsignature: Optional["AppFunctionSignature"] = None + if app.bcfiles.has_vardecl(fname): + srcprototype = app.bcfiles.vardecl(fname) + if fsummary is not None: + appsignature = fsummary.function_signature + elif fname == "main": + astprototype = astree.mk_vinfo_main_function(faddr) + + # Offsets are negated to make them consistent with their internal + # numerical value. + # + # For convenience local stack variable introductions are specified + # with positive offsets. Internally these are negative values (i.e., + # the difference between stackpointer value and stackpointer value + # at function entry, with a stack growing down). + fstackvarintros = { + -off: name + for (off, name) in stackvarintros.get(faddr, {}).items()} + functionannotation = userhints.function_annotation(faddr) + + astinterface = ASTInterface( + astree, + typconverter, + xinfo.architecture, + srcprototype=srcprototype, + astprototype=astprototype, + appsignature=appsignature, + varintros=varintros, + functionannotation=functionannotation, + stackvarintros=fstackvarintros, + patchevents=patchevents, + verbose=verbose) + + # Introduce ssa variables for all reaching definitions referenced in + # xdata records for all instructions in the function. Locations that + # have a common user are merged. Types are provided by lhs_types. + untyped = astinterface.introduce_ssa_variables( + f.rdef_location_partition(), f.register_lhs_types, f.lhs_names) + + # Introduce stack variables for all stack buffers with types + astinterface.introduce_stack_variables( + f.stackframe, f.stack_variable_types) + + regsuntyped: List[str] = [] + for (reg, varlocs) in untyped.items(): + for (var, locs) in varlocs.items(): + if len(locs) == 1 and locs[0] == "init": + continue + if "_spill" in var: + continue + if reg == "SP": + continue + # print(" Untyped: " + reg + ": " + var + " [" + ",".join(locs) + "]") + if not reg in regsuntyped: + regsuntyped.append(reg) + + astfunction = ASTInterfaceFunction( + faddr, fname, f, astinterface, patchevents=patchevents) + + try: + asts = astfunction.mk_asts(support) + except UF.CHBError as e: + UC.print_error( + "Unable to create lifting for " + + faddr + + ":\n" + + ("-" * 80) + + "\n" + + str(e)) + functions_failed += 1 + # continue + raise + except Exception as e: + chklogger.logger.critical_id( + CHKLogID.MISC_ABORT_0001, + "ABORT: Unable to create lifting for " + + faddr + + " (" + + str(e) + + ")") + functions_failed += 1 + continue - print("\n// Lifted code for function " + faddr) - print("// --------------------------------------------------") - annotations: Dict[int, List[str]] = {} - if not hide_annotations: - annotations = astinterface.annotations - prettyprinter = ASTCPrettyPrinter( - localsymboltable, annotations=annotations) - print(prettyprinter.to_c(asts[0], include_globals=(not hide_globals))) - functions_lifted += 1 + if len(asts) >= 2: + astapi.add_function_ast(astree, asts, verbose) - else: - print("\nUnable to generate a lifting for " + faddr) - functions_failed += 1 - continue + print("\n// Lifted code for function " + faddr) + print("// --------------------------------------------------") + annotations: Dict[int, List[str]] = {} + if not hide_annotations: + annotations = astinterface.annotations + prettyprinter = ASTCPrettyPrinter( + localsymboltable, annotations=annotations) + print(prettyprinter.to_c(asts[0], include_globals=(not hide_globals))) + functions_lifted += 1 - if show_reachingdefs: - if output_reachingdefs is None: - UC.print_error("\nSpecify a file to save the reaching defs") + else: + print("\nUnable to generate a lifting for " + faddr) + functions_failed += 1 continue - if len(reachingdefs_registers) == 0: - reachingdefs_registers = regsuntyped - - for register in reachingdefs_registers: - if not register in f.rdef_location_partition(): - UC.print_status_update( - "Register " + register + " not found in rdeflocations") + if show_reachingdefs: + if output_reachingdefs is None: + UC.print_error("\nSpecify a file to save the reaching defs") continue - for reg in reachingdefs_registers: - print_reachingdefs( - app, - astinterface, - output_reachingdefs + "__" + reg, - fileformat, - f, - reg) + if len(reachingdefs_registers) == 0: + reachingdefs_registers = regsuntyped + + for register in reachingdefs_registers: + if not register in f.rdef_location_partition(): + UC.print_status_update( + "Register " + register + " not found in rdeflocations") + continue + + for reg in reachingdefs_registers: + print_reachingdefs( + app, + astinterface, + output_reachingdefs + "__" + reg, + fileformat, + f, + reg) else: UC.print_error("Unable to find function " + faddr) diff --git a/chb/cmdline/chkx b/chb/cmdline/chkx index 5f01af8c..d09597c8 100755 --- a/chb/cmdline/chkx +++ b/chb/cmdline/chkx @@ -803,7 +803,19 @@ def parse() -> argparse.Namespace: choices=["a", "w"], default="a", help="file mode for log file: append (a, default), or write (w)") - + buildast.add_argument( + "--jlogfilename", + help="name of json file to write log messages in json format") + buildast.add_argument( + "--jloglevel", + choices=UL.LogLevel.options(), + default="WARNING", + help="set logging level to the given level (does not activate logging)") + buildast.add_argument( + "--jlogfilemode", + choices=["a", "w"], + default="w", + help="file mode for json log file: append (a), or write (w, default)") buildast.set_defaults(func=AST.buildast) # -- results decompile (abstract syntax tree) -- diff --git a/chb/cmdline/commandutil.py b/chb/cmdline/commandutil.py index 67c9cf59..96524154 100644 --- a/chb/cmdline/commandutil.py +++ b/chb/cmdline/commandutil.py @@ -147,16 +147,27 @@ def get_path_filename(xname: str) -> Tuple[str, str]: def set_logging( level: str, path: str, - logfilename: Optional[str], + logfilename: Optional[str] = None, + jlogfilename: Optional[str] = None, msg: str = "", - mode: str = "a") -> None: + mode: str = "a", + jmode: str = "w", + jlevel: str = "NONE") -> None: if level in LogLevel.all() or logfilename is not None: if logfilename is not None: logfilename = os.path.join(path, logfilename) + if jlogfilename is not None: + jlogfilename = os.path.join(path, jlogfilename) chklogger.set_chkx_logger( - msg, level=level, logfilename=logfilename, mode=mode) + msg, + level=level, + logfilename=logfilename, + mode=mode, + jlogfilename=jlogfilename, + jmode=jmode, + jlevel=jlevel) def create_xinfo(path: str, xfile: str) -> XI.XInfo: diff --git a/chb/invariants/XXprUtil.py b/chb/invariants/XXprUtil.py index 775cbe9e..2eb36b76 100644 --- a/chb/invariants/XXprUtil.py +++ b/chb/invariants/XXprUtil.py @@ -101,7 +101,7 @@ import chb.invariants.XXpr as X import chb.util.fileutil as UF -from chb.util.loggingutil import chklogger +from chb.util.loggingutil import chklogger, CHKLogID if TYPE_CHECKING: @@ -235,9 +235,23 @@ def xconstant_to_ast_expr( else: if not anonymous: - chklogger.logger.error( - "AST conversion of constant %s not yet supported at address %s", - str(xc), iaddr) + if xc.is_random_constant: + chklogger.logger.error_id( + CHKLogID.RSLT_RANDOM_0001, + "RSLT: Encountered random constant at address %s", iaddr) + elif xc.is_false: + chklogger.logger.error_id( + CHKLogID.RSLT_XCFALSE_0001, + "RSLT: Encountered constant FALSE at address %s", iaddr) + elif xc.is_true: + chklogger.logger.error_id( + CHKLogID.RSLT_XCTRUE_0001, + "RSLT: Encountered constant TRUE at address %s", iaddr) + else: + chklogger.logger.error( + "AST conversion of constant %s not yet supported at address %s", + str(xc), iaddr) + return astree.mk_temp_lval_expression() @@ -334,9 +348,11 @@ def vreturn_deref_value_to_ast_lval_expression( return astree.mk_memref_expr(vexpr, anonymous=anonymous) else: if not anonymous: - chklogger.logger.error( - "Non-struct pointer type %s not yet handled at %s", - str(vtype), iaddr) + chklogger.logger.error_id( + CHKLogID.UNSP_RETVAR_0001, + "UNSP: Non-struct pointer type %s for variable " + + "%s not yet handled at %s", + str(vtype), str(vexpr), iaddr) return astree.mk_temp_lval_expression() else: @@ -597,9 +613,18 @@ def stack_variable_to_lval_expression( return astree.mk_vinfo_lval_expression( vinfo, astoffset, anonymous=anonymous) - chklogger.logger.warning( - "Stack variable offset %s of %s not yet handled at address %s", - str(offset.offset), str(vinfo), iaddr) + else: + if not anonymous: + chklogger.logger.error_id( + CHKLogID.UNSP_DATASTR_0001, + "UNSP: Stack variable constant offset %s of %s not yet handled at " + + "address %s", str(offset.offset), str(vinfo), str(iaddr)) + return astree.mk_temp_lval_expression() + + if not anonymous: + chklogger.logger.warning( + "Stack variable offset %s of %s not yet handled at address %s", + str(offset.offset), str(vinfo), iaddr) return astree.mk_temp_lval_expression() def global_variable_to_lval_expression( @@ -671,15 +696,17 @@ def global_variable_to_lval_expression( if not anonymous: if vinfo is None: - chklogger.logger.error( - "Conversion of global variable with address %s and offset " + chklogger.logger.error_id( + CHKLogID.UNSP_GLBVAR_0001, + "UNSP: Conversion of global variable with address %s and offset " + "%s at address %s not yet supported", str(hexgaddr), str(offset.offset), iaddr) else: - chklogger.logger.error( - "Conversion of global variable %s access with offset " - + "%s at address %s not yet supported", - str(vinfo), str(offset.offset), iaddr) + chklogger.logger.error_id( + CHKLogID.UNSP_GLBVAR_0002, + "UNSP: Conversion of global variable %s access with offset " + + "%s at address %s not yet supported", + str(vinfo), str(offset.offset), iaddr) return astree.mk_temp_lval_expression() if not anonymous: @@ -941,8 +968,9 @@ def xvariable_to_ast_def_lval_expression( and xdata.function.has_var_disequality(iaddr, xvar)): if (not anonymous): - chklogger.logger.warning( - "AST def conversion of initial memory value %s that may have " + chklogger.logger.error_id( + CHKLogID.RSLT_ERRGLB_0001, + "RSLT: conversion of initial memory value %s that may have " + "changed reverted to original variable at %s", str(xvar), str(iaddr)) @@ -1002,21 +1030,30 @@ def xvariable_to_ast_def_lval_expression( vinfo, anonymous=anonymous) else: if not anonymous: - chklogger.logger.error( - "Rdef: %s has not yet been introduced at address %s", - regrdefs[0], iaddr) + if "clobber" in regrdefs[0]: + chklogger.logger.error_id( + CHKLogID.RDEF_CLOBBER_0002, + "RDEF: clobbered value %s found at address %s", + regrdefs[0], iaddr) + else: + chklogger.logger.error_id( + CHKLogID.RDEF_MISSING_0001, + "RDEF: %s has not yet been introduced at address %s", + regrdefs[0], iaddr) return astree.mk_temp_lval_expression() if len(regrdefs) == 0: if not anonymous: - chklogger.logger.error( - "No rdefs found for %s at address %s", str(reg), iaddr) + chklogger.logger.error_id( + CHKLogID.RDEF_MISSING_0002, + "RDEF: No rdefs found for %s at address %s", str(reg), iaddr) return astree.mk_temp_lval_expression() else: if not anonymous: - chklogger.logger.error( - "No rdefs found for %s at address %s", str(reg), iaddr) + chklogger.logger.error_id( + CHKLogID.RDEF_MISSING_0003, + "RDEF: No rdefs found for %s at address %s", str(reg), iaddr) return astree.mk_temp_lval_expression() if xvar.is_global_variable: @@ -1029,7 +1066,16 @@ def xvariable_to_ast_def_lval_expression( return stack_variable_to_lval_expression( stackvar.offset, xdata, iaddr, astree, anonymous=anonymous) - if xvar.is_memory_variable: + if xvar.is_stack_argument: + stackarg = cast("VMemoryVariable", xvar.denotation) + if not anonymous: + chklogger.logger.error_id( + CHKLogID.UNSP_STCKARG_0001, + "UNSP: Conversion of stack argument %s not yet supported at address %s", + str(stackarg), iaddr) + return astree.mk_temp_lval_expression() + + if xvar.is_memory_variable and xvar.denotation.is_basevar_variable: memvar = cast("VMemoryVariable", xvar.denotation) return memory_variable_to_lval_expression( memvar.base, memvar.offset, xdata, iaddr, astree, anonymous=anonymous) @@ -1084,10 +1130,16 @@ def xvariable_to_ast_def_lval_expression( return astree.mk_lval_expr(stacklval) if not anonymous: - chklogger.logger.error( - "AST def conversion of variable %s to lval-expression at address " - + "%s not yet supported", - str(xvar), iaddr) + if "amp_0x" in str(xvar): + chklogger.logger.error_id( + CHKLogID.RSLT_ERRFRZ_0001, + "RSLT: Encountered unsimplified frozen value: %s at address %s", + str(xvar), iaddr) + else: + chklogger.logger.error( + "AST def conversion of variable %s to lval-expression at " + + "address %s not yet supported", + str(xvar), iaddr) return astree.mk_temp_lval_expression() @@ -1187,8 +1239,9 @@ def default() -> AST.ASTExpr: if not (axpr1.is_ast_lval_expr or axpr1.is_ast_addressof): if not anonymous: - chklogger.logger.warning( - "AST def conversion of pointer expression encountered unexpected " + chklogger.logger.warning_id( + CHKLogID.UNSP_PTRXPR_0001, + "UNSP: Conversion of pointer expression encountered unexpected " + " base expression %s at address %s", str(axpr1), iaddr) return default() @@ -1308,8 +1361,9 @@ def default() -> AST.ASTExpr: return astree.mk_binary_expression(operator, astxpr1, astxpr2) else: if not anonymous: - chklogger.logger.error( - "AST def conversion of binary expression %s, %s with operator %s " + chklogger.logger.error_id( + CHKLogID.UNSP_BINOP_0001, + "UNSP: conversion of binary expression %s, %s with operator %s " + "at address %s not yet supported", str(xpr1), str(xpr2), operator, iaddr) return astree.mk_temp_lval_expression() @@ -1439,8 +1493,9 @@ def default() -> AST.ASTExpr: if not compinfo.has_fields(): if not anonymous: - chklogger.logger.error( - "Struct definition is missing for %s at address %s " + chklogger.logger.error_id( + CHKLogID.USER_STRCTDEF_0002, + "USER: Struct definition is missing for %s at address %s " + "(no fields found)", compinfo.compname, iaddr) return astree.mk_temp_lval_expression() @@ -1471,8 +1526,9 @@ def default() -> AST.ASTExpr: if not hl_addr.op == "plus": if not anonymous: - chklogger.logger.error( - "Address expression %s with operator %s not yet supported at %s", + chklogger.logger.warning_id( + CHKLogID.UNSP_INDEXEXP_0001, + "UNSP: Address expression %s with operator %s not yet supported at %s", str(xaddr), hl_addr.op, iaddr) return default() @@ -1497,8 +1553,9 @@ def default() -> AST.ASTExpr: if not compinfo.has_fields(): if not anonymous: - chklogger.logger.error( - "Struct definition is missing for %s at address %s " + chklogger.logger.error_id( + CHKLogID.USER_STRCTDEF_0001, + "USER: Struct definition is missing for %s at address %s " + "(no fields found)", compinfo.compname, iaddr) return astree.mk_temp_lval_expression() @@ -1651,10 +1708,11 @@ def stack_variable_to_ast_lval( return astree.mk_vinfo_lval(vinfo, offset=astoffset, anonymous=anonymous) if not anonymous: - chklogger.logger.warning( - "Stack variable with offset %s not yet supported at address %s", - str(stackoffset), iaddr) - return astree.mk_temp_lval() + chklogger.logger.error_id( + CHKLogID.UNSP_DATASTR_0002, + "UNSP: Stack variable constant offset %s of %s not yet handled at " + "address %s", str(stackoffset), str(vinfo), str(iaddr)) + return astree.mk_temp_lval() if not anonymous: chklogger.logger.error( diff --git a/chb/util/loggingutil.py b/chb/util/loggingutil.py index 163f8ed3..af913da4 100644 --- a/chb/util/loggingutil.py +++ b/chb/util/loggingutil.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2024 Aarno Labs, LLC +# Copyright (c) 2024-2026 Aarno Labs, LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,10 +26,24 @@ # ------------------------------------------------------------------------------ +import json +import contextvars +from contextlib import contextmanager import logging from enum import Enum from typing import Optional, List +_current_faddr: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "chkx_faddr", default=None) + +@contextmanager +def function_context(faddr: str): + token = _current_faddr.set(faddr) + try: + yield + finally: + _current_faddr.reset(token) + class LogLevel(str, Enum): """Simple type to restrict CLI log-level choices. @@ -52,14 +66,114 @@ def options(cls) -> List[str]: return [x.value for x in cls] + ["NONE"] +class NativeJsonFormatter(logging.Formatter): + + def format(self, record): + log_payload = { + "ts": self.formatTime(record, self.datefmt), + "lvl": record.levelname, + "faddr": getattr(record, "faddr", None), + "msg": record.getMessage(), + "module": record.module, + "line": record.lineno, + "log_id": getattr(record, "log_id", CHKLogID.MISC_NOTAGID_0001.value) + } + if record.exc_info: + log_payload["exception"] = self.formatException(record.exc_info) + return json.dumps(log_payload) + + +class CHKLogID(str, Enum): + + MISC_ABORT_0001 = "MISC-ABORT-0001" + MISC_NOTAGID_0001 = "MISC-NOTAGID-0001" + + RDEF_CLOBBER_0001 = "RDEF-CLOBBER-0001" + RDEF_CLOBBER_0002 = "RDEF-CLOBBER-0002" + + RDEF_MISSING_0001 = "RDEF-MISSING-0001" + RDEF_MISSING_0002 = "RDEF-MISSING-0002" + RDEF_MISSING_0003 = "RDEF-MISSING-0003" + + RDEF_UNRESOLVED_0001 = "RDEF-UNRESOLVED-0001" + + RSLT_BRNOCC_0001 = "RSLT-BRNOCC-0001" + RSLT_ERRCXPR_0001 = "RSLT-ERRCXPR-0001" + RSLT_ERRCXPR_0002 = "RSLT-ERRCXPR-0002" + RSLT_ERRCXPR_0003 = "RSLT-ERRCXPR-0003" + RSLT_ERRFRZ_0001 = "RSLT-ERRFRZ-0001" # raw frozen value (not simplified) + RSLT_ERRGLB_0001 = "RSLT-ERRGLB-0001" + RSLT_ERRVAL_0001 = "RSLT-ERRVAL-0001" + RSLT_ERRVAL_0002 = "RSLT-ERRVAL-0002" + RSLT_ERRVAL_0003 = "RSLT-ERRVAL-0003" + RSLT_ERRVAL_0004 = "RSLT-ERRVAL-0004" + RSLT_ERRVAL_0005 = "RSLT-ERRVAL-0005" + RSLT_ERRVAL_0006 = "RSLT-ERRVAL-0006" + RSLT_PRNOCC_0001 = "RSLT-PRNOCC-0001" # predicated instruction condition + RSLT_RANDOM_0001 = "RSLT-RANDOM-0001" + RSLT_XCFALSE_0001 = "RSLT-XCFALSE-0001" + RSLT_XCTRUE_0001 = "RSLT-XCTRUE-0001" + + UNSP_ASMINSTR_0001 = "UNSP-ASMINSTR-0001" + UNSP_BINOP_0001 = "UNSP-BINOP-0001" + UNSP_DATASTR_0001 = "UNSP-DATASTR-0001" + UNSP_DATASTR_0002 = "UNSP-DATASTR-0002" + UNSP_GLBVAR_0001 = "UNSP-GLBVAR-0001" + UNSP_GLBVAR_0002 = "UNSP-GLBVAR-0002" + UNSP_INDEXEXP_0001 = "UNSP-INDEXEXP-0001" + UNSP_INDCALL_0001 = "UNSP-INDCALL-0001" + UNSP_INDCALL_0002 = "UNSP-INDCALL-0002" + UNSP_JMPTBL_0001 = "UNSP-JMPTBL-0001" + UNSP_PTRXPR_0001 = "UNSP-PTRXPR-0001" + UNSP_RETVAR_0001 = "UNSP-RETVAR-0001" + UNSP_STCKARG_0001 = "UNSP-STCKARG-0001" + + USER_GLBDECL_0001 = "USER-GLBDECL-0001" # missing decl of global variable + USER_STRCTDEF_0001 = "USER-STRCTDEF-0001" # missing struct definition + USER_STRCTDEF_0002 = "USER-STRCTDEF-0002" # idem + + +class IDLogger(logging.LoggerAdapter): + def process(self, msg, kwargs): + """ + Interrupts the logging call and safely moves the dynamic 'log_id' + from kwargs into the final 'extra' dictionary. + """ + log_id = kwargs.pop('log_id', CHKLogID.MISC_NOTAGID_0001.value) + extra = kwargs.setdefault('extra', {}) + extra['log_id'] = log_id + extra.setdefault('faddr', _current_faddr.get()) + return msg, kwargs + + def log_with_id(self, level, log_id, msg, *args, **kwargs): + kwargs['log_id'] = log_id.value + self.log(level, msg, *args, **kwargs, stacklevel=4) + + def debug_id(self, log_id: CHKLogID, msg, *args, **kwargs): + self.log_with_id(logging.DEBUG, log_id, msg, *args, **kwargs) + + def info_id(self, log_id: CHKLogID, msg, *args, **kwargs): + self.log_with_id(logging.INFO, log_id, msg, *args, **kwargs) + + def warning_id(self, log_id: CHKLogID, msg, *args, **kwargs): + self.log_with_id(logging.WARNING, log_id, msg, *args, **kwargs) + + def error_id(self, log_id: CHKLogID, msg, *args, **kwargs): + self.log_with_id(logging.ERROR, log_id, msg, *args, **kwargs) + + def critical_id(self, log_id: CHKLogID, msg, *args, **kwargs): + self.log_with_id(logging.CRITICAL, log_id, msg, *args, **kwargs) + + class CHKLogger: def __init__(self) -> None: - self._logger = logging.getLogger("silent") - self._logger.addHandler(logging.NullHandler()) + self._baselogger = logging.getLogger("silent") + self._baselogger.addHandler(logging.NullHandler()) + self._logger = IDLogger(self._baselogger) @property - def logger(self) -> logging.Logger: + def logger(self) -> IDLogger: return self._logger def set_chkx_logger( @@ -67,25 +181,39 @@ def set_chkx_logger( initmsg: str = "", level: str = LogLevel.warning, logfilename: Optional[str] = None, - mode: str = "a") -> None: + mode: str = "a", + jlogfilename: Optional[str] = None, + jlevel: str = LogLevel.info, + jmode: str = "w") -> None: if not level in LogLevel.all(): level = LogLevel.warning.value - newlogger = logging.getLogger("chkx") - newlogger.setLevel(level) + baselogger = logging.getLogger("chkx") + baselogger.setLevel(logging.DEBUG) + newlogger: IDLogger = IDLogger(baselogger, {}) handler: logging.Handler if logfilename is not None: handler = logging.FileHandler(logfilename, mode=mode) + handler.setLevel(level) else: handler = logging.StreamHandler() + handler.setLevel(level) formatter = logging.Formatter( - fmt="%(asctime)s:%(name)s:%(levelname)s:%(message)s [%(module)s:%(lineno)d]") + fmt="%(asctime)s:[%(log_id)s]:%(name)s:%(levelname)s:%(message)s [%(module)s:%(lineno)d]", + defaults={"log_id": CHKLogID.MISC_NOTAGID_0001.value}) handler.setFormatter(formatter) - newlogger.addHandler(handler) + baselogger.addHandler(handler) + + if jlogfilename is not None: + jsonhandler = logging.FileHandler( + jlogfilename, encoding="utf-8", mode=jmode) + jsonhandler.setLevel(jlevel) + jsonhandler.setFormatter(NativeJsonFormatter()) + baselogger.addHandler(jsonhandler) self._logger = newlogger