Merge pull request #79 from nathan-weinberg/new-exec

feat: add new InvalidModelError and handling

Author: nathan-weinberg

src/instructlab/eval/exceptions.py

@@ -22,6 +22,23 @@ def __init__(self, path) -> None:
         self.message = f"Model could not be found at {path}"
 
 
+class InvalidModelError(EvalError):
+    """
+    Error raised when model can be found but is invalid
+
+    Attributes
+        message     error message to be printed on raise
+        path        filepath of model location
+        reason      root cause for model invalidity
+    """
+
+    def __init__(self, path, reason) -> None:
+        super().__init__()
+        self.path = path
+        self.reason = reason
+        self.message = f"Model found at {path} but was invalid due to: {reason}"
+
+
 class InvalidGitRepoError(EvalError):
     """
     Error raised when taxonomy dir provided isn't a valid git repo

src/instructlab/eval/mmlu.py

@@ -12,6 +12,7 @@
 # First Party
 from instructlab.eval.evaluator import Evaluator
 from instructlab.eval.exceptions import (
+    InvalidModelError,
     InvalidTasksDirError,
     ModelNotFoundError,
     TasksDirNotFoundError,
@@ -158,6 +159,9 @@ def _simple_evaluate_with_error_handling(self, **kwargs):
                 ose
             ) or "does not appear to have a file named" in str(ose):
                 raise ModelNotFoundError(self.model_path) from ose
+            if "is not a valid JSON file" in str(ose):
+                reason = "Looked for valid JSON file but couldn't find one - are you pointing at a directory with a 'config.json'?"
+                raise InvalidModelError(self.model_path, reason) from ose
             raise