moved trace attribute extraction elsewhere

Author: janvanrijn

openml/runs/functions.py

@@ -70,7 +70,7 @@ def run_task(task, model):
     run = OpenMLRun(task_id=task.task_id, flow_id=flow_id, dataset_id=dataset.dataset_id, model=model)
 
     try:
-        run.data_content, run.trace_content = _run_task_get_arffcontent(model, task, class_labels)
+        run.data_content, run.trace_content, run.trace_attributes = _run_task_get_arffcontent(model, task, class_labels)
     except PyOpenMLError as message:
         run.error_message = str(message)
         warnings.warn("Run terminated with error: %s" %run.error_message)
@@ -159,7 +159,7 @@ def _run_task_get_arffcontent(model, task, class_labels):
                 model_fold.fit(trainX, trainY)
 
                 if isinstance(model_fold, BaseSearchCV):
-                    _add_results_to_arfftrace(arff_tracecontent, fold_no, model_fold, rep_no)
+                    arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no))
                     model_classes = model_fold.best_estimator_.classes_
                 else:
                     model_classes = model_fold.classes_
@@ -181,11 +181,16 @@ def _run_task_get_arffcontent(model, task, class_labels):
 
     if not isinstance(model, BaseSearchCV):
         arff_tracecontent = None
+        arff_trace_attributes = None
+    else:
+        # arff_tracecontent is already set
+        arff_trace_attributes = _extract_arfftrace_attributes(model_fold)
 
-    return arff_datacontent, arff_tracecontent
+    return arff_datacontent, arff_tracecontent, arff_trace_attributes
 
 
-def _add_results_to_arfftrace(arff_tracecontent, fold_no, model, rep_no):
+def _extract_arfftrace(model, rep_no, fold_no):
+    arff_tracecontent = []
     for itt_no in range(0, len(model.cv_results_['mean_test_score'])):
         # we use the string values for True and False, as it is defined in this way by the OpenML server
         selected = 'false'
@@ -197,6 +202,30 @@ def _add_results_to_arfftrace(arff_tracecontent, fold_no, model, rep_no):
             if key.startswith("param_"):
                 arff_line.append(str(model.cv_results_[key][itt_no]))
         arff_tracecontent.append(arff_line)
+    return arff_tracecontent
+
+def _extract_arfftrace_attributes(model):
+    # attributes that will be in trace arff, regardless of the model
+    trace_attributes = [('repeat', 'NUMERIC'),
+                        ('fold', 'NUMERIC'),
+                        ('iteration', 'NUMERIC'),
+                        ('evaluation', 'NUMERIC'),
+                        ('selected', ['true', 'false'])]
+
+    # model dependent attributes for trace arff
+    for key in model.cv_results_:
+        if key.startswith("param_"):
+            if all(isinstance(i, (bool)) for i in model.cv_results_[key]):
+                type = ['True', 'False']
+            elif all(isinstance(i, (int, float)) for i in model.cv_results_[key]):
+                type = 'NUMERIC'
+            else:
+                values = list(set(model.cv_results_[key]))  # unique values
+                type = [str(i) for i in values]
+
+            attribute = ("parameter_" + key[6:], type)
+            trace_attributes.append(attribute)
+    return trace_attributes
 
 
 def get_runs(run_ids):

openml/runs/run.py

@@ -23,8 +23,8 @@ class OpenMLRun(object):
     def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
                  files=None, setup_id=None, tags=None, uploader=None, uploader_name=None,
                  evaluations=None, detailed_evaluations=None,
-                 data_content=None, trace_content=None, model=None, task_type=None,
-                 task_evaluation_measure=None, flow_name=None,
+                 data_content=None, trace_attributes=None, trace_content=None,
+                 model=None, task_type=None, task_evaluation_measure=None, flow_name=None,
                  parameter_settings=None, predictions_url=None, task=None,
                  flow=None, run_id=None):
         self.uploader = uploader
@@ -42,6 +42,7 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
         self.evaluations = evaluations
         self.detailed_evaluations = detailed_evaluations
         self.data_content = data_content
+        self.trace_attributes = trace_attributes
         self.trace_content = trace_content
         self.error_message = None
         self.task = task
@@ -80,7 +81,7 @@ def _generate_arff_dict(self):
         arff_dict['relation'] = 'openml_task_' + str(task.task_id) + '_predictions'
         return arff_dict
 
-    def _generate_trace_arff_dict(self, model):
+    def _generate_trace_arff_dict(self):
         """Generates the arff dictionary for uploading predictions to the server.
 
         Assumes that the run has been executed.
@@ -91,32 +92,13 @@ def _generate_trace_arff_dict(self, model):
             Dictionary representation of the ARFF file that will be uploaded.
             Contains information about the optimization trace.
         """
-        if self.trace_content is None:
+        if self.trace_content is None or len(self.trace_content) == 0:
             raise ValueError('No trace content avaiable.')
-        if not isinstance(model, BaseSearchCV):
-            raise PyOpenMLError('Cannot generate trace on provided classifier. (This should never happen.)')
+        if len(self.trace_attributes) != len(self.trace_content[0]):
+            raise ValueError('Trace_attributes and trace_content not compatible')
 
         arff_dict = {}
-        arff_dict['attributes'] = [('repeat', 'NUMERIC'),
-                                   ('fold', 'NUMERIC'),
-                                   ('iteration', 'NUMERIC'),
-                                   ('evaluation', 'NUMERIC'),
-                                   ('selected', ['true', 'false'])]
-        for key in model.cv_results_:
-            if key.startswith("param_"):
-                type = 'STRING'
-                if all(isinstance(i, (bool)) for i in model.cv_results_[key]):
-                    type = ['True', 'False']
-                elif all(isinstance(i, (int, float)) for i in model.cv_results_[key]):
-                    type = 'NUMERIC'
-                else:
-                    values = list(set(model.cv_results_[key])) # unique values
-                    type = [str(i) for i in values]
-                    print(key + ": " + str(type))
-
-                attribute = ("parameter_" + key[6:], type)
-                arff_dict['attributes'].append(attribute)
-
+        arff_dict['attributes'] = self.trace_attributes
         arff_dict['data'] = self.trace_content
         arff_dict['relation'] = 'openml_task_' + str(self.task_id) + '_predictions'
 
@@ -145,7 +127,7 @@ def publish(self):
             file_elements['predictions'] = ("predictions.arff", predictions)
 
         if self.trace_content is not None:
-            trace_arff = arff.dumps(self._generate_trace_arff_dict(self.model))
+            trace_arff = arff.dumps(self._generate_trace_arff_dict())
             file_elements['trace'] = ("trace.arff", trace_arff)
 
         return_value = _perform_api_call("/run/", file_elements=file_elements)

tests/test_runs/test_run_functions.py

@@ -117,7 +117,7 @@ def test__run_task_get_arffcontent(self):
                                 clf, task, class_labels)
 
         clf = SGDClassifier(loss='log', random_state=1)
-        arff_datacontent, arff_tracecontent = openml.runs.functions._run_task_get_arffcontent(
+        arff_datacontent, arff_tracecontent, _ = openml.runs.functions._run_task_get_arffcontent(
             clf, task, class_labels)
         # predictions
         self.assertIsInstance(arff_datacontent, list)