added skip tests

Author: satvshr

tests/test_flows/test_flow.py

@@ -180,6 +180,7 @@ def test_to_xml_from_xml(self):
         openml.flows.functions.assert_flows_equal(new_flow, flow)
         assert new_flow is not flow
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_publish_flow(self):
@@ -222,6 +223,7 @@ def test_publish_existing_flow(self, flow_exists_mock):
             f"collected from {__file__.split('/')[-1]}: {flow.flow_id}",
         )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_publish_flow_with_similar_components(self):
@@ -273,6 +275,7 @@ def test_publish_flow_with_similar_components(self):
         TestBase._mark_entity_for_removal("flow", flow3.flow_id, flow3.name)
         TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}")
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_semi_legal_flow(self):
@@ -383,6 +386,7 @@ def get_sentinel():
         flow_id = openml.flows.flow_exists(name, version)
         assert not flow_id
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_existing_flow_exists(self):
@@ -424,6 +428,7 @@ def test_existing_flow_exists(self):
             )
             assert downloaded_flow_id == flow.flow_id
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_sklearn_to_upload_to_flow(self):

tests/test_flows/test_flow_functions.py

@@ -309,6 +309,7 @@ def test_get_flow1(self):
         flow = openml.flows.get_flow(1)
         assert flow.external_version is None
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_get_flow_reinstantiate_model(self):
@@ -392,6 +393,7 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self):
         assert flow.flow_id is None
         assert "sklearn==0.19.1" not in flow.dependencies
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_get_flow_id(self):

tests/test_runs/test_run.py

@@ -118,6 +118,7 @@ def _check_array(array, type_):
         else:
             assert run_prime_trace_content is None
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_to_from_filesystem_vanilla(self):
@@ -153,6 +154,7 @@ def test_to_from_filesystem_vanilla(self):
             f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}",
         )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.flaky()
     @pytest.mark.test_server()
@@ -189,6 +191,7 @@ def test_to_from_filesystem_search(self):
             f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}",
         )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_to_from_filesystem_no_model(self):
@@ -295,6 +298,7 @@ def assert_run_prediction_data(task, run, model):
             assert_method(y_pred, saved_y_pred)
             assert_method(y_test, saved_y_test)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_publish_with_local_loaded_flow(self):
@@ -339,6 +343,7 @@ def test_publish_with_local_loaded_flow(self):
             assert openml.flows.flow_exists(flow.name, flow.external_version)
             openml.runs.get_run(loaded_run.run_id)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_offline_and_online_run_identical(self):

tests/test_runs/test_run_functions.py

@@ -397,6 +397,7 @@ def _check_sample_evaluations(
                                 assert evaluation > 0
                             assert evaluation < max_time_allowed
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_regression_on_classif_task(self):
@@ -414,6 +415,7 @@ def test_run_regression_on_classif_task(self):
                 task=task,
             )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_check_erronous_sklearn_flow_fails(self):
@@ -627,6 +629,7 @@ def _run_and_upload_regression(
             sentinel=sentinel,
         )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_logistic_regression(self):
@@ -636,6 +639,7 @@ def test_run_and_upload_logistic_regression(self):
         n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"]
         self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501")
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_linear_regression(self):
@@ -667,6 +671,7 @@ def test_run_and_upload_linear_regression(self):
         n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"]
         self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501")
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_pipeline_dummy_pipeline(self):
@@ -681,6 +686,7 @@ def test_run_and_upload_pipeline_dummy_pipeline(self):
         n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"]
         self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501")
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -798,6 +804,7 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock):
                 call_count += 1
         assert call_count == 3
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_gridsearch(self):
@@ -821,6 +828,7 @@ def test_run_and_upload_gridsearch(self):
         )
         assert len(run.trace.trace_iterations) == 9
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_randomsearch(self):
@@ -854,6 +862,7 @@ def test_run_and_upload_randomsearch(self):
         trace = openml.runs.get_run_trace(run.run_id)
         assert len(trace.trace_iterations) == 5
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_and_upload_maskedarrays(self):
@@ -882,6 +891,7 @@ def test_run_and_upload_maskedarrays(self):
 
     ##########################################################################
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_learning_curve_task_1(self):
@@ -907,6 +917,7 @@ def test_learning_curve_task_1(self):
         )
         self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_learning_curve_task_2(self):
@@ -944,6 +955,7 @@ def test_learning_curve_task_2(self):
         )
         self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.21"),
@@ -1023,6 +1035,7 @@ def _test_local_evaluations(self, run):
                 assert alt_scores[idx] >= 0
                 assert alt_scores[idx] <= 1
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_local_run_swapped_parameter_order_model(self):
@@ -1039,6 +1052,7 @@ def test_local_run_swapped_parameter_order_model(self):
 
         self._test_local_evaluations(run)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1068,6 +1082,7 @@ def test_local_run_swapped_parameter_order_flow(self):
 
         self._test_local_evaluations(run)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1106,6 +1121,7 @@ def test_online_run_metric_score(self):
 
         self._test_local_evaluations(run)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1168,6 +1184,7 @@ def test_initialize_model_from_run(self):
         assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"'
         assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05"
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1228,6 +1245,7 @@ def test__run_exists(self):
             run_ids = run_exists(task.task_id, setup_exists)
             assert run_ids, (run_ids, clf)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_with_illegal_flow_id(self):
@@ -1248,6 +1266,7 @@ def test_run_with_illegal_flow_id(self):
                 avoid_duplicate_runs=True,
             )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_with_illegal_flow_id_after_load(self):
@@ -1280,6 +1299,7 @@ def test_run_with_illegal_flow_id_after_load(self):
             TestBase._mark_entity_for_removal("run", loaded_run.run_id)
             TestBase.logger.info(f"collected from test_run_functions: {loaded_run.run_id}")
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_with_illegal_flow_id_1(self):
@@ -1306,6 +1326,7 @@ def test_run_with_illegal_flow_id_1(self):
                 avoid_duplicate_runs=True,
             )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_with_illegal_flow_id_1_after_load(self):
@@ -1345,6 +1366,7 @@ def test_run_with_illegal_flow_id_1_after_load(self):
             loaded_run.publish,
         )
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1575,6 +1597,7 @@ def test_get_runs_list_by_tag(self):
         runs = openml.runs.list_runs(tag="curves", size=2)
         assert len(runs) >= 1
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1612,6 +1635,7 @@ def test_run_on_dataset_with_missing_labels_dataframe(self):
             # repeat, fold, row_id, 6 confidences, prediction and correct label
             assert len(row) == 12
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1666,6 +1690,7 @@ def test_get_uncached_run(self):
         with pytest.raises(openml.exceptions.OpenMLCacheException):
             openml.runs.functions._get_cached_run(10)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_run_flow_on_task_downloaded_flow(self):
@@ -1767,7 +1792,7 @@ def test_format_prediction_task_regression(self):
         self.assertListEqual(res, [0] * 5)
 
 
-
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
         reason="SimpleImputer doesn't handle mixed type DataFrame as input",
@@ -1864,7 +1889,7 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key):
     assert run_url == mock_delete.call_args.args[0]
     assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
 
-
+@pytest.mark.skip(reason="Pending resolution of #1657")
 @pytest.mark.sklearn()
 @unittest.skipIf(
     Version(sklearn.__version__) < Version("0.21"),
@@ -1946,7 +1971,7 @@ def test__run_task_get_arffcontent_2(parallel_mock):
         err_msg="Observed performance scores deviate from expected ones.",
     )
 
-
+@pytest.mark.skip(reason="Pending resolution of #1657")
 @pytest.mark.sklearn()
 @unittest.skipIf(
     Version(sklearn.__version__) < Version("0.21"),

tests/test_setups/test_setup_functions.py

@@ -34,6 +34,7 @@ def setUp(self):
         self.extension = SklearnExtension()
         super().setUp()
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_nonexisting_setup_exists(self):
@@ -82,6 +83,7 @@ def _existing_setup_exists(self, classif):
         setup_id = openml.setups.setup_exists(flow)
         assert setup_id == run.setup_id
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_existing_setup_exists_1(self):
@@ -98,12 +100,14 @@ def side_effect(self):
             nb = sklearn.naive_bayes.GaussianNB()
             self._existing_setup_exists(nb)
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_exisiting_setup_exists_2(self):
         # Check a flow with one hyperparameter
         self._existing_setup_exists(sklearn.naive_bayes.GaussianNB())
 
+    @pytest.mark.skip(reason="Pending resolution of #1657")
     @pytest.mark.sklearn()
     @pytest.mark.test_server()
     def test_existing_setup_exists_3(self):