From 4b4a142da97a6cec30aa9d41ecec3e79aa2cac3b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 05:36:32 +0000 Subject: [PATCH 01/10] gp-elite: time-bounded restarts and an exact exported model (gp-elite 0.8.0) Each restart now receives an equal share of the remaining budget through the per-call time limit of gp-elite 0.8.0, with a 10 percent margin, so the fit ends within max_time. model() returns the formula the engine writes in the raw input variables and checks against predict at fit time, which fixes the exported NaN for powers of negative quantities and the wrong folding of min-max and z-score scaling. requirements.txt now asks for gp-elite>=0.8.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EWHDf9m7w3MoDeyD7ewSDZ --- algorithms/gp-elite/requirements.txt | 2 +- experiment/methods/gp-elite/regressor.py | 130 +++++++---------------- 2 files changed, 37 insertions(+), 95 deletions(-) diff --git a/algorithms/gp-elite/requirements.txt b/algorithms/gp-elite/requirements.txt index 34e17da5..c8146f9c 100644 --- a/algorithms/gp-elite/requirements.txt +++ b/algorithms/gp-elite/requirements.txt @@ -1 +1 @@ -gp-elite>=0.5.0 +gp-elite>=0.8.0 diff --git a/experiment/methods/gp-elite/regressor.py b/experiment/methods/gp-elite/regressor.py index e0265422..49765400 100644 --- a/experiment/methods/gp-elite/regressor.py +++ b/experiment/methods/gp-elite/regressor.py @@ -2,6 +2,11 @@ Exposes the three objects the harness reads: `est`, `hyper_params`, `eval_kwargs`, plus `model(est, X)` and `complexity(est)`. + +Requires gp-elite >= 0.8.0, which brings the two things this wrapper relies on: +a per-call time limit (`time_limit=`), so a restart can no longer overrun the +harness budget, and an exported formula written in the raw input variables and +checked against `predict` at fit time (`SRResult.sympy`). """ import time @@ -20,8 +25,10 @@ class GPEliteSRBench(GPEliteRegressor): """GP_ELITE for SRBench. fit() runs sequential restarts (each one a full independent evolution) and - keeps the best by validation R2, stopping once the time budget is nearly - spent or `restarts` is reached. + keeps the best by validation R2. Every restart is time-bounded: it receives + an equal share of the budget still available, so the whole fit ends within + `max_time` (minus a safety margin) and the champion is published as soon as + it is found. """ def __init__(self, operators="physical", normalize="auto", generations=60, @@ -34,6 +41,11 @@ def __init__(self, operators="physical", normalize="auto", generations=60, random_state=random_state) self.max_time = max_time + def _publish(self, r): + self.model_ = r + self.equation_ = r.expression + self.is_fitted_ = True + def fit(self, X, y): if validate_data is not None: X, y = validate_data(self, X, y, y_numeric=True, @@ -49,44 +61,37 @@ def fit(self, X, y): names = ["X%d" % i for i in range(self.n_features_in_)] t0 = time.time() budget = max(30.0, float(self.max_time)) + # 10 % margin: final selection, predict and model export happen after + # the last restart, and the harness alarm must never be the one that + # ends the fit. + deadline = t0 + 0.90 * budget + n = max(1, int(self.restarts)) best, best_score = None, -np.inf seed0 = 0 if self.random_state is None else int(self.random_state) - for k in range(max(1, int(self.restarts))): + for k in range(n): + remaining = deadline - time.time() + if k > 0 and remaining <= 0: + break r = symbolic_regression( X, y, feature_names=names, operators=self.operators, normalize=self.normalize, generations=self.generations, speed=self.speed, validation_split=self.validation_split, restarts=1, robust=self.robust, parallel=self.parallel, + time_limit=max(1.0, remaining / (n - k)), seed=seed0 + 1000 * k) score = r.r2_validation if score is None or not np.isfinite(score): p = r.predict(X) ss = float(np.sum((y - y.mean()) ** 2)) or 1e-30 score = 1.0 - float(np.sum((y - p) ** 2) / ss) - if score > best_score: + if best is None or score > best_score: best, best_score = r, score - # Publish the champion AS SOON AS it is found, so that a - # single overrunning restart (symbolic_regression is not itself - # time-bounded) can never leave the estimator without model_. - # If the harness SIGALRM fires mid-restart, evaluate_model - # still finds a (possibly worse) result instead of raising - # AttributeError and losing the run entirely. - self.model_ = best - self.equation_ = best.expression - self.is_fitted_ = True - - elapsed = time.time() - t0 - if elapsed > 0.85 * budget or elapsed + elapsed / (k + 1) > budget: - break + self._publish(best) # published as soon as found - # Guarantee the attributes exist even if the very first restart failed - # to produce a usable result (defensive; best is normally set above). if best is not None and not getattr(self, "is_fitted_", False): - self.model_ = best - self.equation_ = best.expression - self.is_fitted_ = True + self._publish(best) return self @@ -103,81 +108,18 @@ def fit(self, X, y): eval_kwargs = {} -# Map every GP_ELITE operator to a sympy-parsable form. Anything produced by the -# "physical" or "full" operator sets must appear here; unknown names would parse -# as opaque undefined functions (e.g. cube(x) is NOT x**3 to sympy) and break -# numeric evaluation downstream. -_UNARY = { - "neg": lambda a: "(-(%s))" % a, - "abs": lambda a: "Abs(%s)" % a, - "inv": lambda a: "(1/(%s))" % a, - "sq": lambda a: "((%s)**2)" % a, - "cube": lambda a: "((%s)**3)" % a, - "sqrt": lambda a: "sqrt(Abs(%s))" % a, # engine's sqrt is domain-guarded - "log": lambda a: "log(Abs(%s))" % a, # engine's log is domain-guarded - "exp": lambda a: "exp(%s)" % a, - "sin": lambda a: "sin(%s)" % a, - "cos": lambda a: "cos(%s)" % a, - "tan": lambda a: "tan(%s)" % a, - "tanh": lambda a: "tanh(%s)" % a, - "step": lambda a: "Heaviside(%s)" % a, - "is_even": lambda a: "(1 - Mod(floor(%s), 2))" % a, -} -_BINARY = { - "+": lambda a, b: "((%s) + (%s))" % (a, b), - "-": lambda a, b: "((%s) - (%s))" % (a, b), - "*": lambda a, b: "((%s) * (%s))" % (a, b), - "/": lambda a, b: "((%s) / (%s))" % (a, b), - "pow": lambda a, b: "((%s)**(%s))" % (a, b), - "max2": lambda a, b: "Max((%s), (%s))" % (a, b), - "min2": lambda a, b: "Min((%s), (%s))" % (a, b), -} - - -def _to_sympy(node, cols): - """Walk the expression tree, emit a sympy-parsable string with the - dataset's real column names.""" - if node.left is None and node.right is None: - v = node.value - # np.float64 subclasses float, but np.int64 does NOT subclass int, so - # test the numpy hierarchy explicitly — otherwise an integer-valued - # numpy constant would fall through to the name branch and be emitted - # as a feature name instead of a number. - if isinstance(v, (int, float, np.integer, np.floating)) \ - and not isinstance(v, bool): - return "%.12g" % float(v) - s = str(v) # like 'X[3]' or 'X3' - digits = "".join(c for c in s if c.isdigit()) - i = int(digits) if digits else 0 - return cols[i] if i < len(cols) else "x%d" % i - - op = node.value - if node.right is None: # unary - a = _to_sympy(node.left, cols) - f = _UNARY.get(op) - if f is None: - raise ValueError("unmapped unary operator %r in _to_sympy" % op) - return f(a) - - a = _to_sympy(node.left, cols) - b = _to_sympy(node.right, cols) - f = _BINARY.get(op) - if f is None: - raise ValueError("unmapped binary operator %r in _to_sympy" % op) - return f(a, b) - - def model(est, X=None): + """The fitted formula as a sympy-parsable string in the dataset's columns. + + gp-elite >= 0.8 writes it in the raw input variables for every internal + normalisation (division by max|x| by default, min-max or z-score on + request) and checks at fit time that it reproduces predict(); protected + operators are spelled as the engine computes them (sqrt(Abs(u)) where u + can be negative). + """ cols = list(X.columns) if X is not None and hasattr(X, "columns") \ else ["x%d" % i for i in range(getattr(est, "n_features_in_", 0))] - # Fold the input normalization into the string so the formula, evaluated - # on RAW columns, reproduces est.predict exactly. The engine's shift-free - # scaler divides each feature by scale_[i]. - scaler = getattr(est.model_, "scaler", None) - if scaler is not None and hasattr(scaler, "scale_"): - cols = ["((%s)/%.12g)" % (c, float(f)) - for c, f in zip(cols, scaler.scale_)] - return _to_sympy(est.model_.node, cols) + return est.model_.sympy(cols) def complexity(est): From fc38379339563699b86d2635c1ae1c34d40b8504 Mon Sep 17 00:00:00 2001 From: Alcides Fonseca Date: Sat, 10 Oct 2026 23:57:46 +0000 Subject: [PATCH 02/10] Adopt Ruff and mypy in pre-commit and CI --- .github/workflows/code-quality.yml | 23 + .gitignore | 2 + .pre-commit-config.yaml | 18 + CONTRIBUTING.md | 38 +- algorithms/tir/regressor.py | 41 +- docs/CompetitionGuide.md | 69 +- docs/CompetitionGuide2023.md | 2 +- experiment/analyze.py | 562 +++++++++----- experiment/assess_symbolic_model.py | 238 +++--- experiment/conftest.py | 7 +- experiment/docs/source/conf.py | 79 +- experiment/evaluate_model.py | 418 +++++----- experiment/fix_aifeynman_model_size.py | 176 +++-- experiment/methods/afp/regressor.py | 56 +- experiment/methods/afp_ehc/regressor.py | 60 +- experiment/methods/afp_fe/regressor.py | 72 +- experiment/methods/bingo/regressor.py | 32 +- experiment/methods/brush/regressor.py | 15 +- experiment/methods/bsr/regressor.py | 39 +- experiment/methods/e2et/regressor.py | 51 +- experiment/methods/eplex/regressor.py | 54 +- experiment/methods/eql/regressor.py | 23 +- .../methods/experimental/GSGPRegressor.py | 55 +- experiment/methods/feat/regressor.py | 123 ++- experiment/methods/ffx/regressor.py | 13 +- experiment/methods/geneticengine/regressor.py | 10 +- .../methods/geneticengine_1p1/regressor.py | 10 +- .../methods/geneticengine_hc/regressor.py | 10 +- .../methods/geneticengine_rs/regressor.py | 10 +- experiment/methods/gp-elite/regressor.py | 136 ++-- experiment/methods/gpgomea/regressor.py | 4 +- experiment/methods/gplearn/regressor.py | 56 +- experiment/methods/gpzgd/regressor.py | 155 ++-- experiment/methods/itea/regressor.py | 100 +-- experiment/methods/keplearn/regressor.py | 115 ++- experiment/methods/lightgbm/regressor.py | 27 +- experiment/methods/nesymres/regressor.py | 321 ++++---- experiment/methods/operon/regressor.py | 185 +++-- experiment/methods/pantara/regressor.py | 474 +++++++----- experiment/methods/pir/regressor.py | 16 +- experiment/methods/ps-tree/regressor.py | 85 ++- experiment/methods/pysr/regressor.py | 38 +- experiment/methods/qlattice/regressor.py | 129 ++-- experiment/methods/rils-rols/regressor.py | 43 +- experiment/methods/sklearn/regressor.py | 1 - .../methods/sklearn_adaboost/regressor.py | 25 +- experiment/methods/sklearn_lasso/regressor.py | 10 +- .../methods/sklearn_linear/regressor.py | 8 +- experiment/methods/sklearn_mlp/regressor.py | 30 +- .../methods/sklearn_randomforest/regressor.py | 19 +- experiment/methods/sklearn_ridge/regressor.py | 41 +- experiment/methods/sklearn_sgd/regressor.py | 16 +- experiment/methods/tir/regressor.py | 57 +- experiment/methods/tpsr/regressor.py | 525 +++++++------ experiment/methods/tuned/AFPRegressor.py | 28 +- experiment/methods/tuned/AIFeynman.py | 7 +- experiment/methods/tuned/BSRRegressor.py | 10 +- experiment/methods/tuned/DSRRegressor.py | 13 +- experiment/methods/tuned/EHCRegressor.py | 26 +- experiment/methods/tuned/EPLEXRegressor.py | 26 +- experiment/methods/tuned/FEATRegressor.py | 16 +- experiment/methods/tuned/FE_AFPRegressor.py | 29 +- experiment/methods/tuned/FFXRegressor.py | 5 +- experiment/methods/tuned/GPGOMEARegressor.py | 9 +- experiment/methods/tuned/ITEARegressor.py | 11 +- experiment/methods/tuned/MRGPRegressor.py | 11 +- experiment/methods/tuned/OperonRegressor.py | 11 +- experiment/methods/tuned/gplearn.py | 10 +- .../methods/tuned/params/_afpregressor.py | 114 ++- experiment/methods/tuned/params/_aifeynman.py | 10 +- .../methods/tuned/params/_bsrregressor.py | 12 +- .../methods/tuned/params/_dsrregressor.py | 6 +- .../methods/tuned/params/_eplexregressor.py | 114 ++- .../methods/tuned/params/_fe_afpregressor.py | 114 ++- .../methods/tuned/params/_featregressor.py | 46 +- .../methods/tuned/params/_ffxregressor.py | 6 +- .../methods/tuned/params/_gpgomearegressor.py | 32 +- experiment/methods/tuned/params/_gplearn.py | 21 +- .../methods/tuned/params/_itearegressor.py | 7 +- .../methods/tuned/params/_mrgpregressor.py | 10 +- .../methods/tuned/params/_operonregressor.py | 35 +- .../methods/tuned/params/_sembackpropgp.py | 32 +- experiment/methods/tuned/sembackpropgp.py | 13 +- experiment/methods/udsr/regressor.py | 222 +++--- experiment/methods/xgboost/regressor.py | 16 +- experiment/methods_table.ipynb | 161 ++-- experiment/metrics/evaluation.py | 73 +- experiment/metrics/test_evaluation.py | 98 ++- experiment/read_file.py | 37 +- experiment/seeds.py | 110 ++- experiment/submit_jobs.py | 58 +- experiment/symbolic_utils.py | 247 +++--- experiment/test_algorithm.py | 148 ++-- experiment/test_evaluate_model.py | 40 +- experiment/test_tuned_models.py | 72 +- experiment/utils.py | 10 +- postprocessing/blackbox_results.ipynb | 721 ++++++++++-------- postprocessing/collate_blackbox_results.py | 182 ++--- postprocessing/collate_groundtruth_results.py | 140 ++-- postprocessing/genCSVs.py | 195 +++-- postprocessing/groundtruth_results.ipynb | 414 +++++----- postprocessing/paretoCSVs.py | 55 +- postprocessing/pareto_utils.py | 36 +- postprocessing/pick_best_params.py | 85 +-- postprocessing/pmlb_plots.ipynb | 60 +- postprocessing/statistical_comparisons.ipynb | 328 ++++---- postprocessing/symbolic_utils.py | 17 +- pyproject.toml | 15 + requirements-dev.txt | 2 + scripts/check_method_layout.py | 37 +- scripts/typecheck.py | 61 ++ 111 files changed, 5539 insertions(+), 3807 deletions(-) create mode 100644 .github/workflows/code-quality.yml create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 scripts/typecheck.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 00000000..d863df23 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,23 @@ +name: Code quality + +on: + push: + branches: [master, dev] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint-format-typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install pre-commit + run: python -m pip install -r requirements-dev.txt + - name: Lint, format, and type-check all source files + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.gitignore b/.gitignore index ba8c27ea..3b7523c0 100755 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,5 @@ ENV/ # mypy .mypy_cache/ +# Ruff +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..760b2fe7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +default_language_version: + python: python3 +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.0 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy (all Python sources) + entry: python scripts/typecheck.py + language: python + additional_dependencies: [mypy==2.4.0] + pass_filenames: false + always_run: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 033acbbb..4584bb84 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,38 @@ Ground Rules 2. Make the PR detailed and reference [specific issues](https://github.com/cavalab/srbench/issues) if the PR is meant to address any. 3. **Please be kind and please be patient**. We will be, too. +Code quality +============ + +Install the development tools and enable the commit hooks from the repository root: + +```bash +python -m pip install -r requirements-dev.txt +pre-commit install +pre-commit run --all-files +``` + +The hooks run Ruff linting (with safe automatic fixes), Ruff formatting, and mypy. +CI runs the same hooks over the entire repository on every pull request and on +pushes to `master` and `dev`. If a hook changes files, review and stage the changes, +then rerun it before committing. Python scripts and Jupyter notebook code cells +are linted and formatted; Markdown Python examples are also formatted. Notebook +outputs are preserved. + +Ruff rules and mypy settings live in `pyproject.toml`. Tool versions are pinned +in `.pre-commit-config.yaml`; update Ruff's hook revision and `required-version` +together when upgrading. The formatter targets Python 3.7 syntax to accommodate +older algorithm environments; development hooks and CI run on Python 3.12. + +For type checking alone, install `mypy==2.4.0` and run +`python scripts/typecheck.py`. The runner checks all Python files, including +hyphenated method directories that mypy's CLI cannot treat as package names. +Mypy checks unannotated function bodies without requiring type annotations +throughout this existing codebase. Algorithm libraries are installed in separate +containers, so third-party imports are treated as `Any` and site packages are +ignored for consistent results across environments. Python source in notebooks +is covered by Ruff, but is not type-checked by mypy. + How to contribute an SR method ============================== @@ -108,10 +140,10 @@ specify a mapping in the `model` function such as: ```python def model(est, X=None): - mapping = {'x_'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x_" + str(i): k for i, k in enumerate(X.columns)} new_model = est.model_ - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) ``` 2. The operators/functions in the model are available in [sympy's function set](https://docs.sympy.org/latest/modules/functions/index.html). diff --git a/algorithms/tir/regressor.py b/algorithms/tir/regressor.py index 0dd8c647..d06384b3 100644 --- a/algorithms/tir/regressor.py +++ b/algorithms/tir/regressor.py @@ -1,45 +1,50 @@ -import sys -import os import pyTIR as tir -from itertools import product -#os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + +# os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" hyper_params = [ { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Sqrt,Exp,Log,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Sqrt,Exp,Log,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Sqrt,Exp,Log,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Sqrt,Exp,Log,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), }, ] # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = tir.TIRRegressor(npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5,5), error="R^2", alg="MOO") +eval_kwargs = {"scale_x": False, "scale_y": False} +est = tir.TIRRegressor( + npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5, 5), error="R^2", alg="MOO" +) + def pre_train(est, X, y): """Adjust settings based on data before training""" - if X.shape[0]*X.shape[1] <= 1000: + if X.shape[0] * X.shape[1] <= 1000: est.penalty = 0.01 + def complexity(e): return e.len + def model(e, X): - new_model = e.sympy.replace("^","**") - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) + new_model = e.sympy.replace("^", "**") + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) return new_model + def get_population(est): pop = [] - for i in range(min(100, len(est.front))-1,-1,-1): + for i in range(min(100, len(est.front)) - 1, -1, -1): pop.append(est.create_model_from(i)) - return pop + return pop + def get_best_solution(est): return est diff --git a/docs/CompetitionGuide.md b/docs/CompetitionGuide.md index 0dc7b0ac..f45920c5 100644 --- a/docs/CompetitionGuide.md +++ b/docs/CompetitionGuide.md @@ -79,10 +79,10 @@ specify a mapping in the `model` function such as: ```python def model(est, X): - mapping = {'x_'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x_" + str(i): k for i, k in enumerate(X.columns)} new_model = est.model_ - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) ``` 2. The operators/functions in the model are available in [sympy's function set](https://docs.sympy.org/latest/modules/functions/index.html). @@ -92,8 +92,8 @@ If your method uses protected operators (e.g., `my_prot_div(a,b)= a/b if b!=0 el ```python def model(est, X): - ... # any previous pre-processing - new_model = new_model.replace("my_prot_div","/").replace("my_prot_log","log") + ... # any previous pre-processing + new_model = new_model.replace("my_prot_div", "/").replace("my_prot_log", "log") ``` ### CLI methods @@ -146,7 +146,8 @@ In python, one can grab this value as follows: ```python import os -n_cpus = os.environ['OMP_NUM_THREADS'] + +n_cpus = os.environ["OMP_NUM_THREADS"] ``` ### Time Budget @@ -172,18 +173,17 @@ function in `regressor.py` as part of `eval_kwargs`. As an example, one could define the following in `regressor.py`: ```python -def pre_train_fn(est, X, y): +def pre_train_fn(est, X, y): """set max_time in seconds based on length of X.""" - if len(X)<=1000: - max_time = 360 - 1 # 1 second of slack + if len(X) <= 1000: + max_time = 360 - 1 # 1 second of slack else: - max_time = 3600 - 1 # 1 second of slack + max_time = 3600 - 1 # 1 second of slack est.set_params(max_time) + # pass the function to eval_kwargs -eval_kwargs = { - 'pre_train': pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} ``` #### Timeout Handling @@ -195,22 +195,25 @@ Nonetheless, timeout handling looks like this: ```python import signal + class TimeOutException(Exception): pass + def alarm_handler(signum, frame): print(f"raising TimeOutException") raise TimeOutException + def evaluate(est): """model evaluation""" # alarm that sends SIGALRM signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: est.fit() except TimeOutException: - print('evaluate timeout') + print("evaluate timeout") ``` Example approaches are the following: @@ -280,39 +283,45 @@ Here is an example of how to do so: ```python from sklearn.model_selection import GridSearchCV -#from mylibrary import MyMethod + +# from mylibrary import MyMethod from sklearn import linear_model -hyper_params = { 'alpha': (1e-04,0.001,0.01,0.1,1,) } +hyper_params = { + "alpha": ( + 1e-04, + 0.001, + 0.01, + 0.1, + 1, + ) +} # define your base estimator -base_est=linear_model.LassoLars() +base_est = linear_model.LassoLars() # set est to be a GridSearchCV estimator -est = GridSearchCV(estimator=base_est, - param_grid=hyper_params, - cv=5, - refit=True) +est = GridSearchCV(estimator=base_est, param_grid=hyper_params, cv=5, refit=True) # set max time w.r.t. hyper-param tuning num_combos = 5 # five values of alpha -num_cv = 5 # 5 cross-validation splits as above -total_runs = num_combos * num_cv + 1 # +1 because refit=True +num_cv = 5 # 5 cross-validation splits as above +total_runs = num_combos * num_cv + 1 # +1 because refit=True + -def pre_train_fn(est, X, y): +def pre_train_fn(est, X, y): """set max_time in seconds based on length of X & hyper-parameter tuning""" - if len(X)<=1000: + if len(X) <= 1000: # account for hyper-parameter tuning, remove 1 extra second of slack to be sure it will terminate on time - max_time = 360 // total_runs - 1 + max_time = 360 // total_runs - 1 else: max_time = 3600 // total_runs - 1 est.set_params(max_time) + # pass the function eval_kwargs -eval_kwargs = { - 'pre_train': pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} # additional definitions # ... diff --git a/docs/CompetitionGuide2023.md b/docs/CompetitionGuide2023.md index 2362204c..7c39715f 100644 --- a/docs/CompetitionGuide2023.md +++ b/docs/CompetitionGuide2023.md @@ -72,7 +72,7 @@ The final score of each competitor will be composed of: The rank will be calculated for each data set independently such that, with N participants, the k-th ranked competitor (k=1 being the best) will be assigned a value of *N - k + 1*. The final score will be the harmonic mean of all of the scores and each participant will be ranked accordingly: ```python -score = 2*n / sum([ (1/acc[i]) + (1/simpl[i]) for i in (1..n)]) +score = 2 * n / sum([(1 / acc[i]) + (1 / simpl[i]) for i in ((1.0).n)]) ``` # Track 2 - Interpretability diff --git a/experiment/analyze.py b/experiment/analyze.py index 8eb7d11c..d936a71e 100755 --- a/experiment/analyze.py +++ b/experiment/analyze.py @@ -1,242 +1,377 @@ -import pdb -import pandas as pd +import argparse +import os import subprocess -import numpy as np from glob import glob -import argparse -import os, errno, sys + from joblib import Parallel, delayed from seeds import SEEDS -from yaml import load, Loader +from yaml import Loader, load -#TODO make this script smarter about running jobs. +# TODO make this script smarter about running jobs. # have it check to see whether results for that job exist before # resubmitting. That way the same command can be run multiple times. -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="An analyst for quick ML applications.", add_help=False) - parser.add_argument('DATASET_DIR', type=str, - help='Dataset directory like (pmlb/datasets)') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='LEARNERS',default=None, - type=str, help='Comma-separated list of ML methods to use (should ' - 'correspond to a py file name in methods/)') - parser.add_argument('--local', action='store_true', dest='LOCAL', default=False, - help='Run locally as opposed to on LPC') - parser.add_argument('--slurm', action='store_true', dest='SLURM', default=False, - help='Run on a SLURM scheduler as opposed to on LPC') - parser.add_argument('--noskips', action='store_true', dest='NOSKIPS', default=False, - help='Overwite existing results if found') - parser.add_argument('-skip_tuning', action='store_true', dest='SKIP_TUNE', default=False, - help='Skip tuning step') - parser.add_argument('-A', action='store', dest='A', default='plgsrbench', - help='SLURM account') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', default=False, - help='Specify a symbolic dataset') - parser.add_argument('-tuned',action='store_true', dest='TUNED', default=False, - help='Run tuned version of estimators. Only applies when ml=None') - parser.add_argument('-n_jobs',action='store',dest='N_JOBS',default=1,type=int, - help='Number of parallel jobs') - parser.add_argument('-time_limit',action='store',dest='TIME',default='48:00', - type=str, help='Time limit (hr:min) e.g. 24:00') - parser.add_argument('-seed',action='store',dest='SEED',default=None, - type=int, help='A specific random seed') - parser.add_argument('-n_trials',action='store',dest='N_TRIALS',default=1, - type=int, help='Number of parallel jobs') - parser.add_argument('-label',action='store',dest='LABEL',default='class', - type=str,help='Name of class label column') - parser.add_argument('-results',action='store',dest='RDIR',default='results', - type=str,help='Results directory') - parser.add_argument('-q',action='store',dest='QUEUE', - default='epistasis_long', - type=str,help='LSF queue') - parser.add_argument('-script',action='store',dest='SCRIPT', - default='evaluate_model', - type=str,help='Python script to run') - parser.add_argument('-m',action='store',dest='M',default=16384,type=int, - help='LSF memory request and limit (MB)') - parser.add_argument('-starting_seed',action='store',dest='START_SEED', - default=0,type=int, help='seed position to start with') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-job_limit',action='store',dest='JOB_LIMIT', - default=1000, type=int, - help='Limit number of jobs submitted at once') + description="An analyst for quick ML applications.", add_help=False + ) + parser.add_argument( + "DATASET_DIR", type=str, help="Dataset directory like (pmlb/datasets)" + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="LEARNERS", + default=None, + type=str, + help="Comma-separated list of ML methods to use (should " + "correspond to a py file name in methods/)", + ) + parser.add_argument( + "--local", + action="store_true", + dest="LOCAL", + default=False, + help="Run locally as opposed to on LPC", + ) + parser.add_argument( + "--slurm", + action="store_true", + dest="SLURM", + default=False, + help="Run on a SLURM scheduler as opposed to on LPC", + ) + parser.add_argument( + "--noskips", + action="store_true", + dest="NOSKIPS", + default=False, + help="Overwite existing results if found", + ) + parser.add_argument( + "-skip_tuning", + action="store_true", + dest="SKIP_TUNE", + default=False, + help="Skip tuning step", + ) + parser.add_argument( + "-A", action="store", dest="A", default="plgsrbench", help="SLURM account" + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + default=False, + help="Specify a symbolic dataset", + ) + parser.add_argument( + "-tuned", + action="store_true", + dest="TUNED", + default=False, + help="Run tuned version of estimators. Only applies when ml=None", + ) + parser.add_argument( + "-n_jobs", + action="store", + dest="N_JOBS", + default=1, + type=int, + help="Number of parallel jobs", + ) + parser.add_argument( + "-time_limit", + action="store", + dest="TIME", + default="48:00", + type=str, + help="Time limit (hr:min) e.g. 24:00", + ) + parser.add_argument( + "-seed", + action="store", + dest="SEED", + default=None, + type=int, + help="A specific random seed", + ) + parser.add_argument( + "-n_trials", + action="store", + dest="N_TRIALS", + default=1, + type=int, + help="Number of parallel jobs", + ) + parser.add_argument( + "-label", + action="store", + dest="LABEL", + default="class", + type=str, + help="Name of class label column", + ) + parser.add_argument( + "-results", + action="store", + dest="RDIR", + default="results", + type=str, + help="Results directory", + ) + parser.add_argument( + "-q", + action="store", + dest="QUEUE", + default="epistasis_long", + type=str, + help="LSF queue", + ) + parser.add_argument( + "-script", + action="store", + dest="SCRIPT", + default="evaluate_model", + type=str, + help="Python script to run", + ) + parser.add_argument( + "-m", + action="store", + dest="M", + default=16384, + type=int, + help="LSF memory request and limit (MB)", + ) + parser.add_argument( + "-starting_seed", + action="store", + dest="START_SEED", + default=0, + type=int, + help="seed position to start with", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-job_limit", + action="store", + dest="JOB_LIMIT", + default=1000, + type=int, + help="Limit number of jobs submitted at once", + ) args = parser.parse_args() - - if args.SLURM and args.QUEUE == 'epistasis_long': - print('setting queue to plgrid') - args.QUEUE = 'plgrid' - if args.LEARNERS == None: - prefix = 'methods/tuned/' if args.TUNED else 'methods/' - learners = [ml.split('/')[-1][:-3] for ml in glob(prefix+'*.py') - if not ml.split('/')[-1].startswith('_')] + if args.SLURM and args.QUEUE == "epistasis_long": + print("setting queue to plgrid") + args.QUEUE = "plgrid" + + if args.LEARNERS is None: + prefix = "methods/tuned/" if args.TUNED else "methods/" + learners = [ + ml.split("/")[-1][:-3] + for ml in glob(prefix + "*.py") + if not ml.split("/")[-1].startswith("_") + ] if args.TUNED: - learners = ['tuned.'+ml for ml in learners] + learners = ["tuned." + ml for ml in learners] else: - learners = [ml for ml in args.LEARNERS.split(',')] # learners - print('learners:',learners) + learners = [ml for ml in args.LEARNERS.split(",")] # learners + print("learners:", learners) - print('dataset directory:',args.DATASET_DIR) + print("dataset directory:", args.DATASET_DIR) - if args.DATASET_DIR.endswith('.tsv.gz'): + if args.DATASET_DIR.endswith(".tsv.gz"): datasets = [args.DATASET_DIR] - elif args.DATASET_DIR.endswith('*'): - print('capturing glob',args.DATASET_DIR+'/*.tsv.gz') - datasets = glob(args.DATASET_DIR+'*/*.tsv.gz') + elif args.DATASET_DIR.endswith("*"): + print("capturing glob", args.DATASET_DIR + "/*.tsv.gz") + datasets = glob(args.DATASET_DIR + "*/*.tsv.gz") else: - datasets = glob(args.DATASET_DIR+'/*/*.tsv.gz') - print('found',len(datasets),'datasets') + datasets = glob(args.DATASET_DIR + "/*/*.tsv.gz") + print("found", len(datasets), "datasets") ##################################################### ## look for existing jobs ######################## current_jobs = [] if args.SLURM: - res = subprocess.check_output(['squeue -o "%j"'],shell=True) - current_jobs = res.decode().split('\n') + res = subprocess.check_output(['squeue -o "%j"'], shell=True) + current_jobs = res.decode().split("\n") elif not args.LOCAL: - res = subprocess.check_output(['bjobs -o "JOB_NAME" -noheader'],shell=True) - current_jobs = res.decode().split('\n') + res = subprocess.check_output(['bjobs -o "JOB_NAME" -noheader'], shell=True) + current_jobs = res.decode().split("\n") # current_jobs = ['_'.join(cj.split('_')[:-1]) for cj in current_jobs] # write run commands jobs_w_results = [] - jobs_wout_results = [] - suffix = ('.json.updated' if args.SCRIPT=='assess_symbolic_model' else - '.json') + jobs_wout_results = [] + suffix = ".json.updated" if args.SCRIPT == "assess_symbolic_model" else ".json" queued_jobs = [] all_commands = [] - job_info=[] - for t in range(args.START_SEED, args.START_SEED+args.N_TRIALS): + job_info = [] + for t in range(args.START_SEED, args.START_SEED + args.N_TRIALS): # random_state = np.random.randint(2**15-1) - if args.SEED and args.N_TRIALS==1: + if args.SEED and args.N_TRIALS == 1: random_state = args.SEED else: random_state = SEEDS[t] # print('random_seed:',random_state) for dataset in datasets: - if (not args.SYM_DATA - and any([n in dataset for n in ['feynman', 'strogatz']]) - ): + if not args.SYM_DATA and any( + [n in dataset for n in ["feynman", "strogatz"]] + ): continue # grab regression datasets metadata = load( - open('/'.join(dataset.split('/')[:-1])+'/metadata.yaml','r'), - Loader=Loader) - if metadata['task'] != 'regression': + open("/".join(dataset.split("/")[:-1]) + "/metadata.yaml", "r"), + Loader=Loader, + ) + if metadata["task"] != "regression": continue - - dataname = dataset.split('/')[-1].split('.tsv.gz')[0] - results_path = '/'.join([args.RDIR, dataname]) + '/' + + dataname = dataset.split("/")[-1].split(".tsv.gz")[0] + results_path = "/".join([args.RDIR, dataname]) + "/" if not os.path.exists(results_path): os.makedirs(results_path) - + for ml in learners: - save_file = (results_path + '/' + dataname + '_' + ml + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataname + "_" + ml + "_" + str(random_state) + ) # if updated, check if json file exists (required) - if ('updated' in suffix - or args.SCRIPT.startswith('fix_')): - if not os.path.exists(save_file+'.json'): - jobs_wout_results.append([save_file,'json result DNE']) + if "updated" in suffix or args.SCRIPT.startswith("fix_"): + if not os.path.exists(save_file + ".json"): + jobs_wout_results.append([save_file, "json result DNE"]) continue if not args.NOSKIPS: - save_file = (results_path + '/' + dataname + '_' + ml + '_' - + str(random_state)) + save_file = ( + results_path + + "/" + + dataname + + "_" + + ml + + "_" + + str(random_state) + ) if args.Y_NOISE > 0: - save_file += '_target-noise'+str(args.Y_NOISE) + save_file += "_target-noise" + str(args.Y_NOISE) if args.X_NOISE > 0: - save_file += '_feature-noise'+str(args.X_NOISE) + save_file += "_feature-noise" + str(args.X_NOISE) # check if there is already a result for this experiment - if (os.path.exists(save_file+suffix) - and args.SCRIPT != 'fix_aifeynman_model_size'): - jobs_w_results.append([save_file,'exists']) + if ( + os.path.exists(save_file + suffix) + and args.SCRIPT != "fix_aifeynman_model_size" + ): + jobs_w_results.append([save_file, "exists"]) continue # check if there is already a queued job for this experiment - if save_file.split('/')[-1] in current_jobs: - queued_jobs.append([save_file,'queued']) + if save_file.split("/")[-1] in current_jobs: + queued_jobs.append([save_file, "queued"]) continue - - all_commands.append('python {SCRIPT}.py ' - '{DATASET}' - ' -ml {ML}' - ' -results_path {RDIR}' - ' -seed {RS} ' - ' -target_noise {TN} ' - ' -feature_noise {FN} ' - '{TEST} {SYM_DATA} {SKIP_TUNE}'.format( - SCRIPT=args.SCRIPT, - ML=ml, - DATASET=dataset, - RDIR=results_path, - RS=random_state, - TN=args.Y_NOISE, - FN=args.X_NOISE, - TEST=('-test' if args.TEST - else ''), - SYM_DATA=('-sym_data' if args.SYM_DATA - else ''), - SKIP_TUNE=('-skip_tuning' if - args.SKIP_TUNE else '') - ) - ) - job_info.append({'ml':ml, - 'dataset':dataname, - 'seed':str(random_state), - 'results_path':results_path, - 'target_noise':args.Y_NOISE - }) + all_commands.append( + "python {SCRIPT}.py " + "{DATASET}" + " -ml {ML}" + " -results_path {RDIR}" + " -seed {RS} " + " -target_noise {TN} " + " -feature_noise {FN} " + "{TEST} {SYM_DATA} {SKIP_TUNE}".format( + SCRIPT=args.SCRIPT, + ML=ml, + DATASET=dataset, + RDIR=results_path, + RS=random_state, + TN=args.Y_NOISE, + FN=args.X_NOISE, + TEST=("-test" if args.TEST else ""), + SYM_DATA=("-sym_data" if args.SYM_DATA else ""), + SKIP_TUNE=("-skip_tuning" if args.SKIP_TUNE else ""), + ) + ) + job_info.append( + { + "ml": ml, + "dataset": dataname, + "seed": str(random_state), + "results_path": results_path, + "target_noise": args.Y_NOISE, + } + ) if len(all_commands) > args.JOB_LIMIT: - print('shaving jobs down to job limit ({})'.format(args.JOB_LIMIT)) - all_commands = all_commands[:args.JOB_LIMIT] + print("shaving jobs down to job limit ({})".format(args.JOB_LIMIT)) + all_commands = all_commands[: args.JOB_LIMIT] if not args.NOSKIPS: - print('skipped',len(jobs_w_results),'jobs with results. Override with --noskips.') - print('skipped',len(jobs_wout_results),'jobs without results. Override with --noskips.') - print('skipped',len(queued_jobs),'queued jobs. Override with --noskips.') - print('submitting',len(all_commands),'jobs...') + print( + "skipped", + len(jobs_w_results), + "jobs with results. Override with --noskips.", + ) + print( + "skipped", + len(jobs_wout_results), + "jobs without results. Override with --noskips.", + ) + print("skipped", len(queued_jobs), "queued jobs. Override with --noskips.") + print("submitting", len(all_commands), "jobs...") if args.LOCAL: - # run locally - Parallel(n_jobs=args.N_JOBS)(delayed(os.system)(run_cmd) - for run_cmd in all_commands) - #for run_cmd in all_commands: + # run locally + Parallel(n_jobs=args.N_JOBS)( + delayed(os.system)(run_cmd) for run_cmd in all_commands + ) + # for run_cmd in all_commands: # print(run_cmd) # os.system(run_cmd) else: # sbatch - for i,run_cmd in enumerate(all_commands): - job_name = '_'.join([ - job_info[i]['dataset'], - job_info[i]['ml'], - job_info[i]['seed'], - args.SCRIPT - ]) - if args.Y_NOISE>0: - job_name += '_target-noise'+str(args.Y_NOISE) - if args.X_NOISE>0: - job_name += '_feature-noise'+str(args.X_NOISE) - out_file = (job_info[i]['results_path'] - + job_name - + '.%J.out') - error_file = out_file[:-4] + '.err' - + for i, run_cmd in enumerate(all_commands): + job_name = "_".join( + [ + job_info[i]["dataset"], + job_info[i]["ml"], + job_info[i]["seed"], + args.SCRIPT, + ] + ) + if args.Y_NOISE > 0: + job_name += "_target-noise" + str(args.Y_NOISE) + if args.X_NOISE > 0: + job_name += "_feature-noise" + str(args.X_NOISE) + out_file = job_info[i]["results_path"] + job_name + ".%J.out" + error_file = out_file[:-4] + ".err" + if args.SLURM: - batch_script = \ -"""#!/usr/bin/bash + batch_script = """#!/usr/bin/bash #SBATCH -o {OUT_FILE} #SBATCH --error={ERR_FILE} #SBATCH -N 1 @@ -251,49 +386,52 @@ {cmd} """.format( - OUT_FILE=out_file, - ERR_FILE=error_file, - JOB_NAME=job_name, - QUEUE=args.QUEUE, - A=args.A, - N_CORES=args.N_JOBS, - M=args.M, - cmd=run_cmd, - TIME=args.TIME - ) - with open('tmp_script','w') as f: - f.write(batch_script) + OUT_FILE=out_file, + ERR_FILE=error_file, + JOB_NAME=job_name, + QUEUE=args.QUEUE, + A=args.A, + N_CORES=args.N_JOBS, + M=args.M, + cmd=run_cmd, + TIME=args.TIME, + ) + with open("tmp_script", "w") as f: + f.write(batch_script) - # print(batch_script) - print(job_name) - sbatch_response = subprocess.check_output(['sbatch tmp_script'], - shell=True).decode() # submit jobs - print(sbatch_response) + # print(batch_script) + print(job_name) + sbatch_response = subprocess.check_output( + ["sbatch tmp_script"], shell=True + ).decode() # submit jobs + print(sbatch_response) - else: # LPC + else: # LPC # activate srbench env, load modules # pre_run_cmds = ["conda activate srbench", # "source lpc_modules.sh"] # run_cmd = '; '.join(pre_run_cmds + [run_cmd]) - bsub_cmd = ('bsub -o {OUT_FILE} ' - '-e {ERR_FILE} ' - '-n {N_CORES} ' - '-J {JOB_NAME} ' - '-q {QUEUE} ' - '-R "span[hosts=1] rusage[mem={M}]" ' - '-W {TIME} ' - '-M {M} ').format( - OUT_FILE=out_file, - ERR_FILE=error_file, - JOB_NAME=job_name, - QUEUE=args.QUEUE, - N_CORES=args.N_JOBS, - M=args.M, - TIME=args.TIME - ) - - bsub_cmd += '"' + run_cmd + '"' + bsub_cmd = ( + "bsub -o {OUT_FILE} " + "-e {ERR_FILE} " + "-n {N_CORES} " + "-J {JOB_NAME} " + "-q {QUEUE} " + '-R "span[hosts=1] rusage[mem={M}]" ' + "-W {TIME} " + "-M {M} " + ).format( + OUT_FILE=out_file, + ERR_FILE=error_file, + JOB_NAME=job_name, + QUEUE=args.QUEUE, + N_CORES=args.N_JOBS, + M=args.M, + TIME=args.TIME, + ) + + bsub_cmd += '"' + run_cmd + '"' print(bsub_cmd) - os.system(bsub_cmd) # submit jobs + os.system(bsub_cmd) # submit jobs - print('Finished submitting',len(all_commands),'jobs.') + print("Finished submitting", len(all_commands), "jobs.") diff --git a/experiment/assess_symbolic_model.py b/experiment/assess_symbolic_model.py index 7ac429cf..1f379c51 100644 --- a/experiment/assess_symbolic_model.py +++ b/experiment/assess_symbolic_model.py @@ -1,158 +1,204 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingGridSearchCV -from sklearn.model_selection import GridSearchCV, KFold, train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb +from sklearn.experimental import enable_halving_search_cv # noqa import numpy as np import json import os -import inspect -from utils import jsonify -from symbolic_utils import (clean_pred_model,get_sym_model,round_floats, - complexity, rewrite_AIFeynman_model_size) +from utils import jsonify +from symbolic_utils import ( + clean_pred_model, + get_sym_model, + round_floats, + complexity, + rewrite_AIFeynman_model_size, +) from sympy import simplify +import argparse + -def save(r,save_file): - print('saving...') - with open(save_file + '.updated', 'w') as out: +def save(r, save_file): + print("saving...") + with open(save_file + ".updated", "w") as out: json.dump(jsonify(r), out, indent=4) + def assess_symbolic_model_from_file(json_file, dataset): - - print('looking for:',json_file) + + print("looking for:", json_file) if os.path.exists(json_file): - r = json.load(open(json_file, 'r')) + r = json.load(open(json_file, "r")) else: - raise FileNotFoundError(json_file+' not found') + raise FileNotFoundError(json_file + " not found") - est_name = r['algorithm'] + est_name = r["algorithm"] true_model = get_sym_model(dataset, return_str=False) - r['true_model'] = str(true_model) - raw_model = r['symbolic_model'] + r["true_model"] = str(true_model) + raw_model = r["symbolic_model"] - if 'AIFeynman' in est_name: + if "AIFeynman" in est_name: # correct model size - r['model_size'] = rewrite_AIFeynman_model_size(raw_model) + r["model_size"] = rewrite_AIFeynman_model_size(raw_model) try: cleaned_model = clean_pred_model(raw_model, dataset, est_name) - r['simplified_symbolic_model'] = str(cleaned_model) - r['simplified_complexity'] = complexity(cleaned_model) - + r["simplified_symbolic_model"] = str(cleaned_model) + r["simplified_complexity"] = complexity(cleaned_model) + # save simplified model in case this is as far as we get save(r, json_file) # if the model is somewhat accurate, check and see if it # is an exact symbolic match - if r['r2_test'] > 0.5: + if r["r2_test"] > 0.5: sym_diff = round_floats(true_model - cleaned_model) - sym_frac = round_floats(cleaned_model/true_model) - print('sym_diff:',sym_diff) - print('sym_frac:',sym_frac) + sym_frac = round_floats(cleaned_model / true_model) + print("sym_diff:", sym_diff) + print("sym_frac:", sym_frac) # check if we can skip simplification - + if not sym_diff.is_constant() or sym_frac.is_constant(): sym_diff = round_floats(simplify(sym_diff, ratio=1)) - print('simplified sym_diff:',sym_diff) - r['symbolic_error'] = str(sym_diff) - r['symbolic_fraction'] = str(sym_frac) - r['symbolic_error_is_zero'] = str(sym_diff) == '0' - r['symbolic_error_is_constant'] = sym_diff.is_constant() - r['symbolic_fraction_is_constant'] = sym_frac.is_constant() + print("simplified sym_diff:", sym_diff) + r["symbolic_error"] = str(sym_diff) + r["symbolic_fraction"] = str(sym_frac) + r["symbolic_error_is_zero"] = str(sym_diff) == "0" + r["symbolic_error_is_constant"] = sym_diff.is_constant() + r["symbolic_fraction_is_constant"] = sym_frac.is_constant() else: raise ValueError("Model isnt accurate enough to check") except Exception as e: - r['sympy_exception'] = str(e) - if 'symbolic_error_is_zero' not in r.keys(): - r['symbolic_error_is_zero'] = False - if 'symbolic_error_is_constant' not in r.keys(): - r['symbolic_error_is_constant'] = False - if 'symbolic_fraction_is_constant' not in r.keys(): - r['symbolic_fraction_is_constant'] = False + r["sympy_exception"] = str(e) + if "symbolic_error_is_zero" not in r.keys(): + r["symbolic_error_is_zero"] = False + if "symbolic_error_is_constant" not in r.keys(): + r["symbolic_error_is_constant"] = False + if "symbolic_fraction_is_constant" not in r.keys(): + r["symbolic_fraction_is_constant"] = False print(json.dumps(r, indent=4)) save(r, json_file) - print('done.') + print("done.") + -def assess_symbolic_model(dataset, results_path, random_state, est_name, - target_noise=0.0, feature_noise=0.0): +def assess_symbolic_model( + dataset, results_path, random_state, est_name, target_noise=0.0, feature_noise=0.0 +): - print(40*'=','Assessing '+est_name+' model for ',dataset,40*'=',sep='\n') + print( + 40 * "=", "Assessing " + est_name + " model for ", dataset, 40 * "=", sep="\n" + ) np.random.seed(random_state) ################################################# # load json file ################################################# - dataset_name = dataset.split('/')[-1][:-7] + dataset_name = dataset.split("/")[-1][:-7] - save_file = (results_path + '/' + dataset_name + '_' + est_name + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataset_name + "_" + est_name + "_" + str(random_state) + ) if target_noise > 0: - save_file += '_target-noise'+str(target_noise) + save_file += "_target-noise" + str(target_noise) if feature_noise > 0: - save_file += '_feature-noise'+str(feature_noise) + save_file += "_feature-noise" + str(feature_noise) + + assess_symbolic_model_from_file(save_file + ".json", dataset) - assess_symbolic_model_from_file(save_file+'.json', dataset) ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', - help='Use symbolic dataset settings') - parser.add_argument('-json_file',action='store', dest='JSON_FILE', type=str, - default='',help='JSON results file') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + help="Use symbolic dataset settings", + ) + parser.add_argument( + "-json_file", + action="store", + dest="JSON_FILE", + type=str, + default="", + help="JSON results file", + ) args = parser.parse_args() print(args.__dict__) - if args.JSON_FILE != '': + if args.JSON_FILE != "": assess_symbolic_model_from_file(args.JSON_FILE, args.INPUT_FILE) else: - assess_symbolic_model(args.INPUT_FILE, args.RDIR, args.RANDOM_STATE, - args.ALG, target_noise=args.Y_NOISE, - feature_noise=args.X_NOISE) + assess_symbolic_model( + args.INPUT_FILE, + args.RDIR, + args.RANDOM_STATE, + args.ALG, + target_noise=args.Y_NOISE, + feature_noise=args.X_NOISE, + ) diff --git a/experiment/conftest.py b/experiment/conftest.py index e336f7c2..8d0b6fe7 100644 --- a/experiment/conftest.py +++ b/experiment/conftest.py @@ -1,12 +1,13 @@ -# configures pytest to allow CLI arguments. +# configures pytest to allow CLI arguments. + def pytest_addoption(parser): - parser.addoption("--ml", action="store", default="AFPRegressor") + parser.addoption("--ml", action="store", default="AFPRegressor") def pytest_generate_tests(metafunc): # This is called for every test. Only get/set command line arguments # if the argument is specified in the list of test "fixturenames". option_value = metafunc.config.option.ml - if 'ml' in metafunc.fixturenames and option_value is not None: + if "ml" in metafunc.fixturenames and option_value is not None: metafunc.parametrize("ml", [option_value]) diff --git a/experiment/docs/source/conf.py b/experiment/docs/source/conf.py index f0c2f3fa..671b2620 100755 --- a/experiment/docs/source/conf.py +++ b/experiment/docs/source/conf.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. @@ -19,14 +21,14 @@ # -- Project information ----------------------------------------------------- -project = 'Symbolic Regression Benchmark' -copyright = '2019, William La Cava, Patryk Orzechowski, Jason H. Moore' -author = 'William La Cava, Patryk Orzechowski, Jason H. Moore' +project = "Symbolic Regression Benchmark" +copyright = "2019, William La Cava, Patryk Orzechowski, Jason H. Moore" +author = "William La Cava, Patryk Orzechowski, Jason H. Moore" # The short X.Y version -version = '' +version = "" # The full version, including alpha/beta/rc tags -release = '0.0' +release = "0.0" # -- General configuration --------------------------------------------------- @@ -39,26 +41,26 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -70,10 +72,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns: list[str] = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -81,7 +83,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -92,7 +94,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -108,24 +110,21 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'SymbolicRegressionBenchmarkdoc' +htmlhelp_basename = "SymbolicRegressionBenchmarkdoc" # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { +latex_elements: dict[str, str] = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -135,8 +134,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'SymbolicRegressionBenchmark.tex', 'Symbolic Regression Benchmark Documentation', - 'William La Cava, Patryk Orzechowski, Jason H. Moore', 'manual'), + ( + master_doc, + "SymbolicRegressionBenchmark.tex", + "Symbolic Regression Benchmark Documentation", + "William La Cava, Patryk Orzechowski, Jason H. Moore", + "manual", + ), ] @@ -145,8 +149,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'symbolicregressionbenchmark', 'Symbolic Regression Benchmark Documentation', - [author], 1) + ( + master_doc, + "symbolicregressionbenchmark", + "Symbolic Regression Benchmark Documentation", + [author], + 1, + ) ] @@ -156,9 +165,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'SymbolicRegressionBenchmark', 'Symbolic Regression Benchmark Documentation', - author, 'SymbolicRegressionBenchmark', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "SymbolicRegressionBenchmark", + "Symbolic Regression Benchmark Documentation", + author, + "SymbolicRegressionBenchmark", + "One line description of project.", + "Miscellaneous", + ), ] @@ -167,4 +182,4 @@ # -- Options for todo extension ---------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True \ No newline at end of file +todo_include_todos = True diff --git a/experiment/evaluate_model.py b/experiment/evaluate_model.py index e82c7be2..84f8ef65 100755 --- a/experiment/evaluate_model.py +++ b/experiment/evaluate_model.py @@ -1,38 +1,36 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -from sklearn.model_selection import train_test_split -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb -import numpy as np +import argparse +import importlib +import inspect import json import os -import inspect -from utils import jsonify -from symbolic_utils import complexity, get_sym_model, get_sympy_model +import signal +import time +import numpy as np +import pandas as pd from metrics.evaluation import simplicity +from read_file import read_file +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler +from symbolic_utils import complexity, get_sym_model, get_sympy_model +from utils import jsonify + -import signal class TimeOutException(Exception): pass + def alarm_handler(signum, frame): - print(f"raising TimeOutException") + print("raising TimeOutException") raise TimeOutException + def set_env_vars(n_jobs): - os.environ['OMP_NUM_THREADS'] = n_jobs - os.environ['OPENBLAS_NUM_THREADS'] = n_jobs - os.environ['MKL_NUM_THREADS'] = n_jobs + os.environ["OMP_NUM_THREADS"] = n_jobs + os.environ["OPENBLAS_NUM_THREADS"] = n_jobs + os.environ["MKL_NUM_THREADS"] = n_jobs + def evaluate_model( *, @@ -47,7 +45,6 @@ def evaluate_model( target_noise=0.0, feature_noise=0.0, fit_time_limit=3600, - ########## # valid options for eval_kwargs ########## @@ -60,32 +57,27 @@ def evaluate_model( use_dataframe=True, ): - print(40*'=','Evaluating '+est_name+' on ',dataset,40*'=',sep='\n') + print(40 * "=", "Evaluating " + est_name + " on ", dataset, 40 * "=", sep="\n") np.random.seed(random_state) - if hasattr(est, 'random_state'): + if hasattr(est, "random_state"): est.random_state = random_state - features, labels, feature_names = read_file( - dataset, - use_dataframe=use_dataframe - ) - print('feature_names:',feature_names) + features, labels, feature_names = read_file(dataset, use_dataframe=use_dataframe) + print("feature_names:", feature_names) if sym_data: true_model = get_sym_model(dataset) # generate train/test split - X_train, X_test, y_train, y_test = train_test_split(features, labels, - train_size=0.75, - test_size=0.25, - random_state=random_state) + X_train, X_test, y_train, y_test = train_test_split( + features, labels, train_size=0.75, test_size=0.25, random_state=random_state + ) - # if dataset is large, subsample the training set + # if dataset is large, subsample the training set if max_train_samples > 0 and len(y_train) > max_train_samples: - print('subsampling training data from',len(X_train), - 'to',max_train_samples) - sample_idx = np.random.choice(np.arange(len(X_train)), - size=max_train_samples, - replace=False) + print("subsampling training data from", len(X_train), "to", max_train_samples) + sample_idx = np.random.choice( + np.arange(len(X_train)), size=max_train_samples, replace=False + ) y_train = y_train[sample_idx] if isinstance(X_train, pd.DataFrame): X_train = X_train.iloc[sample_idx] @@ -94,47 +86,50 @@ def evaluate_model( # scale and normalize the data if scale_x: - print('scaling X') - sc_X = StandardScaler() + print("scaling X") + sc_X = StandardScaler() X_train_scaled = sc_X.fit_transform(X_train) X_test_scaled = sc_X.transform(X_test) if use_dataframe: - X_train_scaled = pd.DataFrame(X_train_scaled, - columns=feature_names) - X_test_scaled = pd.DataFrame(X_test_scaled, - columns=feature_names) + X_train_scaled = pd.DataFrame(X_train_scaled, columns=feature_names) + X_test_scaled = pd.DataFrame(X_test_scaled, columns=feature_names) else: X_train_scaled = X_train X_test_scaled = X_test - if scale_y: - print('scaling y') + print("scaling y") sc_y = StandardScaler() - y_train_scaled = sc_y.fit_transform(y_train.reshape(-1,1)).flatten() + y_train_scaled = sc_y.fit_transform(y_train.reshape(-1, 1)).flatten() else: y_train_scaled = y_train - - ################################################## + ################################################## # noise - ################################################## + ################################################## if target_noise > 0: - print('adding',target_noise,'noise to target') - y_train_scaled += np.random.normal(0, - target_noise*np.sqrt(np.mean(np.square(y_train_scaled))), - size=len(y_train_scaled)) + print("adding", target_noise, "noise to target") + y_train_scaled += np.random.normal( + 0, + target_noise * np.sqrt(np.mean(np.square(y_train_scaled))), + size=len(y_train_scaled), + ) # add noise to the features if feature_noise > 0: - print('adding',target_noise,'noise to features') - X_train_scaled = np.array([x - + np.random.normal(0, feature_noise*np.sqrt(np.mean(np.square(x))), - size=len(x)) - for x in X_train_scaled.T]).T + print("adding", target_noise, "noise to features") + X_train_scaled = np.array( + [ + x + + np.random.normal( + 0, feature_noise * np.sqrt(np.mean(np.square(x))), size=len(x) + ) + for x in X_train_scaled.T + ] + ).T - ################################################## + ################################################## # run any method-specific pre_train routines - ################################################## + ################################################## if pre_train: pre_train(est, X_train_scaled, y_train_scaled) @@ -142,42 +137,42 @@ def evaluate_model( if test and len(test_params) != 0: est.set_params(**test_params) - ################################################## + ################################################## # Fit models - ################################################## - if test and fit_time_limit > 360: # lets give them 6 minutes + ################################################## + if test and fit_time_limit > 360: # lets give them 6 minutes fit_time_limit = 360 MAXTIME = fit_time_limit if len(y_train) > 1000 and not test and MAXTIME < 36000: MAXTIME = 36000 - print('max time:', MAXTIME) + print("max time:", MAXTIME) - if not use_dataframe: + if not use_dataframe: assert isinstance(X_train_scaled, np.ndarray) assert isinstance(X_test_scaled, np.ndarray) - print('X_train:',type(X_train_scaled),X_train_scaled.shape) - print('y_train:',y_train_scaled.shape) - print('training',est) + print("X_train:", type(X_train_scaled), X_train_scaled.shape) + print("y_train:", y_train_scaled.shape) + print("training", est) - if hasattr(est, 'max_time'): + if hasattr(est, "max_time"): est.max_time = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'timeout_in_seconds'): + print("max time set:", MAXTIME) + elif hasattr(est, "timeout_in_seconds"): est.timeout_in_seconds = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'timeout'): + print("max time set:", MAXTIME) + elif hasattr(est, "timeout"): est.timeout = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'stop_time'): + print("max time set:", MAXTIME) + elif hasattr(est, "stop_time"): est.stop_time = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'time_limit'): + print("max time set:", MAXTIME) + elif hasattr(est, "time_limit"): est.time_limit = MAXTIME - print('max time set:', MAXTIME) + print("max time set:", MAXTIME) else: - print('max time not set. Program will be killed if execution takes too long') + print("max time not set. Program will be killed if execution takes too long") signal.signal(signal.SIGALRM, alarm_handler) alarm_grace = 360 if test else 600 @@ -187,79 +182,79 @@ def evaluate_model( try: est.fit(X_train_scaled, y_train_scaled) except TimeOutException: - print('=' * 80) - print('WARNING: fitting timed out. If the program does not handle SIGALRM, it will be killed') - print('=' * 80) + print("=" * 80) + print( + "WARNING: fitting timed out. If the program does not handle SIGALRM, it will be killed" + ) + print("=" * 80) finally: signal.alarm(0) time_time = time.time() - t0t - print('Training time measure:', time_time) + print("Training time measure:", time_time) - if 'geneticengine' in est_name: + if "geneticengine" in est_name: est._is_fitted = True - + ################################################## # store results ################################################## - dataset_name = dataset.split('/')[-1].split('.')[0] + dataset_name = dataset.split("/")[-1].split(".")[0] results = { - 'dataset':dataset_name, - 'algorithm':est_name, - 'params':jsonify(est.get_params()), - 'random_state':random_state, - 'time_time': time_time, + "dataset": dataset_name, + "algorithm": est_name, + "params": jsonify(est.get_params()), + "random_state": random_state, + "time_time": time_time, } if sym_data: - results['true_model'] = true_model + results["true_model"] = true_model # get the final symbolic model as a string - print('fitted est:',est) + print("fitted est:", est) if model is None: - results['symbolic_model'] = 'not implemented' - elif 'X' in inspect.signature(model).parameters.keys(): + results["symbolic_model"] = "not implemented" + elif "X" in inspect.signature(model).parameters.keys(): if not isinstance(X_train_scaled, pd.DataFrame): - X_df = pd.DataFrame(X_train_scaled, - columns=feature_names) + X_df = pd.DataFrame(X_train_scaled, columns=feature_names) else: X_df = X_train_scaled - results['symbolic_model'] = model(est, X_df) + results["symbolic_model"] = model(est, X_df) else: - results['symbolic_model'] = model(est) - print('symbolic model:',results['symbolic_model']) + results["symbolic_model"] = model(est) + print("symbolic model:", results["symbolic_model"]) ################################################## # scores ################################################## - for fold, target, X in [ - ['train', y_train, X_train_scaled], - ['test', y_test, X_test_scaled] - ]: - - y_pred = np.asarray(est.predict(X)).reshape(-1,1) + for fold, target, X in [ + ["train", y_train, X_train_scaled], + ["test", y_test, X_test_scaled], + ]: + y_pred = np.asarray(est.predict(X)).reshape(-1, 1) if scale_y: y_pred = sc_y.inverse_transform(y_pred) - for score, scorer in [('mse',mean_squared_error), - ('mae',mean_absolute_error), - ('r2', r2_score) - ]: - results[score + '_' + fold] = scorer(target, y_pred) - - if results['symbolic_model'] != 'not implemented': - results['simplicity'] = simplicity(results['symbolic_model'], feature_names) + for score, scorer in [ + ("mse", mean_squared_error), + ("mae", mean_absolute_error), + ("r2", r2_score), + ]: + results[score + "_" + fold] = scorer(target, y_pred) + + if results["symbolic_model"] != "not implemented": + results["simplicity"] = simplicity(results["symbolic_model"], feature_names) else: - results['simplicity'] = np.nan + results["simplicity"] = np.nan def sympy_complexity(estimator): sympy_str = None if model is None: return -1 - if 'X' in inspect.signature(model).parameters.keys(): + if "X" in inspect.signature(model).parameters.keys(): if not isinstance(X_train_scaled, pd.DataFrame): - X_df = pd.DataFrame(X_train_scaled, - columns=feature_names) + X_df = pd.DataFrame(X_train_scaled, columns=feature_names) else: X_df = X_train_scaled sympy_str = model(estimator, X_df) @@ -276,101 +271,156 @@ def sympy_complexity(estimator): return -1 cplx = sympy_complexity(est) - results['complexity_function'] = 'sympy' + results["complexity_function"] = "sympy" - if cplx == -1 and ('complexity' in dir(algorithm) and algorithm.complexity is not None): + if cplx == -1 and ( + "complexity" in dir(algorithm) and algorithm.complexity is not None + ): cplx = algorithm.complexity(est) - results['complexity_function'] = 'user_defined' + results["complexity_function"] = "user_defined" - results['model_size'] = cplx - results['target_noise'] = target_noise - results['feature_noise'] = feature_noise + results["model_size"] = cplx + results["target_noise"] = target_noise + results["feature_noise"] = feature_noise ################################################## # write to file ################################################## - print('results:') + print("results:") print(json.dumps(results, indent=4)) - print('---') + print("---") if not os.path.exists(results_path): os.makedirs(results_path) save_file = os.path.join( - results_path, - '_'.join([dataset_name, est_name, str(random_state)]) + results_path, "_".join([dataset_name, est_name, str(random_state)]) ) - print('save_file:',save_file) + print("save_file:", save_file) - with open(save_file + '.json', 'w') as out: + with open(save_file + ".json", "w") as out: json.dump(jsonify(results), out, indent=4) - return save_file + '.json' + return save_file + ".json" + ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-n_jobs',action='store', type=str, default='4', - help='number of cores available') - parser.add_argument('-max_samples',action='store', type=int, default=0, - help='number of training samples') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', - help='Use symbolic dataset settings') - parser.add_argument('-skip_tuning',action='store_true', dest='SKIP_TUNE', - default=False, help='Dont tune the estimator') - parser.add_argument('-fit_time_limit',action='store',dest='FITTIME',default=3600, - type=int, help='Fit time limit (seconds) e.g. 3600 (1 hour). This is ' - 'the maximum time for the fit method, not the job, make sure job time ' - 'limit is greater than this.') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-n_jobs", + action="store", + type=str, + default="4", + help="number of cores available", + ) + parser.add_argument( + "-max_samples", + action="store", + type=int, + default=0, + help="number of training samples", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", action="store_true", help="Use symbolic dataset settings" + ) + parser.add_argument( + "-skip_tuning", + action="store_true", + dest="SKIP_TUNE", + default=False, + help="Dont tune the estimator", + ) + parser.add_argument( + "-fit_time_limit", + action="store", + dest="FITTIME", + default=3600, + type=int, + help="Fit time limit (seconds) e.g. 3600 (1 hour). This is " + "the maximum time for the fit method, not the job, make sure job time " + "limit is greater than this.", + ) args = parser.parse_args() set_env_vars(args.n_jobs) - # import algorithm - print('import from','methods.'+args.ALG+'.regressor') - algorithm = importlib.__import__('methods.'+args.ALG+'.regressor', - globals(), - locals(), - ['*'] - ) + # import algorithm + print("import from", "methods." + args.ALG + ".regressor") + algorithm = importlib.__import__( + "methods." + args.ALG + ".regressor", globals(), locals(), ["*"] + ) - print('algorithm:',algorithm.est) + print("algorithm:", algorithm.est) # optional keyword arguments passed to evaluate - eval_kwargs, test_params = {},{} - if 'eval_kwargs' in dir(algorithm): + eval_kwargs = {} + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs if args.max_samples != 0: - eval_kwargs['max_train_samples'] = args.max_samples + eval_kwargs["max_train_samples"] = args.max_samples evaluate_model( dataset=args.INPUT_FILE, @@ -384,5 +434,5 @@ def sympy_complexity(estimator): fit_time_limit=args.FITTIME, target_noise=args.Y_NOISE, feature_noise=args.X_NOISE, - **eval_kwargs + **eval_kwargs, ) diff --git a/experiment/fix_aifeynman_model_size.py b/experiment/fix_aifeynman_model_size.py index ccc1d852..59215156 100644 --- a/experiment/fix_aifeynman_model_size.py +++ b/experiment/fix_aifeynman_model_size.py @@ -1,114 +1,144 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingGridSearchCV -from sklearn.model_selection import GridSearchCV, KFold, train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb +from sklearn.experimental import enable_halving_search_cv # noqa import numpy as np import json import os -import inspect -from utils import jsonify -from symbolic_utils import (clean_pred_model,get_sym_model,round_floats, - complexity, rewrite_AIFeynman_model_size) -from sympy import simplify +from utils import jsonify +from symbolic_utils import rewrite_AIFeynman_model_size +import argparse + -def save(r,save_file): +def save(r, save_file): print(json.dumps(r, indent=4)) - print('saving...') - with open(save_file + '.json', 'w') as out: + print("saving...") + with open(save_file + ".json", "w") as out: json.dump(jsonify(r), out, indent=4) -def fix_aifeynman_model_size(dataset, results_path, random_state, est_name, - target_noise=0.0, feature_noise=0.0): +def fix_aifeynman_model_size( + dataset, results_path, random_state, est_name, target_noise=0.0, feature_noise=0.0 +): - print(40*'=','Assessing '+est_name+' model for ',dataset,40*'=',sep='\n') + print( + 40 * "=", "Assessing " + est_name + " model for ", dataset, 40 * "=", sep="\n" + ) np.random.seed(random_state) ################################################# # load json file ################################################# - dataset_name = dataset.split('/')[-1][:-7] + dataset_name = dataset.split("/")[-1][:-7] - save_file = (results_path + '/' + dataset_name + '_' + est_name + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataset_name + "_" + est_name + "_" + str(random_state) + ) if target_noise > 0: - save_file += '_target-noise'+str(target_noise) + save_file += "_target-noise" + str(target_noise) if feature_noise > 0: - save_file += '_feature-noise'+str(feature_noise) + save_file += "_feature-noise" + str(feature_noise) - print('looking for:',save_file+'.json') + print("looking for:", save_file + ".json") - if os.path.exists(save_file+'.json'): - r = json.load(open(save_file+'.json', 'r')) + if os.path.exists(save_file + ".json"): + r = json.load(open(save_file + ".json", "r")) else: - raise FileNotFoundError(save_file+'.json not found') + raise FileNotFoundError(save_file + ".json not found") - raw_model = r['symbolic_model'] + raw_model = r["symbolic_model"] - if 'AIFeynman' in est_name: + if "AIFeynman" in est_name: # correct model size model_size = rewrite_AIFeynman_model_size(raw_model) - print('changing model size from {} to {}'.format( - model_size,r['model_size'])) - r['model_size'] = model_size + print("changing model size from {} to {}".format(model_size, r["model_size"])) + r["model_size"] = model_size else: - raise ValueError('est_name not AIFeynman (it is %s)' % est_name) + raise ValueError("est_name not AIFeynman (it is %s)" % est_name) save(r, save_file) - print('done.') + print("done.") + ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', - help='Use symbolic dataset settings') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + help="Use symbolic dataset settings", + ) args = parser.parse_args() print(args.__dict__) - fix_aifeynman_model_size(args.INPUT_FILE, args.RDIR, args.RANDOM_STATE, args.ALG, - target_noise=args.Y_NOISE, feature_noise=args.X_NOISE - ) + fix_aifeynman_model_size( + args.INPUT_FILE, + args.RDIR, + args.RANDOM_STATE, + args.ALG, + target_noise=args.Y_NOISE, + feature_noise=args.X_NOISE, + ) diff --git a/experiment/methods/afp/regressor.py b/experiment/methods/afp/regressor.py index 3e868ef7..f4dd4de4 100644 --- a/experiment/methods/afp/regressor.py +++ b/experiment/methods/afp/regressor.py @@ -3,53 +3,51 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt', - 'sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=1, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g = 250, - time_limit=2*60*60 - ) +est = ellyn( + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=1, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=250, + time_limit=2 * 60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) - diff --git a/experiment/methods/afp_ehc/regressor.py b/experiment/methods/afp_ehc/regressor.py index de6a718a..57603516 100644 --- a/experiment/methods/afp_ehc/regressor.py +++ b/experiment/methods/afp_ehc/regressor.py @@ -1,14 +1,13 @@ from ellyn import ellyn # 500,000 evaluations = 100,000 with 1 eval, 1 constant hill climbing, and 3 -# EHC iterations +# EHC iterations pop_sizes = [250, 500, 1000] gs = [2500, 1000, 500] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt', - 'sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] @@ -16,44 +15,43 @@ # there is a time limit setting for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model est = ellyn( - eHC_on=True, - eHC_its=3, - selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g=100, - time_limit=60*60 - ) + eHC_on=True, + eHC_its=3, + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=100, + time_limit=60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/afp_fe/regressor.py b/experiment/methods/afp_fe/regressor.py index 29b00c40..525f157e 100644 --- a/experiment/methods/afp_fe/regressor.py +++ b/experiment/methods/afp_fe/regressor.py @@ -3,10 +3,10 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt','sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] @@ -14,55 +14,55 @@ # there is a time limit setting for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g=250, - time_limit=60*60, - - EstimateFitness=True, - FE_pop_size=100, - FE_ind_size=10, - FE_train_size=10, - FE_train_gens=10, - FE_rank=True, - ) +est = ellyn( + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=250, + time_limit=60 * 60, + EstimateFitness=True, + FE_pop_size=100, + FE_ind_size=10, + FE_train_size=10, + FE_train_gens=10, + FE_rank=True, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + def pre_train(est, X, y): """Adjust settings based on data before training""" # adjust generations based on size of X versus FE size g = est.g - est.g = int(g*len(X)/est.FE_ind_size) - print('FE ellyn gens adjusted from',g,'to',est.g) + est.g = int(g * len(X) / est.FE_ind_size) + print("FE ellyn gens adjusted from", g, "to", est.g) + eval_kwargs = dict(pre_train=pre_train, use_dataframe=False) diff --git a/experiment/methods/bingo/regressor.py b/experiment/methods/bingo/regressor.py index 6c80c1ca..25f8583e 100644 --- a/experiment/methods/bingo/regressor.py +++ b/experiment/methods/bingo/regressor.py @@ -1,18 +1,26 @@ -from sklearn.base import RegressorMixin +from __future__ import annotations + +from typing import Any -from bingo.symbolic_regression.symbolic_regressor import SymbolicRegressor from bingo.symbolic_regression.srbench_interface import ( - model, - get_population, - get_best_solution, eval_kwargs as base_eval_kwargs, ) +from bingo.symbolic_regression.srbench_interface import ( + get_best_solution as get_best_solution, +) +from bingo.symbolic_regression.srbench_interface import ( + get_population as get_population, +) +from bingo.symbolic_regression.srbench_interface import ( + model as model, +) +from bingo.symbolic_regression.symbolic_regressor import SymbolicRegressor eval_kwargs = dict(base_eval_kwargs) eval_kwargs["use_dataframe"] = False # sklearn's interface should not modify constructor parameters. Bingo modifies -# max_time, so here I'm wrapping it into an sklearn-compatible estimator +# max_time, so here I'm wrapping it into an sklearn-compatible estimator # so gridsearch will work. # class BingoSymbolicRegressor(SymbolicRegressor, RegressorMixin): @@ -88,7 +96,7 @@ # self.best_ind_ = min(self.est_.best_pop_, key=lambda x: x.fitness) # return self - + # def get_params(self, deep=True): # out = dict() # for (key, value) in self.__dict__.items(): @@ -144,10 +152,12 @@ # ) population_size = [100, 500, 1000] -operators=[["+", "-", "*", "/", "sin", "cos", "exp", "log", "sqrt"], - ["+", "-", "*", "/", "log", "sqrt"]] +operators = [ + ["+", "-", "*", "/", "sin", "cos", "exp", "log", "sqrt"], + ["+", "-", "*", "/", "log", "sqrt"], +] -hyper_params = [] +hyper_params: list[dict[str, Any]] = [] # for p in population_size: # for ops in operators: # hyper_params.append({ @@ -169,4 +179,4 @@ max_evals=int(1e19), evolutionary_algorithm="AgeFitnessEA", clo_threshold=1.0e-5, -) \ No newline at end of file +) diff --git a/experiment/methods/brush/regressor.py b/experiment/methods/brush/regressor.py index 837c5228..57ab82fd 100644 --- a/experiment/methods/brush/regressor.py +++ b/experiment/methods/brush/regressor.py @@ -1,5 +1,6 @@ -from pybrush import BrushRegressor +from __future__ import annotations +from pybrush import BrushRegressor kwargs = { "verbosity": 1, @@ -10,7 +11,6 @@ "initialization": "uniform", "validation_size": 0.33, "cx_prob": 1 / 7, - "weights_init": False, "mutation_probs": { "point": 1 / 6, @@ -20,16 +20,13 @@ "toggle_weight_on": 1 / 6, "toggle_weight_off": 1 / 6, }, - "sel": "lexicase", "algorithm": "nsga2", "objectives": ["scorer", "complexity"], "scorer": "mse", - "bandit": "dynamic_thompson", "num_islands": 1, "shuffle_split": True, - "functions": [ # Arithmetic "Add", @@ -37,7 +34,6 @@ "Mul", "Div", "Pow", - # Unary functions "Sin", "Cos", @@ -45,11 +41,9 @@ "Exp", "Log", "Sqrt", - # Optional split operators "SplitBest", "SplitOn", - # Terminals "Constant", "Terminal", @@ -73,7 +67,6 @@ "Div": 2, "Add": 2, "Pow": 2, - "Sin": 1, "Cos": 1, "Tanh": 1, @@ -90,7 +83,7 @@ def pretify_expr(string, feature_names): tokens = string.replace(" ", "").replace(")", "").replace("(", ",").split(",") new_string = "" - stack = [] + stack: list[tuple[str, list[str]]] = [] for token in tokens: stack.append((token, [])) @@ -149,4 +142,4 @@ def complexity(estimator): if hasattr(estimator, "fitness"): return estimator.fitness.size - raise AttributeError("Estimator does not expose model complexity.") \ No newline at end of file + raise AttributeError("Estimator does not expose model complexity.") diff --git a/experiment/methods/bsr/regressor.py b/experiment/methods/bsr/regressor.py index e764ab0b..563ccbd2 100644 --- a/experiment/methods/bsr/regressor.py +++ b/experiment/methods/bsr/regressor.py @@ -1,33 +1,38 @@ from bsr.bsr_class import BSR hyper_params = [] -for val, itrNum in zip([50,100,250],[250,100,50]): - for treeNum in [2,3]: +for val, itrNum in zip([50, 100, 250], [250, 100, 50]): + for treeNum in [2, 3]: hyper_params.append( - {'treeNum': [treeNum], - 'itrNum': [itrNum], - 'val': [val], - }) + { + "treeNum": [treeNum], + "itrNum": [itrNum], + "val": [val], + } + ) # initialize est = BSR( - val=100, - itrNum=500, - treeNum=3, - alpha1= 0.4, - alpha2= 0.4, - beta= -1, - disp=False, - max_time=60*60) + val=100, + itrNum=500, + treeNum=3, + alpha1=0.4, + alpha2=0.4, + beta=-1, + disp=False, + max_time=60 * 60, +) + def complexity(est): return est.complexity() + def model(est): model_str = est.model() # get rid of square brackets - new_model_str = model_str.replace('[','').replace(']','') + new_model_str = model_str.replace("[", "").replace("]", "") # BSR uses caret notation for powers; SymPy parses ^ as XOR here. - new_model_str = new_model_str.replace('^', '**') - + new_model_str = new_model_str.replace("^", "**") + return new_model_str diff --git a/experiment/methods/e2et/regressor.py b/experiment/methods/e2et/regressor.py index 36ccdc62..8f0a1ed6 100644 --- a/experiment/methods/e2et/regressor.py +++ b/experiment/methods/e2et/regressor.py @@ -1,55 +1,50 @@ -import torch -import os, sys -import symbolicregression +import os + import requests -import numexpr +import symbolicregression +import torch model_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "/pretrained/model.pt" + os.path.dirname(os.path.abspath(__file__)), "/pretrained/model.pt" ) try: - if not os.path.isfile(model_path): + if not os.path.isfile(model_path): url = "https://dl.fbaipublicfiles.com/symbolicregression/model1.pt" r = requests.get(url, allow_redirects=True) - open(model_path, 'wb').write(r.content) + open(model_path, "wb").write(r.content) if not torch.cuda.is_available(): - model = torch.load(model_path, map_location=torch.device('cpu')) + pretrained_model = torch.load(model_path, map_location=torch.device("cpu")) else: - model = torch.load(model_path) - model = model.cuda() - print(model.device) + pretrained_model = torch.load(model_path) + pretrained_model = pretrained_model.cuda() + print(pretrained_model.device) print("Model successfully loaded!") except Exception as e: print("ERROR: model not loaded! path was: {}".format(model_path)) - print(e) + print(e) est = symbolicregression.model.SymbolicTransformerRegressor( - model=model, - max_input_points=200, - n_trees_to_refine=100, - rescale=True - ) + model=pretrained_model, max_input_points=200, n_trees_to_refine=100, rescale=True +) + def model(est, X=None): replace_ops = {"add": "+", "mul": "*", "sub": "-", "pow": "**", "inv": "1/"} model_str = est.retrieve_tree(dataset_idx=0).infix() - for op,replace_op in replace_ops.items(): - model_str = model_str.replace(op,replace_op) + for op, replace_op in replace_ops.items(): + model_str = model_str.replace(op, replace_op) return model_str + def my_pre_train_fn(est, X, y): - """In this example we adjust FEAT generations based on the size of X - versus relative to FEAT's batch size setting. + """In this example we adjust FEAT generations based on the size of X + versus relative to FEAT's batch size setting. """ return + # define eval_kwargs. eval_kwargs = dict( - test_params = { - 'max_input_points':100, - 'n_trees_to_refine':10 - }, - use_dataframe=False - ) + test_params={"max_input_points": 100, "n_trees_to_refine": 10}, use_dataframe=False +) diff --git a/experiment/methods/eplex/regressor.py b/experiment/methods/eplex/regressor.py index 35f91d49..8841edde 100644 --- a/experiment/methods/eplex/regressor.py +++ b/experiment/methods/eplex/regressor.py @@ -3,45 +3,45 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','sin','cos','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "sin", "cos", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], +] hyper_params = [] for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='epsilon_lexicase', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=500, - g=500, - time_limit=2*60*60 - ) +est = ellyn( + selection="epsilon_lexicase", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=500, + g=500, + time_limit=2 * 60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): return est.stack_2_eqn(est.best_estimator_) -eval_kwargs = dict(use_dataframe=False) \ No newline at end of file + +eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/eql/regressor.py b/experiment/methods/eql/regressor.py index c9a78168..3b511792 100644 --- a/experiment/methods/eql/regressor.py +++ b/experiment/methods/eql/regressor.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 -from eql.est import EQL import pandas as pd -from sklearn.model_selection import GridSearchCV - +from eql.est import EQL base = EQL(n_iter=10_000) @@ -10,33 +8,30 @@ n_layers = (1, 2) functions = [ "id;mul;cos;sin;exp;square;sqrt;id;mul;cos;sin;exp;square;sqrt;log", - "id;mul;cos;div;sqrt;cos;sin;div;mul;mul;cos;id;log" + "id;mul;cos;div;sqrt;cos;sin;div;mul;mul;cos;id;log", ] hyper_params = [] -for r, l in zip(reg, n_layers): +for r, layers in zip(reg, n_layers): for f in functions: - hyper_params.append({ - 'reg':[r], - 'n_layers':[l], - 'functions':[f] - }) - + hyper_params.append({"reg": [r], "n_layers": [layers], "functions": [f]}) + # est = GridSearchCV(estimator=base, param_grid=hyper_params, cv=2, refit=True, n_jobs=4, verbose=0) est = base + def model(est, X=None): model_str = str(est.get_eqn()) if isinstance(X, pd.DataFrame): - for i,f in reversed(list(enumerate(X.columns))): - model_str = model_str.replace(f'x{i}',f) + for i, f in reversed(list(enumerate(X.columns))): + model_str = model_str.replace(f"x{i}", f) return model_str eval_kwargs = { # 'test_params': {'n_iter': [10], **hyper_params[-1]}, - 'use_dataframe':False + "use_dataframe": False } diff --git a/experiment/methods/experimental/GSGPRegressor.py b/experiment/methods/experimental/GSGPRegressor.py index 5371e90b..34fd65ea 100755 --- a/experiment/methods/experimental/GSGPRegressor.py +++ b/experiment/methods/experimental/GSGPRegressor.py @@ -1,45 +1,58 @@ from .src.gsgp import GSGPRegressor - hyper_params = [ { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.0,),'rt_mut':(1.0,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.0,), + "rt_mut": (1.0,), }, { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.1,),'rt_mut':(0.9,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.1,), + "rt_mut": (0.9,), }, { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.2,),'rt_mut':(0.8,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.2,), + "rt_mut": (0.8,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.0,),'rt_mut':(1.0,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.0,), + "rt_mut": (1.0,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.1,),'rt_mut':(0.9,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.1,), + "rt_mut": (0.9,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.2,),'rt_mut':(0.8,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.2,), + "rt_mut": (0.8,), }, ] -est=GSGPRegressor() +est = GSGPRegressor() + -#TODO: define complexity +# TODO: define complexity def complexity(est): return -1 + model = None model = None diff --git a/experiment/methods/feat/regressor.py b/experiment/methods/feat/regressor.py index 5162187b..32ded796 100644 --- a/experiment/methods/feat/regressor.py +++ b/experiment/methods/feat/regressor.py @@ -1,45 +1,33 @@ -# This example submission shows the submission of FEAT (cavalab.org/feat). -from feat import FeatRegressor -from sklearn.base import BaseEstimator, RegressorMixin -from feat import Feat, FeatRegressor, FeatClassifier +# This example submission shows the submission of FEAT (cavalab.org/feat). -from sklearn.datasets import load_diabetes, make_blobs -from sklearn.base import clone -from sklearn.pipeline import make_pipeline -from sklearn.metrics.pairwise import rbf_kernel -from sklearn.metrics import r2_score -from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin -import unittest -import argparse -import sys -import pandas as pd import numpy as np -import pickle - +from feat import FeatRegressor +from sklearn.base import RegressorMixin +from sklearn.metrics import r2_score """ est: a sklearn-compatible regressor. if you don't have one they are fairly easy to create. see https://scikit-learn.org/stable/developers/develop.html """ -est:RegressorMixin = FeatRegressor( - pop_size=100, - gens=100, - max_time=8*60*60, # 8 hrs. Your algorithm should have this feature - max_depth=6, - verbosity=2, - batch_size=100, - functions=['+','-','*','/','^2','^3','sqrt','sin','cos','exp','log'], - otype='f' - ) -# want to tune your estimator? wrap it in a sklearn CV class. +est: RegressorMixin = FeatRegressor( + pop_size=100, + gens=100, + max_time=8 * 60 * 60, # 8 hrs. Your algorithm should have this feature + max_depth=6, + verbosity=2, + batch_size=100, + functions=["+", "-", "*", "/", "^2", "^3", "sqrt", "sin", "cos", "exp", "log"], + otype="f", +) +# want to tune your estimator? wrap it in a sklearn CV class. class FeatPopEstimator(RegressorMixin): """ FeatPopEstimator is a custom regressor that wraps a fitted FEAT estimator to call `model` and `predict` from its archive. - + Attributes: est (object): The fitted FEAT estimator. id (int): The identifier for the specific model in the estimator's archive. @@ -57,57 +45,59 @@ class FeatPopEstimator(RegressorMixin): model(): Retrieves the model equation from the estimator's archive. """ + def __init__(self, est, id): self.est = est - self.id = id + self.id = id def fit(self, X, y): self.is_fitted_ = True def predict(self, X): - + X = self.est._prep_X(X) return self.est.cfeat_.predict_archive(self.id, X) def score(self, X, y): yhat = self.predict(X).flatten() - return r2_score(y,yhat) - + return r2_score(y, yhat) + def model(self): archive = self.est.cfeat_.get_archive(False) - ind = [i for i in archive if i['id']==self.id][0] + ind = [i for i in archive if i["id"] == self.id][0] eqn = f"{np.round(ind['ml']['bias'], 5)}" - for eq, w in zip(ind['eqn'].replace('[', '').split(']'), ind['w']): - if str(w)[0]=='-': - eqn = eqn + f'{np.round(float(w), 2)}*{eq}' + for eq, w in zip(ind["eqn"].replace("[", "").split("]"), ind["w"]): + if str(w)[0] == "-": + eqn = eqn + f"{np.round(float(w), 2)}*{eq}" else: - eqn = eqn + f'+{np.round(float(w), 2)}*{eq}' + eqn = eqn + f"+{np.round(float(w), 2)}*{eq}" return eqn + def model(est, X=None) -> str: """ - Return a sympy-compatible string of the final model. + Return a sympy-compatible string of the final model. Parameters ---------- est: sklearn regressor - The fitted model. + The fitted model. X: pd.DataFrame, default=None The training data. This argument can be dropped if desired. Returns ------- - A sympy-compatible string of the final model. + A sympy-compatible string of the final model. Notes ----- - Ensure that the variable names appearing in the model are identical to - those in the training data, `X`, which is a `pd.Dataframe`. - If your method names variables some other way, e.g. `[x_0 ... x_m]`, + Ensure that the variable names appearing in the model are identical to + those in the training data, `X`, which is a `pd.Dataframe`. + If your method names variables some other way, e.g. `[x_0 ... x_m]`, you can specify a mapping in the `model` function such as: ``` @@ -119,9 +109,9 @@ def model(est, X): ``` If you have special operators such as protected division or protected log, - you will need to handle these to assure they conform to sympy format. + you will need to handle these to assure they conform to sympy format. One option is to replace them with the unprotected versions. Post an issue - if you have further questions: + if you have further questions: https://github.com/cavalab/srbench/issues/new/choose """ @@ -132,20 +122,20 @@ def model(est, X): model_str = est.cfeat_.get_eqn() # Here we replace "|" with "" to handle - # protected sqrt (expressed as sqrt(|.|)) in FEAT) - model_str = model_str.replace('|','') + # protected sqrt (expressed as sqrt(|.|)) in FEAT) + model_str = model_str.replace("|", "") # use python syntax for exponents - model_str = model_str.replace('^','**') + model_str = model_str.replace("^", "**") return model_str def complexity(est): if isinstance(est, FeatPopEstimator): - est = est.est # It is wrapped + est = est.est # It is wrapped - return est.get_n_nodes() + 2*est.get_n_params() + 2*est.get_dim() + return est.get_n_nodes() + 2 * est.get_n_params() + 2 * est.get_dim() def get_population(est) -> list[RegressorMixin]: @@ -156,7 +146,7 @@ def get_population(est) -> list[RegressorMixin]: Also, it is expected that the `model()` function can operate with them, so they should have a way of getting a simpy string representation. - + Returns ------- A list of scikit-learn compatible estimators that can be used for prediction. @@ -170,21 +160,19 @@ def get_population(est) -> list[RegressorMixin]: # archive contains individuals serialized in json objects. let's get their ids for ind in archive: # Archive is sorted by complexity - pop.append( - FeatPopEstimator(est, ind['id']) - ) + pop.append(FeatPopEstimator(est, ind["id"])) # Stopping here to avoid too many models if len(pop) >= 100: break - return pop + def get_best_solution(est) -> RegressorMixin: """ - Return the best solution from the final model. - + Return the best solution from the final model. + Returns ------- A scikit-learn compatible estimator @@ -224,21 +212,18 @@ def get_best_solution(est) -> RegressorMixin: y: training labels. """ + def my_pre_train_fn(est, X, y): - """In this example we adjust FEAT generations based on the size of X - versus relative to FEAT's batch size setting. + """In this example we adjust FEAT generations based on the size of X + versus relative to FEAT's batch size setting. """ if est.batch_size < len(X): - est.gens = int(est.gens*len(X)/est.batch_size) - print('FEAT gens adjusted to',est.gens) + est.gens = int(est.gens * len(X) / est.batch_size) + print("FEAT gens adjusted to", est.gens) # adjust max dim - est.max_dim=min(max(est.max_dim, X.shape[1]), 20) - print('FEAT max_dim set to',est.max_dim) + est.max_dim = min(max(est.max_dim, X.shape[1]), 20) + print("FEAT max_dim set to", est.max_dim) + # define eval_kwargs. -eval_kwargs = dict( - pre_train=my_pre_train_fn, - test_params = {'gens': 5, - 'pop_size': 10 - } - ) +eval_kwargs = dict(pre_train=my_pre_train_fn, test_params={"gens": 5, "pop_size": 10}) diff --git a/experiment/methods/ffx/regressor.py b/experiment/methods/ffx/regressor.py index fceb55d6..5e74e96b 100644 --- a/experiment/methods/ffx/regressor.py +++ b/experiment/methods/ffx/regressor.py @@ -1,15 +1,20 @@ +from __future__ import annotations + +from typing import Any + from ffx import FFXRegressor # FFX has no parameters! -hyper_params = [ - { - }, +hyper_params: list[dict[str, Any]] = [ + {}, ] est = FFXRegressor() + def complexity(est): return est.model_.complexity() + def model(est, X=None): - return est.model_.str2().replace('^', '**') + return est.model_.str2().replace("^", "**") diff --git a/experiment/methods/geneticengine/regressor.py b/experiment/methods/geneticengine/regressor.py index cd5890a2..5ec0e18d 100644 --- a/experiment/methods/geneticengine/regressor.py +++ b/experiment/methods/geneticengine/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import GeneticProgrammingRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = GeneticProgrammingRegressor( - max_time=60*60, # 8 Your algorithm should have this feature + max_time=60 * 60, # 8 Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_1p1/regressor.py b/experiment/methods/geneticengine_1p1/regressor.py index 8ba80594..55402a4c 100644 --- a/experiment/methods/geneticengine_1p1/regressor.py +++ b/experiment/methods/geneticengine_1p1/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import RandomSearchRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = RandomSearchRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_hc/regressor.py b/experiment/methods/geneticengine_hc/regressor.py index 4e58916b..353598a8 100644 --- a/experiment/methods/geneticengine_hc/regressor.py +++ b/experiment/methods/geneticengine_hc/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import HillClimbingRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = HillClimbingRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_rs/regressor.py b/experiment/methods/geneticengine_rs/regressor.py index 6db0995d..c14a8df6 100644 --- a/experiment/methods/geneticengine_rs/regressor.py +++ b/experiment/methods/geneticengine_rs/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import OnePlusOneRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = OnePlusOneRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/gp-elite/regressor.py b/experiment/methods/gp-elite/regressor.py index e0265422..04b6f68b 100644 --- a/experiment/methods/gp-elite/regressor.py +++ b/experiment/methods/gp-elite/regressor.py @@ -3,13 +3,17 @@ Exposes the three objects the harness reads: `est`, `hyper_params`, `eval_kwargs`, plus `model(est, X)` and `complexity(est)`. """ + +from __future__ import annotations + import time +from typing import Any import numpy as np try: from sklearn.utils.validation import validate_data -except Exception: # older sklearn +except Exception: # older sklearn validate_data = None from gp_elite import GPEliteRegressor @@ -24,20 +28,37 @@ class GPEliteSRBench(GPEliteRegressor): spent or `restarts` is reached. """ - def __init__(self, operators="physical", normalize="auto", generations=60, - speed="fast", validation_split=0.20, max_time=3600, - restarts=10, robust=False, parallel=None, random_state=0): - super().__init__(operators=operators, normalize=normalize, - generations=generations, speed=speed, - validation_split=validation_split, restarts=restarts, - robust=robust, parallel=parallel, - random_state=random_state) + def __init__( + self, + operators="physical", + normalize="auto", + generations=60, + speed="fast", + validation_split=0.20, + max_time=3600, + restarts=10, + robust=False, + parallel=None, + random_state=0, + ): + super().__init__( + operators=operators, + normalize=normalize, + generations=generations, + speed=speed, + validation_split=validation_split, + restarts=restarts, + robust=robust, + parallel=parallel, + random_state=random_state, + ) self.max_time = max_time def fit(self, X, y): if validate_data is not None: - X, y = validate_data(self, X, y, y_numeric=True, - ensure_min_samples=2, dtype="numeric") + X, y = validate_data( + self, X, y, y_numeric=True, ensure_min_samples=2, dtype="numeric" + ) else: X = np.asarray(X, dtype=float) y = np.asarray(y, dtype=float) @@ -54,12 +75,19 @@ def fit(self, X, y): for k in range(max(1, int(self.restarts))): r = symbolic_regression( - X, y, feature_names=names, - operators=self.operators, normalize=self.normalize, - generations=self.generations, speed=self.speed, - validation_split=self.validation_split, restarts=1, - robust=self.robust, parallel=self.parallel, - seed=seed0 + 1000 * k) + X, + y, + feature_names=names, + operators=self.operators, + normalize=self.normalize, + generations=self.generations, + speed=self.speed, + validation_split=self.validation_split, + restarts=1, + robust=self.robust, + parallel=self.parallel, + seed=seed0 + 1000 * k, + ) score = r.r2_validation if score is None or not np.isfinite(score): p = r.predict(X) @@ -92,15 +120,20 @@ def fit(self, X, y): # ── the three objects SRBench reads ───────────────────────────────────────── -est = GPEliteSRBench(operators="physical", generations=60, speed="fast", - restarts=10, max_time=3600, random_state=0) +est = GPEliteSRBench( + operators="physical", + generations=60, + speed="fast", + restarts=10, + max_time=3600, + random_state=0, +) hyper_params = [ - {"operators": ("physical", "full"), - "generations": (30, 60)}, + {"operators": ("physical", "full"), "generations": (30, 60)}, ] -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} # Map every GP_ELITE operator to a sympy-parsable form. Anything produced by the @@ -108,27 +141,27 @@ def fit(self, X, y): # as opaque undefined functions (e.g. cube(x) is NOT x**3 to sympy) and break # numeric evaluation downstream. _UNARY = { - "neg": lambda a: "(-(%s))" % a, - "abs": lambda a: "Abs(%s)" % a, - "inv": lambda a: "(1/(%s))" % a, - "sq": lambda a: "((%s)**2)" % a, - "cube": lambda a: "((%s)**3)" % a, - "sqrt": lambda a: "sqrt(Abs(%s))" % a, # engine's sqrt is domain-guarded - "log": lambda a: "log(Abs(%s))" % a, # engine's log is domain-guarded - "exp": lambda a: "exp(%s)" % a, - "sin": lambda a: "sin(%s)" % a, - "cos": lambda a: "cos(%s)" % a, - "tan": lambda a: "tan(%s)" % a, - "tanh": lambda a: "tanh(%s)" % a, - "step": lambda a: "Heaviside(%s)" % a, + "neg": lambda a: "(-(%s))" % a, + "abs": lambda a: "Abs(%s)" % a, + "inv": lambda a: "(1/(%s))" % a, + "sq": lambda a: "((%s)**2)" % a, + "cube": lambda a: "((%s)**3)" % a, + "sqrt": lambda a: "sqrt(Abs(%s))" % a, # engine's sqrt is domain-guarded + "log": lambda a: "log(Abs(%s))" % a, # engine's log is domain-guarded + "exp": lambda a: "exp(%s)" % a, + "sin": lambda a: "sin(%s)" % a, + "cos": lambda a: "cos(%s)" % a, + "tan": lambda a: "tan(%s)" % a, + "tanh": lambda a: "tanh(%s)" % a, + "step": lambda a: "Heaviside(%s)" % a, "is_even": lambda a: "(1 - Mod(floor(%s), 2))" % a, } _BINARY = { - "+": lambda a, b: "((%s) + (%s))" % (a, b), - "-": lambda a, b: "((%s) - (%s))" % (a, b), - "*": lambda a, b: "((%s) * (%s))" % (a, b), - "/": lambda a, b: "((%s) / (%s))" % (a, b), - "pow": lambda a, b: "((%s)**(%s))" % (a, b), + "+": lambda a, b: "((%s) + (%s))" % (a, b), + "-": lambda a, b: "((%s) - (%s))" % (a, b), + "*": lambda a, b: "((%s) * (%s))" % (a, b), + "/": lambda a, b: "((%s) / (%s))" % (a, b), + "pow": lambda a, b: "((%s)**(%s))" % (a, b), "max2": lambda a, b: "Max((%s), (%s))" % (a, b), "min2": lambda a, b: "Min((%s), (%s))" % (a, b), } @@ -143,16 +176,17 @@ def _to_sympy(node, cols): # test the numpy hierarchy explicitly — otherwise an integer-valued # numpy constant would fall through to the name branch and be emitted # as a feature name instead of a number. - if isinstance(v, (int, float, np.integer, np.floating)) \ - and not isinstance(v, bool): + if isinstance(v, (int, float, np.integer, np.floating)) and not isinstance( + v, bool + ): return "%.12g" % float(v) - s = str(v) # like 'X[3]' or 'X3' + s = str(v) # like 'X[3]' or 'X3' digits = "".join(c for c in s if c.isdigit()) i = int(digits) if digits else 0 return cols[i] if i < len(cols) else "x%d" % i op = node.value - if node.right is None: # unary + if node.right is None: # unary a = _to_sympy(node.left, cols) f = _UNARY.get(op) if f is None: @@ -161,22 +195,24 @@ def _to_sympy(node, cols): a = _to_sympy(node.left, cols) b = _to_sympy(node.right, cols) - f = _BINARY.get(op) - if f is None: + binary_fn = _BINARY.get(op) + if binary_fn is None: raise ValueError("unmapped binary operator %r in _to_sympy" % op) - return f(a, b) + return binary_fn(a, b) def model(est, X=None): - cols = list(X.columns) if X is not None and hasattr(X, "columns") \ + cols = ( + list(X.columns) + if X is not None and hasattr(X, "columns") else ["x%d" % i for i in range(getattr(est, "n_features_in_", 0))] + ) # Fold the input normalization into the string so the formula, evaluated # on RAW columns, reproduces est.predict exactly. The engine's shift-free # scaler divides each feature by scale_[i]. scaler = getattr(est.model_, "scaler", None) if scaler is not None and hasattr(scaler, "scale_"): - cols = ["((%s)/%.12g)" % (c, float(f)) - for c, f in zip(cols, scaler.scale_)] + cols = ["((%s)/%.12g)" % (c, float(f)) for c, f in zip(cols, scaler.scale_)] return _to_sympy(est.model_.node, cols) diff --git a/experiment/methods/gpgomea/regressor.py b/experiment/methods/gpgomea/regressor.py index 0f9e8e2c..07bf3e60 100644 --- a/experiment/methods/gpgomea/regressor.py +++ b/experiment/methods/gpgomea/regressor.py @@ -1,7 +1,5 @@ -from sklearn.base import RegressorMixin - from pymgpg.sk import MGPGRegressor - +from sklearn.base import RegressorMixin est: RegressorMixin = MGPGRegressor( verbose=False, diff --git a/experiment/methods/gplearn/regressor.py b/experiment/methods/gplearn/regressor.py index 2b7c3564..0fa2c4a0 100644 --- a/experiment/methods/gplearn/regressor.py +++ b/experiment/methods/gplearn/regressor.py @@ -1,40 +1,40 @@ -from gplearn.genetic import SymbolicRegressor import re +from gplearn.genetic import SymbolicRegressor + hyper_params = [] -for p, g in zip([1000,500,100],[500,1000,5000]): - for fs in [('add', 'sub', 'mul', 'div', 'log','sqrt'), - ('add', 'sub', 'mul', 'div', 'log','sqrt', 'sin','cos') - ]: - hyper_params.append({ - 'population_size' : [p], - 'generations' : [g], - 'function_set': [fs] - }) +for p, g in zip([1000, 500, 100], [500, 1000, 5000]): + for fs in [ + ("add", "sub", "mul", "div", "log", "sqrt"), + ("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos"), + ]: + hyper_params.append( + {"population_size": [p], "generations": [g], "function_set": [fs]} + ) est = SymbolicRegressor( - tournament_size=20, - init_depth=(2, 6), - init_method='half and half', - metric='mean absolute error', - parsimony_coefficient=0.001, - p_crossover=0.9, - p_subtree_mutation=0.01, - p_hoist_mutation=0.01, - p_point_mutation=0.01, - p_point_replace=0.05, - max_samples=1.0, - function_set= ('add', 'sub', 'mul', 'div', 'log', - 'sqrt', 'sin','cos'), - population_size=1000, - generations=500 - ) + tournament_size=20, + init_depth=(2, 6), + init_method="half and half", + metric="mean absolute error", + parsimony_coefficient=0.001, + p_crossover=0.9, + p_subtree_mutation=0.01, + p_hoist_mutation=0.01, + p_point_mutation=0.01, + p_point_replace=0.05, + max_samples=1.0, + function_set=("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos"), + population_size=1000, + generations=500, +) def model(est, X=None): # print(dir(est)) return str(est._program) + def complexity(est): - #TODO: check - return len(re.split(r'\(|,',model(est))) + # TODO: check + return len(re.split(r"\(|,", model(est))) diff --git a/experiment/methods/gpzgd/regressor.py b/experiment/methods/gpzgd/regressor.py index 6cb268f5..7583e94d 100644 --- a/experiment/methods/gpzgd/regressor.py +++ b/experiment/methods/gpzgd/regressor.py @@ -13,24 +13,38 @@ """ -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_X_y, check_array, check_is_fitted -from sklearn.metrics import mean_squared_error, r2_score - import os -from tempfile import TemporaryDirectory import subprocess +from tempfile import TemporaryDirectory + import numpy as np import pandas as pd +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.metrics import mean_squared_error, r2_score +from sklearn.utils.validation import check_array, check_is_fitted, check_X_y + class GPZGD(BaseEstimator, RegressorMixin): - def __init__(self, pop_size=200, generations=250, - tournament_size=3, validation_prop=0.0, - crossover_rate=0.3, sub_mutation_rate=0.4, point_mutation_rate=0.3, mutation_sigma=0.1, - min_tree_init=2, max_tree_init=4, max_tree_nodes=50, - opset="ADD,SUB,MUL,SIN,ERC,VAR", - learning_rate=0.01, learning_epochs=3, timeout=0, random_state=-1): - """ Builds a Symbolic Regression using the cli interface of your algorithm. + def __init__( + self, + pop_size=200, + generations=250, + tournament_size=3, + validation_prop=0.0, + crossover_rate=0.3, + sub_mutation_rate=0.4, + point_mutation_rate=0.3, + mutation_sigma=0.1, + min_tree_init=2, + max_tree_init=4, + max_tree_nodes=50, + opset="ADD,SUB,MUL,SIN,ERC,VAR", + learning_rate=0.01, + learning_epochs=3, + timeout=0, + random_state=-1, + ): + """Builds a Symbolic Regression using the cli interface of your algorithm. Examples -------- >>> from gpzgd import GPZGD @@ -42,7 +56,7 @@ def __init__(self, pop_size=200, generations=250, """ self.validation_prop = validation_prop - + self.pop_size = pop_size self.generations = generations self.tournament_size = tournament_size @@ -55,7 +69,7 @@ def __init__(self, pop_size=200, generations=250, self.max_tree_init = max_tree_init self.max_tree_nodes = max_tree_nodes self.learning_rate = learning_rate - self.learning_epochs =learning_epochs + self.learning_epochs = learning_epochs self.timeout = timeout self.random_state = random_state @@ -79,12 +93,12 @@ def fit(self, X_train, y_train): # 2. validate the consistency of the data matrices and create a single 2D array with X and y X_train, y_train = check_X_y(X_train, y_train, accept_sparse=False) if len(y_train.shape) == 1: - Z_train = np.hstack((X_train, y_train[:,None])) + Z_train = np.hstack((X_train, y_train[:, None])) else: Z_train = np.hstack((X_train, y_train)) # 3. create a temp config file - cname = temp_dir + "/config" + cname = temp_dir + "/config" with open(cname, "w") as config_file: config_file.write(f"timeout={self.timeout}\n") config_file.write(f"validation_prop={self.validation_prop}\n") @@ -101,41 +115,63 @@ def fit(self, X_train, y_train): config_file.write(f"learning_rate={self.learning_rate}\n") config_file.write(f"learning_epochs={self.learning_epochs}\n") config_file.write(f"opset={self.opset}\n") - config_file.write(f"elitism_rate=1\n") - config_file.write(f"max_tree_depth=-1\n") - config_file.write(f"standardise=Y\n") - config_file.write(f"coef_op=Y\n") - + config_file.write("elitism_rate=1\n") + config_file.write("max_tree_depth=-1\n") + config_file.write("standardise=Y\n") + config_file.write("coef_op=Y\n") + # 4. create a temp file and store the data - fname = temp_dir + "/tmpdata" - np.savetxt(f"{fname}", Z_train, delimiter=" ", header=f"{Z_train.shape[0]} {Z_train.shape[1]}", comments="") + fname = temp_dir + "/tmpdata" + np.savetxt( + f"{fname}", + Z_train, + delimiter=" ", + header=f"{Z_train.shape[0]} {Z_train.shape[1]}", + comments="", + ) # 5. call your cli binary with the parameters cwd = os.path.dirname(os.path.realpath(__file__)) if self.random_state >= 0: - ans = subprocess.check_output([ "gpzgd_regressor", f"{fname}", f"{cname}", "-p", f"rng_seed={self.random_state}" ], cwd=cwd, universal_newlines=True) + ans = subprocess.check_output( + [ + "gpzgd_regressor", + f"{fname}", + f"{cname}", + "-p", + f"rng_seed={self.random_state}", + ], + cwd=cwd, + universal_newlines=True, + ) else: - ans = subprocess.check_output([ "gpzgd_regressor", f"{fname}", f"{cname}" ], cwd=cwd, universal_newlines=True) - - xbar, s, mdl, l, e = ans.split(";") - - self.xbar = eval(xbar) - self.s = eval(s) - self.expr = mdl - self.len = int(l) - self.cli_mse = float(e) - self.train_mse = mean_squared_error(y_train, self.eval_expr(X_train)) ## useful to compare with internal score! - self.score = r2_score(y_train, self.eval_expr(X_train)) - + ans = subprocess.check_output( + ["gpzgd_regressor", f"{fname}", f"{cname}"], + cwd=cwd, + universal_newlines=True, + ) + + xbar, s, mdl, length, e = ans.split(";") + + self.xbar = eval(xbar) + self.s = eval(s) + self.expr = mdl + self.len = int(length) + self.cli_mse = float(e) + self.train_mse = mean_squared_error( + y_train, self.eval_expr(X_train) + ) ## useful to compare with internal score! + self.score = r2_score(y_train, self.eval_expr(X_train)) + self.is_fitted_ = True return self def expr_str(self): return self.expr - + def eval_expr(self, X): - """ Evaluates the expression with data point x. We assume that the expression is compatible with numpy """ + """Evaluates the expression with data point x. We assume that the expression is compatible with numpy""" X = (X - self.xbar) / self.s y = eval(self.expr) @@ -145,7 +181,7 @@ def eval_expr(self, X): return y def predict(self, X_test, ic=None): - """ A reference implementation of a predicting function. + """A reference implementation of a predicting function. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) @@ -161,31 +197,33 @@ def predict(self, X_test, ic=None): ##### TODO: Reverse Transform the predictions return self.eval_expr(X_test) + est = GPZGD() -# want to tune your estimator? wrap it in a sklearn CV class. +# want to tune your estimator? wrap it in a sklearn CV class. + def model(est, X=None): """ - Return a sympy-compatible string of the final model. + Return a sympy-compatible string of the final model. Parameters ---------- est: sklearn regressor - The fitted model. + The fitted model. X: pd.DataFrame, default=None The training data. This argument can be dropped if desired. Returns ------- - A sympy-compatible string of the final model. + A sympy-compatible string of the final model. Notes ----- - Ensure that the variable names appearing in the model are identical to - those in the training data, `X`, which is a `pd.Dataframe`. - If your method names variables some other way, e.g. `[x_0 ... x_m]`, + Ensure that the variable names appearing in the model are identical to + those in the training data, `X`, which is a `pd.Dataframe`. + If your method names variables some other way, e.g. `[x_0 ... x_m]`, you can specify a mapping in the `model` function such as: ``` @@ -197,26 +235,27 @@ def model(est, X): ``` If you have special operators such as protected division or protected log, - you will need to handle these to assure they conform to sympy format. + you will need to handle these to assure they conform to sympy format. One option is to replace them with the unprotected versions. Post an issue - if you have further questions: + if you have further questions: https://github.com/cavalab/srbench/issues/new/choose """ model_str = est.expr_str() - model_str = model_str.replace("np.","").replace("[:,", "").replace("]", "") + model_str = model_str.replace("np.", "").replace("[:,", "").replace("]", "") # use python syntax for exponents - model_str = model_str.replace('^','**') + model_str = model_str.replace("^", "**") names = X.columns.tolist() if isinstance(X, pd.DataFrame) else None if names is not None: - mapping = { 'X'+str(i):k for i, k in enumerate(names) } + mapping = {"X" + str(i): k for i, k in enumerate(names)} for k, v in reversed(mapping.items()): - model_str = model_str.replace(k,v) + model_str = model_str.replace(k, v) return model_str + ################################################################################ # Optional Settings ################################################################################ @@ -248,22 +287,22 @@ def model(est, X): y: training labels. """ + def pre_train_fn(est, X, y): if len(X) <= 1000: pop_size = 500 generations = 5000 - max_time = 3564 ## 99% of one hour + max_time = 3564 ## 99% of one hour else: pop_size = 200 generations = 5000 - max_time = 35640 ## 99% of ten hours + max_time = 35640 ## 99% of ten hours est.tournament_size = 10 est.pop_size = pop_size est.generations = generations est.timeout = max_time - + + # define eval_kwargs. -eval_kwargs = { - "pre_train" : pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} diff --git a/experiment/methods/itea/regressor.py b/experiment/methods/itea/regressor.py index 329f9914..d4cd54fb 100644 --- a/experiment/methods/itea/regressor.py +++ b/experiment/methods/itea/regressor.py @@ -1,83 +1,89 @@ -import sys import os + os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + import pyITEA as itea -from itertools import product hyper_params = [ { - 'exponents' : ((0,5),), - 'termlimit' : ((2,5),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((0, 5),), + "termlimit": ((2, 5),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((0,5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), - 'nonzeroexps' : (10,), + "exponents": ((0, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), + "nonzeroexps": (10,), }, { - 'exponents' : ((-5,5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((-5, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((-5, 5),), - 'termlimit' : ((2, 5),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((-5, 5),), + "termlimit": ((2, 5),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((0, 5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((0, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((-5, 5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), - 'nonzeroexps' : (10,), + "exponents": ((-5, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), + "nonzeroexps": (10,), }, ] def pre_train(est, X, y): """Adjusting ngens to 500 if dataset is too large""" - + # adjust generations based onsize of X versus batch size if len(X) >= 10_000: est.ngens = 250 - print('ITEA ngens adjusted to', est.gens) + print("ITEA ngens adjusted to", est.gens) # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = itea.ITEARegressor(npop=1000, ngens=500, exponents=(-1, 1), termlimit=(2, - 2), nonzeroexps=1, - transfunctions= '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]' - ) +eval_kwargs = {"scale_x": False, "scale_y": False} +est = itea.ITEARegressor( + npop=1000, + ngens=500, + exponents=(-1, 1), + termlimit=(2, 2), + nonzeroexps=1, + transfunctions="[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]", +) + def complexity(e): return e.len + def model(e, X): # get sympy compatible - new_model = e.sympy.replace("^","**") - - new_model = new_model.replace('sqrtAbs','PSQRT') + new_model = e.sympy.replace("^", "**") + + new_model = new_model.replace("sqrtAbs", "PSQRT") + + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) - return new_model diff --git a/experiment/methods/keplearn/regressor.py b/experiment/methods/keplearn/regressor.py index 1b4081ce..29ca2461 100644 --- a/experiment/methods/keplearn/regressor.py +++ b/experiment/methods/keplearn/regressor.py @@ -1,3 +1,16 @@ +from __future__ import annotations + +import json +import math as _math +import os +import re +import subprocess +from typing import Any + +import numpy as np +import sympy as sp +from sklearn.base import BaseEstimator, RegressorMixin + """SRBench wrapper for the Keplearn engine. Keplearn is a deterministic symbolic-regression engine written in Rust, built @@ -12,31 +25,32 @@ as `--timeout `; the engine stops gracefully at that budget and emits best-so-far. """ -import json -import os -import re -import subprocess - -import numpy as np -import sympy as sp -from sklearn.base import BaseEstimator, RegressorMixin # Path to the engine binary (installed by algorithms/keplearn/install.sh). BIN = os.environ.get("KEPLEARN_BIN", "/opt/conda/bin/keplearn") # numpy evaluators matching the engine's emitted grammar (same set as the harness). _NP = { - "sqrt": np.sqrt, "sin": np.sin, "cos": np.cos, "tan": np.tan, - "exp": np.exp, "log": np.log, "arcsin": np.arcsin, "arccos": np.arccos, - "tanh": np.tanh, "abs": np.abs, + "sqrt": np.sqrt, + "sin": np.sin, + "cos": np.cos, + "tan": np.tan, + "exp": np.exp, + "log": np.log, + "arcsin": np.arcsin, + "arccos": np.arccos, + "tanh": np.tanh, + "abs": np.abs, } class KeplearnRegressor(BaseEstimator, RegressorMixin): def __init__(self, max_time=60, sample=500, random_state=None, max_depth=None): - self.max_time = max_time # SRBench MAXTIME (seconds); overwritten by harness + self.max_time = max_time # SRBench MAXTIME (seconds); overwritten by harness self.sample = sample - self.random_state = random_state # engine is deterministic; seeds the holdout shuffle + self.random_state = ( + random_state # engine is deterministic; seeds the holdout shuffle + ) self.max_depth = max_depth def fit(self, X, y): @@ -55,9 +69,15 @@ def fit(self, X, y): tsv = header + "\n" + "\n".join(rows) + "\n" cmd = [ - BIN, "--target", "target", "--top-k", "1", - "--sample", str(int(self.sample)), - "--timeout", str(int(float(self.max_time) * 1000)), # seconds -> ms + BIN, + "--target", + "target", + "--top-k", + "1", + "--sample", + str(int(self.sample)), + "--timeout", + str(int(float(self.max_time) * 1000)), # seconds -> ms ] if self.random_state is not None: cmd += ["--seed", str(int(self.random_state))] @@ -67,12 +87,15 @@ def fit(self, X, y): # grace over the engine's own --timeout so the subprocess wrapper never # pre-empts the engine's graceful best-so-far emit. proc = subprocess.run( - cmd, input=tsv, capture_output=True, text=True, + cmd, + input=tsv, + capture_output=True, + text=True, timeout=float(self.max_time) + 30, ) out = proc.stdout or "" try: - res = json.loads(out[out.index("{"):]) + res = json.loads(out[out.index("{") :]) except Exception: res = {"results": []} results = res.get("results") or [] @@ -86,13 +109,19 @@ def _build_canonical(self): """Build ONE canonical SymPy expression from the engine string; predict, model() and complexity() all use this same object, so the predicting function and the reported model string cannot diverge.""" - raw = (self.model_str_.replace("arcsin", "asin") - .replace("arccos", "acos") - .replace("^", "**")) + raw = ( + self.model_str_.replace("arcsin", "asin") + .replace("arccos", "acos") + .replace("^", "**") + ) try: loc = {"abs": sp.Abs} - loc.update({f"x{i + 1}": sp.Symbol(f"x{i + 1}") - for i in range(self.n_features_in_)}) + loc.update( + { + f"x{i + 1}": sp.Symbol(f"x{i + 1}") + for i in range(self.n_features_in_) + } + ) expr = _normalize(sp.sympify(raw, locals=loc)) syms = [sp.Symbol(f"x{i + 1}") for i in range(self.n_features_in_)] self.expr_ = expr @@ -111,10 +140,14 @@ def predict(self, X): else: env = {f"x{i + 1}": X[:, i] for i in range(X.shape[1])} env.update(_NP) - v = eval(self.model_str_.replace("^", "**"), {"__builtins__": {}}, env) + v = eval( + self.model_str_.replace("^", "**"), {"__builtins__": {}}, env + ) except Exception: v = self.y_mean_ - v = np.asarray(np.broadcast_to(np.asarray(v, dtype=float), (X.shape[0],)), dtype=float) + v = np.asarray( + np.broadcast_to(np.asarray(v, dtype=float), (X.shape[0],)), dtype=float + ) # Sole deviation from evaluating the model expression verbatim: rows # where it is non-finite (poles, domain edges) fall back to the # training mean so downstream metric computation stays defined. @@ -124,7 +157,7 @@ def predict(self, X): est = KeplearnRegressor() # Deterministic engine: a single (empty) config, i.e. no hyperparameter tuning. -hyper_params = [{}] +hyper_params: list[dict[str, Any]] = [{}] # Raw features (our affine-monomial fit bakes constants; scaling x would perturb # them). fit consumes a plain ndarray. @@ -140,11 +173,10 @@ def predict(self, X): # genuine approximation's messy coefficients are left as-is (its wrong STRUCTURE keeps # it correctly rejected). Any failure falls back to the un-normalized string, so this # can never make the pipeline worse than before. -import math as _math -_SNAP_Q = (2, 3, 4, 5, 6) # simple-rational denominators considered -_SNAP_REL = 1e-7 # coefficient/exponent snap tolerance (relative) -_PI_REL = 1e-6 # transcendental (pi) recognition tolerance +_SNAP_Q = (2, 3, 4, 5, 6) # simple-rational denominators considered +_SNAP_REL = 1e-7 # coefficient/exponent snap tolerance (relative) +_PI_REL = 1e-6 # transcendental (pi) recognition tolerance def _snap_float(f): @@ -161,8 +193,12 @@ def _snap_float(f): p = round(v * q) if p != 0 and abs(v - p / q) <= _SNAP_REL * max(a, 1): return sp.Rational(p, q) - for base, sym in ((_math.pi, sp.pi), (_math.pi ** 2, sp.pi ** 2), - (1.0 / _math.pi, 1 / sp.pi), (1.0 / (_math.pi ** 2), 1 / sp.pi ** 2)): + for base, sym in ( + (_math.pi, sp.pi), + (_math.pi**2, sp.pi**2), + (1.0 / _math.pi, 1 / sp.pi), + (1.0 / (_math.pi**2), 1 / sp.pi**2), + ): k = v / base for q in (1, 2, 3, 4, 5, 6): p = round(k * q) @@ -199,7 +235,7 @@ def _fold_mul(m): # coef*p_d*z*sqrt(..)) that must stay out — folding it regressed those. f = sp.Mul(*nonrad) if f.free_symbols and f.free_symbols <= rad[0].free_symbols: - return sp.Mul(*const) * sp.sqrt(sp.expand(f ** 2 * rad[0])) + return sp.Mul(*const) * sp.sqrt(sp.expand(f**2 * rad[0])) return m return m @@ -244,7 +280,11 @@ def model(est, X=None): else: subs = {sp.Symbol(f"x{i + 1}"): sp.Symbol(names[i]) for i in range(n)} out = str(expr.subs(subs, simultaneous=True)) - return out if (out and out.lower() not in ("nan", "zoo", "oo")) else _string_model(est, names) + return ( + out + if (out and out.lower() not in ("nan", "zoo", "oo")) + else _string_model(est, names) + ) except Exception: return _string_model(est, names) @@ -274,5 +314,8 @@ def complexity(est): except Exception: pass raw = getattr(est, "model_str_", "0") - return len(re.findall( - r"\*\*|[+\-*/]|\b(?:sin|cos|tan|sqrt|exp|log|tanh|arcsin|arccos|abs)\b", raw)) + return len( + re.findall( + r"\*\*|[+\-*/]|\b(?:sin|cos|tan|sqrt|exp|log|tanh|arcsin|arccos|abs)\b", raw + ) + ) diff --git a/experiment/methods/lightgbm/regressor.py b/experiment/methods/lightgbm/regressor.py index 1960eee0..a8b88eb4 100644 --- a/experiment/methods/lightgbm/regressor.py +++ b/experiment/methods/lightgbm/regressor.py @@ -1,7 +1,6 @@ import lightgbm import numpy as np - # original set of hyperparameters (too many for the experimental design): # { # 'n_estimators' : (10, 100, 1000), @@ -13,21 +12,21 @@ hyper_params = [] for n_estimators in (10, 100, 1000): for learning_rate in (0.0001, 0.01): - for boosting_type in ('gbdt', 'dart', 'goss'): - hyper_params.append({ - 'n_estimators' : [n_estimators], - 'learning_rate' : [learning_rate], - 'boosting_type' : [boosting_type] - }) - -est=lightgbm.LGBMRegressor( - max_depth=6, - deterministic = True, - force_row_wise = True - ) + for boosting_type in ("gbdt", "dart", "goss"): + hyper_params.append( + { + "n_estimators": [n_estimators], + "learning_rate": [learning_rate], + "boosting_type": [boosting_type], + } + ) + +est = lightgbm.LGBMRegressor(max_depth=6, deterministic=True, force_row_wise=True) + def complexity(est): - c = np.sum([x['num_leaves'] for x in est._Booster.dump_model()['tree_info']]) + c = np.sum([x["num_leaves"] for x in est._Booster.dump_model()["tree_info"]]) return int(c) + model = None diff --git a/experiment/methods/nesymres/regressor.py b/experiment/methods/nesymres/regressor.py index ee1db048..98c966a0 100644 --- a/experiment/methods/nesymres/regressor.py +++ b/experiment/methods/nesymres/regressor.py @@ -1,23 +1,20 @@ -import time -import json -import os, sys +from __future__ import annotations -import torch -import numpy as np -from sklearn import feature_selection - -from nesymres.architectures.model import Model -from nesymres.dclasses import FitParams, BFGSParams from functools import partial -import sympy as sp -from sympy import lambdify -import omegaconf - from tempfile import TemporaryDirectory +from typing import Any +import numpy as np +import omegaconf +import sympy as sp +import torch +from nesymres.architectures.model import Model +from nesymres.dclasses import BFGSParams, FitParams +from sklearn import feature_selection from sklearn.base import BaseEstimator, RegressorMixin +from sympy import lambdify -eq_setting = { +eq_setting: dict[str, Any] = { "config": { "max_len": 20, "positive": True, @@ -27,13 +24,9 @@ "int_base": 10, "precision": 10, "rewrite_functions": "", - "variables": [ - "x_1", - "x_2", - "x_3" - ], + "variables": ["x_1", "x_2", "x_3"], "eos_index": 1, - "pad_index": 0 + "pad_index": 0, }, "total_coefficients": [ "cm_0", @@ -115,127 +108,119 @@ "ca_36", "ca_37", "ca_38", - "ca_39" - ], - "total_variables": [ - "x_1", - "x_2", - "x_3" + "ca_39", ], + "total_variables": ["x_1", "x_2", "x_3"], "word2id": { - "0": 28, - "1": 29, - "2": 30, - "3": 31, - "4": 32, - "5": 33, - "x_1": 4, - "x_2": 5, - "x_3": 6, - "abs": 7, - "acos": 8, - "add": 9, - "asin": 10, - "atan": 11, - "cos": 12, - "cosh": 13, - "coth": 14, - "div": 15, - "exp": 16, - "ln": 17, - "mul": 18, - "pow": 19, - "sin": 20, - "sinh": 21, - "sqrt": 22, - "tan": 23, - "tanh": 24, - "-3": 25, - "-2": 26, - "-1": 27, - "P": 0, - "S": 1, - "F": 2, - "c": 3 + "0": 28, + "1": 29, + "2": 30, + "3": 31, + "4": 32, + "5": 33, + "x_1": 4, + "x_2": 5, + "x_3": 6, + "abs": 7, + "acos": 8, + "add": 9, + "asin": 10, + "atan": 11, + "cos": 12, + "cosh": 13, + "coth": 14, + "div": 15, + "exp": 16, + "ln": 17, + "mul": 18, + "pow": 19, + "sin": 20, + "sinh": 21, + "sqrt": 22, + "tan": 23, + "tanh": 24, + "-3": 25, + "-2": 26, + "-1": 27, + "P": 0, + "S": 1, + "F": 2, + "c": 3, }, "id2word": { - "1": "S", - "2": "F", - "3": "c", - "4": "x_1", - "5": "x_2", - "6": "x_3", - "7": "abs", - "8": "acos", - "9": "add", - "10": "asin", - "11": "atan", - "12": "cos", - "13": "cosh", - "14": "coth", - "15": "div", - "16": "exp", - "17": "ln", - "18": "mul", - "19": "pow", - "20": "sin", - "21": "sinh", - "22": "sqrt", - "23": "tan", - "24": "tanh", - "25": "-3", - "26": "-2", - "27": "-1", - "28": "0", - "29": "1", - "30": "2", - "31": "3", - "32": "4", - "33": "5" + "1": "S", + "2": "F", + "3": "c", + "4": "x_1", + "5": "x_2", + "6": "x_3", + "7": "abs", + "8": "acos", + "9": "add", + "10": "asin", + "11": "atan", + "12": "cos", + "13": "cosh", + "14": "coth", + "15": "div", + "16": "exp", + "17": "ln", + "18": "mul", + "19": "pow", + "20": "sin", + "21": "sinh", + "22": "sqrt", + "23": "tan", + "24": "tanh", + "25": "-3", + "26": "-2", + "27": "-1", + "28": "0", + "29": "1", + "30": "2", + "31": "3", + "32": "4", + "33": "5", }, "una_ops": [ - "asin", - "cos", - "exp", - "ln", - "pow2", - "pow3", - "pow4", - "pow5", - "sin", - "sqrt", - "tan" + "asin", + "cos", + "exp", + "ln", + "pow2", + "pow3", + "pow4", + "pow5", + "sin", + "sqrt", + "tan", ], "bin_ops": [ - "asin", - "cos", - "exp", - "ln", - "pow2", - "pow3", - "pow4", - "pow5", - "sin", - "sqrt", - "tan" + "asin", + "cos", + "exp", + "ln", + "pow2", + "pow3", + "pow4", + "pow5", + "sin", + "sqrt", + "tan", ], "rewrite_functions": [], "total_number_of_eqs": 200, "eqs_per_hdf": 200, "generator_details": { - "max_len": 20, - "operators": "add:10,mul:10,sub:5,div:5,sqrt:4,pow2:4,pow3:2,pow4:1,pow5:1,ln:4,exp:4,sin:4,cos:4,tan:4,asin:2", - "max_ops": 5, - "rewrite_functions": "", - "variables": [ - "x_1", - "x_2", - "x_3" - ], - "eos_index": 1, - "pad_index": 0 + "max_len": 20, + "operators": "add:10,mul:10,sub:5,div:5,sqrt:4,pow2:4,pow3:2,pow4:1,pow5:1,ln:4,exp:4,sin:4,cos:4,tan:4,asin:2", + "max_ops": 5, + "rewrite_functions": "", + "variables": ["x_1", "x_2", "x_3"], + "eos_index": 1, + "pad_index": 0, }, - "unique_index": None + "unique_index": None, } cfg = None @@ -355,14 +340,14 @@ sweep: dir: runs/${dataset_train.predict_c}/${now:%Y-%m-%d}/${now:%H-%M-%S} """) - + cfg = omegaconf.OmegaConf.load(cname) def get_top_k_features(X, y, k=10): - if y.ndim==2: - y=y[:,0] - if X.shape[1]<=k: + if y.ndim == 2: + y = y[:, 0] + if X.shape[1] <= k: return [i for i in range(X.shape[1])] else: kbest = feature_selection.SelectKBest(feature_selection.r_regression, k=k) @@ -372,8 +357,9 @@ def get_top_k_features(X, y, k=10): print("keeping only the top-{} features. Order was {}".format(k, top_features)) return list(top_features[:k]) + def get_variables(equation): - """ Parse all free variables in equations and return them in + """Parse all free variables in equations and return them in lexicographic order""" expr = sp.parse_expr(equation) @@ -398,10 +384,10 @@ def get_variables(equation): def evaluate_func(func_str, vars_list, X): assert X.ndim == 2 - assert len(set(vars_list)) == len(vars_list), 'Duplicates in vars_list!' + assert len(set(vars_list)) == len(vars_list), "Duplicates in vars_list!" order_list = vars_list - indeces = [int(x[2:])-1 for x in order_list] + indeces = [int(x[2:]) - 1 for x in order_list] if not order_list: # Empty order list. Constant function predicted @@ -412,8 +398,7 @@ def evaluate_func(func_str, vars_list, X): # that are not in the ground-truth equation X_padded = np.zeros((X.shape[0], len(vars_list))) - - X_padded[:, :X.shape[1]] = X[:,:X_padded.shape[1]] + X_padded[:, : X.shape[1]] = X[:, : X_padded.shape[1]] # Subselect columns of X that corrspond to provided variables X_subsel = X_padded[:, indeces] @@ -423,19 +408,18 @@ def evaluate_func(func_str, vars_list, X): return f(*X_subsel.T) -n_restarts = [1, 10, 20] +n_restarts = [1, 10, 20] total_number_of_eqs = [100, 250, 500] -max_ops = [10, 20, 50] +max_ops = [10, 20, 50] hyper_params = [] for restarts, eqs in zip(n_restarts, total_number_of_eqs): for ops in max_ops: - hyper_params.append({ - 'n_restarts' : [restarts], - 'total_number_of_eqs' : [eqs], - 'max_ops' : [ops] - }) - + hyper_params.append( + {"n_restarts": [restarts], "total_number_of_eqs": [eqs], "max_ops": [ops]} + ) + + class NeSymResRegressor(BaseEstimator, RegressorMixin): def __init__(self, n_restarts=10, total_number_of_eqs=200, max_ops=5): self.n_restarts = n_restarts @@ -443,14 +427,16 @@ def __init__(self, n_restarts=10, total_number_of_eqs=200, max_ops=5): self.max_ops = max_ops def fit(self, X, y): + if cfg is None: + raise RuntimeError("NeSymRes configuration has not been loaded") # Setting parameters cfg.inference.bfgs.n_restarts = self.n_restarts - eq_setting['total_number_of_eqs'] = self.total_number_of_eqs - eq_setting['config']['max_ops'] = self.max_ops + eq_setting["total_number_of_eqs"] = self.total_number_of_eqs + eq_setting["config"]["max_ops"] = self.max_ops ## Set up BFGS load rom the hydra config yaml bfgs = BFGSParams( - activated= cfg.inference.bfgs.activated, + activated=cfg.inference.bfgs.activated, n_restarts=cfg.inference.bfgs.n_restarts, add_coefficients_if_not_existing=cfg.inference.bfgs.add_coefficients_if_not_existing, normalization_o=cfg.inference.bfgs.normalization_o, @@ -463,40 +449,41 @@ def fit(self, X, y): self.top_k_features = get_top_k_features(X, y, k=3) X = X[:, self.top_k_features] - eq_setting["total_variables"] = [f"x_{i+1}" for i in range(X.shape[1])] + eq_setting["total_variables"] = [f"x_{i + 1}" for i in range(X.shape[1])] eq_setting["num_variables"] = int(X.shape[1]) - eq_setting["variables"] = [f"x_{i+1}" for i in range(X.shape[1])] + eq_setting["variables"] = [f"x_{i + 1}" for i in range(X.shape[1])] print(eq_setting["total_variables"]) - params_fit = FitParams(word2id=eq_setting["word2id"], - id2word={int(k): v for k,v in eq_setting["id2word"].items()}, - una_ops=eq_setting["una_ops"], - bin_ops=eq_setting["bin_ops"], - total_variables=eq_setting["total_variables"], - total_coefficients=eq_setting["total_coefficients"], - rewrite_functions=eq_setting["rewrite_functions"], - bfgs=bfgs, - beam_size=cfg.inference.beam_size #This parameter is a tradeoff between accuracy and fitting time - ) + params_fit = FitParams( + word2id=eq_setting["word2id"], + id2word={int(k): v for k, v in eq_setting["id2word"].items()}, + una_ops=eq_setting["una_ops"], + bin_ops=eq_setting["bin_ops"], + total_variables=eq_setting["total_variables"], + total_coefficients=eq_setting["total_coefficients"], + rewrite_functions=eq_setting["rewrite_functions"], + bfgs=bfgs, + beam_size=cfg.inference.beam_size, # This parameter is a tradeoff between accuracy and fitting time + ) weights_path = "/srbench_pretrained/nesymres_100M.ckpt" ## Load architecture, set into eval mode, and pass the config parameters - #cfg.architecture.num_features = eq_setting["num_variables"] - + # cfg.architecture.num_features = eq_setting["num_variables"] + model = Model.load_from_checkpoint(weights_path, cfg=cfg.architecture) model.eval() - if torch.cuda.is_available(): + if torch.cuda.is_available(): model.cuda() fitfunc = partial(model.fitfunc, cfg_params=params_fit) - output_ref = fitfunc(X,y) + output_ref = fitfunc(X, y) self.model_skeleton_ = output_ref - self.model_eq_ = model.get_equation() + self.model_eq_ = model.get_equation() print("NeSymReS output ref: ", output_ref) print("NeSymReS Equation Skeleton: ", self.model_skeleton_) @@ -505,19 +492,19 @@ def fit(self, X, y): self.pred_variables_ = get_variables(self.model_eq_[0]) return self - def predict(self, X): X = X[:, self.top_k_features] - return evaluate_func(self.model_eq_[0], - self.pred_variables_, X) + return evaluate_func(self.model_eq_[0], self.pred_variables_, X) est = NeSymResRegressor() + def model(est, X=None): return str(est.model_eq_[0]) + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/operon/regressor.py b/experiment/methods/operon/regressor.py index bbe40bef..bab3e185 100644 --- a/experiment/methods/operon/regressor.py +++ b/experiment/methods/operon/regressor.py @@ -1,17 +1,17 @@ import copy -import numpy as np -import optuna import os -import pandas as pd import time +import numpy as np +import optuna +import pandas as pd import pyoperon as Operon from pyoperon.sklearn import SymbolicRegressor +from sklearn.base import BaseEstimator, RegressorMixin from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error from sklearn.model_selection import KFold -from sklearn.base import BaseEstimator, RegressorMixin -from sympy import lambdify, parse_expr, Symbol +from sympy import Symbol, lambdify, parse_expr def compute_sigma(X, y): @@ -26,9 +26,13 @@ def compute_mdl(tree, X, y): problem.TrainingRange = Operon.Range(0, X.shape[0]) target = max(ds.Variables, key=lambda x: x.Index) problem.Target = target - problem.InputHashes = [ v.Hash for v in sorted(ds.Variables, key=lambda x: x.Index) if v.Hash != target.Hash ] + problem.InputHashes = [ + v.Hash + for v in sorted(ds.Variables, key=lambda x: x.Index) + if v.Hash != target.Hash + ] dtable = Operon.DispatchTable() - mdl_eval = Operon.MinimumDescriptionLengthEvaluator(problem, dtable, 'gauss') + mdl_eval = Operon.MinimumDescriptionLengthEvaluator(problem, dtable, "gauss") mdl_eval.Sigma = [compute_sigma(X, y)] rng = Operon.RomuTrio(1234) ind = Operon.Individual() @@ -53,13 +57,13 @@ class SympyExprModel(RegressorMixin): Constructor which builds a lambda from the expression string using the specified variable names. """ + def __init__(self, expr_str, variable_names): - self.model = expr_str + self.model = expr_str self.variable_names = variable_names - self.symbols = [Symbol(x) for x in variable_names] + self.symbols = [Symbol(x) for x in variable_names] self.best_estimator = lambdify(self.symbols, parse_expr(expr_str)) - self.is_fitted = True - + self.is_fitted = True def predict(self, X): values = X.values if isinstance(X, pd.DataFrame) else X @@ -77,43 +81,52 @@ def __init__(self, X, y, n_folds, seed, fixed_params={}, suggest_callback=None): self.params = fixed_params self.suggest_callback = suggest_callback - def __call__(self, trial): - hyper_params = {} if self.suggest_callback is None else self.suggest_callback(trial) + hyper_params = ( + {} if self.suggest_callback is None else self.suggest_callback(trial) + ) params = copy.deepcopy(self.params) params.update(hyper_params) - params.update({'random_state': self.seed}) + params.update({"random_state": self.seed}) self.reg = SymbolicRegressor(**params) X_, y_ = np.asarray(self.X), np.asarray(self.y) - if self.n_folds > 1: kf = KFold(n_splits=self.n_folds) score = 0 for train, test in kf.split(X_, y_): X_train, y_train = X_[train, :], y_[train] - X_test, y_test = X_[test, :], y_[test] + X_test, y_test = X_[test, :], y_[test] self.reg.fit(X_train, y_train) - best = min(self.reg.pareto_front_, key=lambda x: x['minimum_description_length']) - score += compute_mdl(best['tree'], X_test, y_test) - - return score/self.n_folds + best = min( + self.reg.pareto_front_, + key=lambda x: x["minimum_description_length"], + ) + score += compute_mdl(best["tree"], X_test, y_test) + return score / self.n_folds self.reg.fit(X_, y_) - return min((x['minimum_description_length'] for x in self.reg.pareto_front_)) - + return min((x["minimum_description_length"] for x in self.reg.pareto_front_)) def callback(self, study, trial): if self.best_reg is None or study.best_trial == trial: self.best_reg = self.reg - class OperonOptuna(BaseEstimator, RegressorMixin): - def __init__(self, name, n_folds=3, n_trials=100, max_time=3600, fixed_params={}, suggest_callback=None, random_state=None): + def __init__( + self, + name, + n_folds=3, + n_trials=100, + max_time=3600, + fixed_params={}, + suggest_callback=None, + random_state=None, + ): self.name = name self.n_folds = n_folds self.n_trials = n_trials @@ -122,69 +135,97 @@ def __init__(self, name, n_folds=3, n_trials=100, max_time=3600, fixed_params={} self.fixed_params = fixed_params self.suggest_callback = suggest_callback self.best_estimator = None # to hold the best estimator - self.best_parameters = None # to hold the best parameters - + self.best_parameters = None # to hold the best parameters def fit(self, X, y): t0 = time.time() sErr = compute_sigma(X, y) fixed_params = copy.deepcopy(self.fixed_params) - fixed_params.update({'uncertainty': [sErr]}) + fixed_params.update({"uncertainty": [sErr]}) t1 = time.time() - curr_time = np.ceil(t1-t0) - - objective = Objective(X, y, self.n_folds, fixed_params=fixed_params, suggest_callback=self.suggest_callback, seed=self.random_state) - study = optuna.create_study(study_name=self.name, direction='minimize') - study.optimize(objective, n_trials=self.n_trials, callbacks=[objective.callback], timeout=(self.max_time - curr_time)) - self.best_estimator = objective.best_reg + curr_time = np.ceil(t1 - t0) + + objective = Objective( + X, + y, + self.n_folds, + fixed_params=fixed_params, + suggest_callback=self.suggest_callback, + seed=self.random_state, + ) + study = optuna.create_study(study_name=self.name, direction="minimize") + study.optimize( + objective, + n_trials=self.n_trials, + callbacks=[objective.callback], + timeout=(self.max_time - curr_time), + ) + self.best_estimator = objective.best_reg + if self.best_estimator is None: + raise RuntimeError("Operon optimization produced no fitted estimator") self.best_parameters = self.best_estimator.get_params() - self.best_parameters['completed trials'] = len(study.trials) + self.best_parameters["completed trials"] = len(study.trials) # save the best solution as a SympyExprModel best = self.best_estimator - names = X.columns.tolist() if isinstance(X, pd.DataFrame) else list(best.variables_.values()) - model_string = lambda x: best.get_model_string(x, 6, names).replace('^', '**') + names = ( + X.columns.tolist() + if isinstance(X, pd.DataFrame) + else list(best.variables_.values()) + ) - self.best_solution = SympyExprModel(model_string(best.model_), names) - self.model = self.best_solution.model - self.population = [SympyExprModel(model_string(x['tree']), names) for x in best.pareto_front_[:100]] + def model_string(x): + return best.get_model_string(x, 6, names).replace("^", "**") + self.best_solution = SympyExprModel(model_string(best.model_), names) + self.model = self.best_solution.model + self.population = [ + SympyExprModel(model_string(x["tree"]), names) + for x in best.pareto_front_[:100] + ] def predict(self, X): + if self.best_estimator is None: + raise RuntimeError("Operon must be fitted before prediction") return self.best_estimator.predict(X) -num_threads = int(os.environ['OMP_NUM_THREADS']) if 'OMP_NUM_THREADS' in os.environ else 1 +num_threads = ( + int(os.environ["OMP_NUM_THREADS"]) if "OMP_NUM_THREADS" in os.environ else 1 +) default_params = { - 'offspring_generator': 'basic', - 'initialization_method': 'btc', - 'n_threads': num_threads, - 'objectives': ['r2', 'length'], - 'epsilon': 1e-5, - 'optimizer_iterations': 1, - 'random_state': None, - 'reinserter': 'keep-best', - 'max_evaluations': int(1e6), - 'female_selector' : 'tournament', - 'male_selector': 'tournament', - 'brood_size': 5, - 'population_size': 1000, - 'pool_size': None, - 'max_time': 300, - 'model_selection_criterion': 'minimum_description_length', - } + "offspring_generator": "basic", + "initialization_method": "btc", + "n_threads": num_threads, + "objectives": ["r2", "length"], + "epsilon": 1e-5, + "optimizer_iterations": 1, + "random_state": None, + "reinserter": "keep-best", + "max_evaluations": int(1e6), + "female_selector": "tournament", + "male_selector": "tournament", + "brood_size": 5, + "population_size": 1000, + "pool_size": None, + "max_time": 300, + "model_selection_criterion": "minimum_description_length", +} def suggest_params(trial): return { - 'allowed_symbols' : trial.suggest_categorical('allowed_symbols', [ - 'add,sub,mul,div,constant,variable', - 'add,sub,mul,div,sin,cos,constant,variable', - 'add,sub,mul,div,exp,log,constant,variable', - 'add,sub,mul,div,sin,cos,exp,log,sqrt,pow,constant,variable' - ]), - 'max_length' : trial.suggest_int('max_length', low=10, high=50, step=10), + "allowed_symbols": trial.suggest_categorical( + "allowed_symbols", + [ + "add,sub,mul,div,constant,variable", + "add,sub,mul,div,sin,cos,constant,variable", + "add,sub,mul,div,exp,log,constant,variable", + "add,sub,mul,div,sin,cos,exp,log,sqrt,pow,constant,variable", + ], + ), + "max_length": trial.suggest_int("max_length", low=10, high=50, step=10), } @@ -193,7 +234,7 @@ def model(est, X=None): def complexity(est): - return est.best_estimator.stats_['model_complexity'] + return est.best_estimator.stats_["model_complexity"] def get_best_solution(est): @@ -204,7 +245,13 @@ def get_population(est): return [est] if isinstance(est, SympyExprModel) else est.population -est = OperonOptuna('operon-optuna', n_folds=1, n_trials=100, fixed_params=default_params, suggest_callback=suggest_params) +est = OperonOptuna( + "operon-optuna", + n_folds=1, + n_trials=100, + fixed_params=default_params, + suggest_callback=suggest_params, +) ################################################################################ # Optional Settings @@ -243,7 +290,7 @@ def pre_train_fn(est, X, y): # pass the function to eval_kwargs eval_kwargs = { - 'pre_train': pre_train_fn, - 'test_params': {'max_time': 60 }, - 'use_dataframe' : False, + "pre_train": pre_train_fn, + "test_params": {"max_time": 60}, + "use_dataframe": False, } diff --git a/experiment/methods/pantara/regressor.py b/experiment/methods/pantara/regressor.py index 1bd96642..bd15c37e 100644 --- a/experiment/methods/pantara/regressor.py +++ b/experiment/methods/pantara/regressor.py @@ -25,18 +25,29 @@ block is silently dropped — predict() returns partial sum. """ -import os -import sys -import numpy as np -import warnings +from __future__ import annotations -import torch +from typing import Any + +import numpy as np import pantara +import torch from pantara import ( - OracleClassifier, N_CLASSES, DEVICE, FAMILIES, N_POINTS, MIN_SCORE, - best_per_family_multispace, make_state_stats, make_oracle_input, - fit_trig, detect_and_fit_power_law, bloc_score, power_law_bloc_score, - transform_space, inverse_transform_y, corr_np, standardize, + DEVICE, + FAMILIES, + MIN_SCORE, + N_CLASSES, + N_POINTS, + OracleClassifier, + best_per_family_multispace, + bloc_score, + detect_and_fit_power_law, + fit_trig, + inverse_transform_y, + make_oracle_input, + make_state_stats, + power_law_bloc_score, + transform_space, ) _MODEL_PATH = pantara.MODEL_PATH @@ -46,10 +57,11 @@ # Each recipe is a dict with a 'type' key and enough state to call predict(). # ───────────────────────────────────────────────────────────────────────────── + def _predict_power_law(block, X): """y = C · x0^n0 · x1^n1 · ...""" eps = 1e-10 - coeffs = block['coeffs'] # (D+1,): [log C, n1, n2, ...] + coeffs = block["coeffs"] # (D+1,): [log C, n1, n2, ...] N = len(X) log_X = np.log(np.abs(X) + eps) A = np.column_stack([np.ones(N), log_X]) @@ -59,36 +71,36 @@ def _predict_power_law(block, X): def _predict_trig(block, X): """y = A · sin/cos(ω·xj + φ) + C (or + B·xj for linear variant)""" - j = block['j_var'] - x = X[:, j] - A = block['A'] - w = block['omega'] - phi = block['phi'] - C = block['C'] - form = block['form'] - if form == 'sin': + j = block["j_var"] + x = X[:, j] + A = block["A"] + w = block["omega"] + phi = block["phi"] + C = block["C"] + form = block["form"] + if form == "sin": return A * np.sin(w * x + phi) + C - elif form == 'cos': + elif form == "cos": return A * np.cos(w * x + phi) + C - elif form == 'sin+lin': - B = block.get('B', 0.0) + elif form == "sin+lin": + B = block.get("B", 0.0) return A * np.sin(w * x + phi) + B * x + C else: # cos+lin - B = block.get('B', 0.0) + B = block.get("B", 0.0) return A * np.cos(w * x + phi) + B * x + C def _predict_1d_linear(block, X): """Replay a 1-D OLS block on new X.""" - space = block['space'] - family = block['family'] - indices = block['indices'] # tuple of feature indices - mn_c = block['mn_c'] - iqr_c = block['iqr_c'] - mn_r = block['mn_r'] - iqr_r = block['iqr_r'] - w = block['w'] - b = block['b'] + space = block["space"] + family = block["family"] + indices = block["indices"] # tuple of feature indices + mn_c = block["mn_c"] + iqr_c = block["iqr_c"] + mn_r = block["mn_r"] + iqr_r = block["iqr_r"] + w = block["w"] + b = block["b"] # Dummy y must be positive: transform_space rejects y<=0 for log_y/log_log # (it would return valid=False and make this block silently predict 0). @@ -101,24 +113,24 @@ def _predict_1d_linear(block, X): if col_new is None: return np.zeros(len(X)) - col_n = (col_new - mn_c) / (iqr_c + 1e-10) - pred_t = (w * col_n + b) * iqr_r + mn_r + col_n = (col_new - mn_c) / (iqr_c + 1e-10) + pred_t = (w * col_n + b) * iqr_r + mn_r return inverse_transform_y(pred_t, space) def _predict_multi_linear(block, X): """Replay a multivariate OLS block (log-space) on new X.""" - space = block['space'] - mu_x = block['mu_x'] - s_x = block['s_x'] - mu_r = block['mu_r'] - s_r = block['s_r'] - W = block['W'] - b = block['b'] - - if space == 'log_log_multi': - eps = 1e-10 - X_t = np.log(np.abs(X) + eps) + space = block["space"] + mu_x = block["mu_x"] + s_x = block["s_x"] + mu_r = block["mu_r"] + s_r = block["s_r"] + W = block["W"] + b = block["b"] + + if space == "log_log_multi": + eps = 1e-10 + X_t = np.log(np.abs(X) + eps) else: # Dummy y must be positive (see _predict_1d_linear): transform_space # rejects y<=0 for log_y/log_log, which would zero out this block. @@ -128,18 +140,18 @@ def _predict_multi_linear(block, X): # Pad or truncate to match stored dimension D_stored = len(mu_x) - D_new = X_t.shape[1] + D_new = X_t.shape[1] if D_new < D_stored: X_t = np.column_stack([X_t, np.zeros((len(X_t), D_stored - D_new))]) elif D_new > D_stored: X_t = X_t[:, :D_stored] - X_std = (X_t - mu_x) / (s_x + 1e-10) + X_std = (X_t - mu_x) / (s_x + 1e-10) pred_s = X_std @ W + b pred_t = pred_s * s_r + mu_r - if space == 'log_log_multi': - return np.exp(pred_t) # Fix B from v7d + if space == "log_log_multi": + return np.exp(pred_t) # Fix B from v7d return inverse_transform_y(pred_t, space) @@ -147,30 +159,35 @@ def _predict_multi_linear(block, X): # FEATURE HELPERS # ───────────────────────────────────────────────────────────────────────────── + def _compute_feature_col(family, indices, X_t): j = indices[0] D = X_t.shape[1] if j >= D: return None - if family == 'lin': + if family == "lin": return X_t[:, j].copy() - elif family == 'sq': + elif family == "sq": return X_t[:, j] ** 2 - elif family == 'pair': + elif family == "pair": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] * X_t[:, k] - elif family == 'qc': + elif family == "qc": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] * X_t[:, k] ** 2 - elif family == 'trip': - k, l = indices[1], indices[2] - if k >= D or l >= D: return None - return X_t[:, j] * X_t[:, k] * X_t[:, l] - elif family == 'qinv': + elif family == "trip": + k, third = indices[1], indices[2] + if k >= D or third >= D: + return None + return X_t[:, j] * X_t[:, k] * X_t[:, third] + elif family == "qinv": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] ** 2 / (X_t[:, k] + 1e-10) return None @@ -181,35 +198,37 @@ def _find_feature_indices(family, col, X_t): def match(c): # Compare in standardized form (shape-invariant) - c_s = c - c.mean(); c_n = c_s / (np.std(c_s) + 1e-10) - co_s = col - col.mean(); co_n = co_s / (np.std(co_s) + 1e-10) + c_s = c - c.mean() + c_n = c_s / (np.std(c_s) + 1e-10) + co_s = col - col.mean() + co_n = co_s / (np.std(co_s) + 1e-10) return float(np.mean((c_n - co_n) ** 2)) < 1e-4 - if family == 'lin': + if family == "lin": for j in range(D): if match(X_t[:, j]): return (j,) - elif family == 'sq': + elif family == "sq": for j in range(D): if match(X_t[:, j] ** 2): return (j,) - elif family == 'pair': + elif family == "pair": for j in range(D): for k in range(j + 1, D): if match(X_t[:, j] * X_t[:, k]): return (j, k) - elif family == 'qc': + elif family == "qc": for j in range(D): for k in range(D): if j != k and match(X_t[:, j] * X_t[:, k] ** 2): return (j, k) - elif family == 'trip': + elif family == "trip": for j in range(D): for k in range(j + 1, D): - for l in range(k + 1, D): - if match(X_t[:, j] * X_t[:, k] * X_t[:, l]): - return (j, k, l) - elif family == 'qinv': + for third in range(k + 1, D): + if match(X_t[:, j] * X_t[:, k] * X_t[:, third]): + return (j, k, third) + elif family == "qinv": for j in range(D): for k in range(D): if j != k and np.all(np.abs(X_t[:, k]) > 1e-10): @@ -220,12 +239,12 @@ def match(c): def _fit_ols_1d(col_t, resid_t): """OLS linear fit in normalized space. Returns (mn_c, iqr_c, mn_r, iqr_r, w, b).""" - mn_c = float(np.median(col_t)) + mn_c = float(np.median(col_t)) iqr_c = float(np.percentile(col_t, 75) - np.percentile(col_t, 25)) + 1e-10 - mn_r = float(np.median(resid_t)) + mn_r = float(np.median(resid_t)) iqr_r = float(np.percentile(resid_t, 75) - np.percentile(resid_t, 25)) + 1e-10 - col_n = (col_t - mn_c) / iqr_c + col_n = (col_t - mn_c) / iqr_c resid_n = (resid_t - mn_r) / iqr_r A = np.column_stack([col_n, np.ones(len(col_n))]) @@ -236,13 +255,13 @@ def _fit_ols_1d(col_t, resid_t): def _fit_ols_multi(X_t, resid_t): """OLS multivariate fit. Returns (mu_x, s_x, mu_r, s_r, W, b).""" - N, D = X_t.shape - mu_x = X_t.mean(axis=0) - s_x = X_t.std(axis=0) + 1e-10 - X_std = (X_t - mu_x) / s_x + N, D = X_t.shape + mu_x = X_t.mean(axis=0) + s_x = X_t.std(axis=0) + 1e-10 + X_std = (X_t - mu_x) / s_x - mu_r = float(resid_t.mean()) - s_r = float(resid_t.std()) + 1e-10 + mu_r = float(resid_t.mean()) + s_r = float(resid_t.std()) + 1e-10 resid_s = (resid_t - mu_r) / s_r A = np.column_stack([X_std, np.ones(N)]) @@ -256,6 +275,7 @@ def _fit_ols_multi(X_t, resid_t): # PANTARA REGRESSOR # ───────────────────────────────────────────────────────────────────────────── + class PantaraRegressor: """ Pantara v7d — Matching-pursuit symbolic regression. @@ -273,7 +293,7 @@ class PantaraRegressor: """ def __init__(self, max_time=3600, random_state=42): - self.max_time = max_time # set by SRBench harness + self.max_time = max_time # set by SRBench harness self.random_state = random_state # ── helpers ────────────────────────────────────────────────────────────── @@ -294,14 +314,15 @@ def _load_oracle(self): def _to_numpy(self, X): """Accept DataFrame or ndarray, return float64 ndarray (N, D).""" import pandas as pd + if isinstance(X, pd.DataFrame): self.feature_names_ = list(X.columns) return X.values.astype(np.float64) arr = np.asarray(X, dtype=np.float64) if arr.ndim == 1: arr = arr.reshape(-1, 1) - if not hasattr(self, 'feature_names_') or self.feature_names_ is None: - self.feature_names_ = [f'x{j}' for j in range(arr.shape[1])] + if not hasattr(self, "feature_names_") or self.feature_names_ is None: + self.feature_names_ = [f"x{j}" for j in range(arr.shape[1])] return arr # ── fit ────────────────────────────────────────────────────────────────── @@ -309,14 +330,14 @@ def _to_numpy(self, X): def fit(self, X, y): np.random.seed(self.random_state) - X_np = self._to_numpy(X) - y_np = np.asarray(y, dtype=np.float64).ravel() + X_np = self._to_numpy(X) + y_np = np.asarray(y, dtype=np.float64).ravel() self.fitted_blocks_ = [] - self.chosen_ = [] + self.chosen_ = [] oracle = self._load_oracle() - resid = y_np.copy() + resid = y_np.copy() for _step in range(5): ratio = resid.std() / (y_np.std() + 1e-10) @@ -324,7 +345,7 @@ def fit(self, X, y): break scores, best_config, _ = best_per_family_multispace(X_np, resid, y_np) - stats = make_state_stats(scores, resid, y_np, X_np) + stats = make_state_stats(scores, resid, y_np, X_np) points = make_oracle_input(X_np, resid, y_np, N_POINTS) ranked, probs = oracle.predict_ranked(points, stats) @@ -334,13 +355,15 @@ def fit(self, X, y): if pl is not None: pred_pl, coeffs_pl, _ = pl new_resid = resid - pred_pl - quality = power_law_bloc_score(resid, new_resid, y_np) + quality = power_law_bloc_score(resid, new_resid, y_np) if quality > 0.30: - self.fitted_blocks_.append({ - 'type': 'power_law', - 'coeffs': coeffs_pl.copy(), - }) - self.chosen_.append('power_law[log_log_lstsq]') + self.fitted_blocks_.append( + { + "type": "power_law", + "coeffs": coeffs_pl.copy(), + } + ) + self.chosen_.append("power_law[log_log_lstsq]") resid = new_resid if quality > 0.80: break @@ -350,15 +373,15 @@ def fit(self, X, y): actions = list(ranked[:4]) # bonus lin[original] - lin_idx = FAMILIES.index('lin') - sc_l, sp_l, _, _, _ = best_config['lin'] - if sp_l == 'original' and sc_l >= 0.85 and lin_idx not in actions: + lin_idx = FAMILIES.index("lin") + sc_l, sp_l, _, _, _ = best_config["lin"] + if sp_l == "original" and sc_l >= 0.85 and lin_idx not in actions: actions.append(lin_idx) # bonus pair[original] - pair_idx = FAMILIES.index('pair') - sc_p, sp_p, _, _, _ = best_config['pair'] - if sp_p == 'original' and sc_p >= 0.85 and pair_idx not in actions: + pair_idx = FAMILIES.index("pair") + sc_p, sp_p, _, _, _ = best_config["pair"] + if sp_p == "original" and sc_p >= 0.85 and pair_idx not in actions: actions.append(pair_idx) candidates = [] @@ -371,38 +394,46 @@ def fit(self, X, y): continue # ── trig: use specialized estimator ───────────────────── - if fam in ('sin', 'cos'): + if fam in ("sin", "cos"): try: - omega_init = float(sp.split('_w')[-1]) + omega_init = float(sp.split("_w")[-1]) except Exception: omega_init = None try: - j_var = int(sp.split('_x')[1].split('_')[0]) + j_var = int(sp.split("_x")[1].split("_")[0]) except Exception: j_var = 0 new_resid, pred_orig, trig_info = fit_trig( - X_np[:, j_var], resid, y_np, + X_np[:, j_var], + resid, + y_np, omega_candidates=[omega_init] if omega_init else None, try_linear=False, steps=600, ) sc_after, _, _ = best_per_family_multispace(X_np, new_resid, y_np) - quality = bloc_score(resid, new_resid, y_np, sc_after) - clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) + quality = bloc_score(resid, new_resid, y_np, sc_after) + clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) combined = quality * (0.6 + 0.4 * clf_w) - candidates.append({ - 'family': fam, 'space': sp, 'action': action, - 'quality': quality, 'combined': combined, - 'new_resid': new_resid, 'pred': pred_orig, - 'trig_info': trig_info, - 'j_var': j_var, - }) + candidates.append( + { + "family": fam, + "space": sp, + "action": action, + "quality": quality, + "combined": combined, + "new_resid": new_resid, + "pred": pred_orig, + "trig_info": trig_info, + "j_var": j_var, + } + ) continue # ── non-trig: run original pipeline step ───────────────── # Determine if multifeature applies use_multi = ( - sp in ('log_log', 'log_x', 'log_y', 'log_log_multi') + sp in ("log_log", "log_x", "log_y", "log_log_multi") and X_t is not None and X_t.ndim == 2 and X_t.shape[1] > 1 @@ -411,20 +442,24 @@ def fit(self, X, y): # Reproduce prediction (same as fit_block in v7d) if use_multi: mu_x, s_x, mu_r, s_r, W, b = _fit_ols_multi(X_t, resid_t) - X_std = (X_t - mu_x) / s_x - pred_s = X_std @ W + b - pred_t = pred_s * s_r + mu_r - if sp == 'log_log_multi': + X_std = (X_t - mu_x) / s_x + pred_s = X_std @ W + b + pred_t = pred_s * s_r + mu_r + if sp == "log_log_multi": pred_orig = np.exp(pred_t) else: pred_orig = inverse_transform_y(pred_t, sp) new_resid = resid - pred_orig block_store = { - 'type': 'multi_linear', - 'family': fam, 'space': sp, - 'mu_x': mu_x, 's_x': s_x, - 'mu_r': mu_r, 's_r': s_r, - 'W': W, 'b': b, + "type": "multi_linear", + "family": fam, + "space": sp, + "mu_x": mu_x, + "s_x": s_x, + "mu_r": mu_r, + "s_r": s_r, + "W": W, + "b": b, } else: # 1-D feature block @@ -432,67 +467,77 @@ def fit(self, X, y): if indices is None: continue mn_c, iqr_c, mn_r, iqr_r, w, b = _fit_ols_1d(col, resid_t) - col_n = (col - mn_c) / iqr_c - pred_t = (w * col_n + b) * iqr_r + mn_r + col_n = (col - mn_c) / iqr_c + pred_t = (w * col_n + b) * iqr_r + mn_r pred_orig = inverse_transform_y(pred_t, sp) new_resid = resid - pred_orig block_store = { - 'type': '1d_linear', - 'family': fam, 'space': sp, - 'indices': indices, - 'mn_c': mn_c, 'iqr_c': iqr_c, - 'mn_r': mn_r, 'iqr_r': iqr_r, - 'w': w, 'b': b, + "type": "1d_linear", + "family": fam, + "space": sp, + "indices": indices, + "mn_c": mn_c, + "iqr_c": iqr_c, + "mn_r": mn_r, + "iqr_r": iqr_r, + "w": w, + "b": b, } sc_after, _, _ = best_per_family_multispace(X_np, new_resid, y_np) - quality = bloc_score(resid, new_resid, y_np, sc_after) - clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) + quality = bloc_score(resid, new_resid, y_np, sc_after) + clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) combined = quality * (0.6 + 0.4 * clf_w) # Bonuses (identical to v7d) - if sp == 'original' and sc >= 0.97: + if sp == "original" and sc >= 0.97: combined *= 1.5 - elif fam == 'lin' and sp == 'original' and sc >= 0.85: + elif fam == "lin" and sp == "original" and sc >= 0.85: combined *= 1.3 - elif fam == 'pair' and sp == 'original' and sc >= 0.85: + elif fam == "pair" and sp == "original" and sc >= 0.85: combined *= 1.2 - candidates.append({ - 'family': fam, 'space': sp, 'action': action, - 'quality': quality, 'combined': combined, - 'new_resid': new_resid, 'pred': pred_orig, - 'block_store': block_store, - }) + candidates.append( + { + "family": fam, + "space": sp, + "action": action, + "quality": quality, + "combined": combined, + "new_resid": new_resid, + "pred": pred_orig, + "block_store": block_store, + } + ) if not candidates: break - best = max(candidates, key=lambda c: c['combined']) - if best['combined'] < MIN_SCORE: + best = max(candidates, key=lambda c: c["combined"]) + if best["combined"] < MIN_SCORE: break # Commit this block - if best['family'] in ('sin', 'cos'): - ti = best['trig_info'] + if best["family"] in ("sin", "cos"): + ti = best["trig_info"] blk = { - 'type': 'trig', - 'j_var': best['j_var'], - 'form': ti.get('form', best['family']), - 'A': ti.get('A', 0.0), - 'omega': ti.get('omega', 0.0), - 'phi': ti.get('phi', 0.0), - 'C': ti.get('C', 0.0), - 'B': ti.get('B', 0.0), + "type": "trig", + "j_var": best["j_var"], + "form": ti.get("form", best["family"]), + "A": ti.get("A", 0.0), + "omega": ti.get("omega", 0.0), + "phi": ti.get("phi", 0.0), + "C": ti.get("C", 0.0), + "B": ti.get("B", 0.0), } self.fitted_blocks_.append(blk) else: - self.fitted_blocks_.append(best['block_store']) + self.fitted_blocks_.append(best["block_store"]) self.chosen_.append(f"{best['family']}[{best['space']}]") - resid = best['new_resid'] + resid = best["new_resid"] - if best['quality'] > 0.80: + if best["quality"] > 0.80: break return self @@ -500,24 +545,24 @@ def fit(self, X, y): # ── predict ────────────────────────────────────────────────────────────── def predict(self, X): - if not hasattr(self, 'fitted_blocks_') or not self.fitted_blocks_: + if not hasattr(self, "fitted_blocks_") or not self.fitted_blocks_: # No blocks fitted — return zeros (will produce R²<0 on test) - n = len(X) if hasattr(X, '__len__') else 1 + n = len(X) if hasattr(X, "__len__") else 1 return np.zeros(n) X_np = self._to_numpy(X) - acc = np.zeros(len(X_np)) + acc = np.zeros(len(X_np)) for block in self.fitted_blocks_: - t = block['type'] + t = block["type"] try: - if t == 'power_law': + if t == "power_law": acc += _predict_power_law(block, X_np) - elif t == 'trig': + elif t == "trig": acc += _predict_trig(block, X_np) - elif t == '1d_linear': + elif t == "1d_linear": acc += _predict_1d_linear(block, X_np) - elif t == 'multi_linear': + elif t == "multi_linear": acc += _predict_multi_linear(block, X_np) except Exception: pass # drop silently — partial sum is still meaningful @@ -527,7 +572,7 @@ def predict(self, X): # ── sklearn interface ───────────────────────────────────────────────────── def get_params(self, deep=True): - return {'max_time': self.max_time, 'random_state': self.random_state} + return {"max_time": self.max_time, "random_state": self.random_state} def set_params(self, **params): for k, v in params.items(): @@ -535,7 +580,7 @@ def set_params(self, **params): return self def __repr__(self): - chosen = getattr(self, 'chosen_', []) + chosen = getattr(self, "chosen_", []) return f"PantaraRegressor(blocks={chosen})" @@ -545,12 +590,12 @@ def __repr__(self): est = PantaraRegressor(max_time=3600, random_state=42) -hyper_params = [{}] # no hyperparameter search for now +hyper_params: list[dict[str, Any]] = [{}] # no hyperparameter search for now # Skip StandardScaler: Pantara needs raw (positive) physical values. eval_kwargs = { - 'scale_x': False, - 'scale_y': False, + "scale_x": False, + "scale_y": False, } @@ -568,11 +613,11 @@ def model(est, X=None): """ import sympy as sp - blocks = getattr(est, 'fitted_blocks_', []) + blocks = getattr(est, "fitted_blocks_", []) if not blocks: return None - f_names = getattr(est, 'feature_names_', None) + f_names = getattr(est, "feature_names_", None) if (not f_names) and X is not None: try: f_names = list(X.columns) @@ -582,23 +627,23 @@ def model(est, X=None): def feat(j): if f_names is not None and j < len(f_names): return sp.Symbol(str(f_names[j])) - return sp.Symbol(f'x{j}') + return sp.Symbol(f"x{j}") def xt(j, space): """Feature j after the block's input-space transform (cf. transform_space).""" x = feat(j) - if space in ('log_x', 'log_log', 'log_log_multi'): + if space in ("log_x", "log_log", "log_log_multi"): return sp.log(x) - if space == 'inv_x': + if space == "inv_x": return 1 / x # original, log_y, sq_y leave X untouched return x def inv_y(expr, space): """Invert the y-space transform (cf. inverse_transform_y).""" - if space in ('log_y', 'log_log', 'log_log_multi'): + if space in ("log_y", "log_log", "log_log_multi"): return sp.exp(expr) - if space == 'sq_y': + if space == "sq_y": return sp.sqrt(sp.Abs(expr)) # original, log_x, inv_x return expr @@ -606,69 +651,76 @@ def inv_y(expr, space): def feature_col(family, indices, space): """Symbolic column for a 1-D block family (cf. _compute_feature_col).""" xj = xt(indices[0], space) - if family == 'lin': + if family == "lin": return xj - if family == 'sq': - return xj ** 2 - if family == 'pair': + if family == "sq": + return xj**2 + if family == "pair": return xj * xt(indices[1], space) - if family == 'qc': + if family == "qc": return xj * xt(indices[1], space) ** 2 - if family == 'trip': + if family == "trip": return xj * xt(indices[1], space) * xt(indices[2], space) - if family == 'qinv': - return xj ** 2 / xt(indices[1], space) + if family == "qinv": + return xj**2 / xt(indices[1], space) return None def block_expr(b): - t = b['type'] + t = b["type"] - if t == 'power_law': - coeffs = b['coeffs'] + if t == "power_law": + coeffs = b["coeffs"] expr = sp.Float(float(np.exp(coeffs[0]))) for j, n in enumerate(coeffs[1:]): if abs(float(n)) > 1e-4: expr = expr * feat(j) ** sp.Float(float(n)) return expr - if t == 'trig': - x = feat(b['j_var']) - A = sp.Float(float(b['A'])); w = sp.Float(float(b['omega'])) - phi = sp.Float(float(b['phi'])); C = sp.Float(float(b['C'])) - B = sp.Float(float(b.get('B', 0.0))) + if t == "trig": + x = feat(b["j_var"]) + A = sp.Float(float(b["A"])) + w = sp.Float(float(b["omega"])) + phi = sp.Float(float(b["phi"])) + C = sp.Float(float(b["C"])) + B = sp.Float(float(b.get("B", 0.0))) arg = w * x + phi - form = b['form'] - if form == 'sin': + form = b["form"] + if form == "sin": return A * sp.sin(arg) + C - if form == 'cos': + if form == "cos": return A * sp.cos(arg) + C - if form == 'sin+lin': + if form == "sin+lin": return A * sp.sin(arg) + B * x + C - if form == 'cos+lin': + if form == "cos+lin": return A * sp.cos(arg) + B * x + C return C - if t == '1d_linear': - space = b['space'] - col = feature_col(b['family'], b['indices'], space) + if t == "1d_linear": + space = b["space"] + col = feature_col(b["family"], b["indices"], space) if col is None: return sp.Integer(0) - col_n = (col - sp.Float(float(b['mn_c']))) / sp.Float(float(b['iqr_c']) + 1e-10) - pred_t = (sp.Float(float(b['w'])) * col_n + sp.Float(float(b['b']))) \ - * sp.Float(float(b['iqr_r'])) + sp.Float(float(b['mn_r'])) + col_n = (col - sp.Float(float(b["mn_c"]))) / sp.Float( + float(b["iqr_c"]) + 1e-10 + ) + pred_t = ( + sp.Float(float(b["w"])) * col_n + sp.Float(float(b["b"])) + ) * sp.Float(float(b["iqr_r"])) + sp.Float(float(b["mn_r"])) return inv_y(pred_t, space) - if t == 'multi_linear': - space = b['space'] - mu_x, s_x, W = b['mu_x'], b['s_x'], b['W'] + if t == "multi_linear": + space = b["space"] + mu_x, s_x, W = b["mu_x"], b["s_x"], b["W"] n_feats = len(f_names) if f_names is not None else len(W) - pred_s = sp.Float(float(b['b'])) + pred_s = sp.Float(float(b["b"])) for j in range(len(W)): # predict() zero-pads transformed features beyond the input width x_t = xt(j, space) if j < n_feats else sp.Integer(0) - x_std = (x_t - sp.Float(float(mu_x[j]))) / sp.Float(float(s_x[j]) + 1e-10) + x_std = (x_t - sp.Float(float(mu_x[j]))) / sp.Float( + float(s_x[j]) + 1e-10 + ) pred_s = pred_s + sp.Float(float(W[j])) * x_std - pred_t = pred_s * sp.Float(float(b['s_r'])) + sp.Float(float(b['mu_r'])) + pred_t = pred_s * sp.Float(float(b["s_r"])) + sp.Float(float(b["mu_r"])) return inv_y(pred_t, space) return sp.Integer(0) @@ -687,4 +739,4 @@ def block_expr(b): def complexity(est): """Number of basis blocks chosen.""" - return len(getattr(est, 'fitted_blocks_', [])) + return len(getattr(est, "fitted_blocks_", [])) diff --git a/experiment/methods/pir/regressor.py b/experiment/methods/pir/regressor.py index 960c5d6c..1f832c7a 100644 --- a/experiment/methods/pir/regressor.py +++ b/experiment/methods/pir/regressor.py @@ -25,17 +25,15 @@ import numpy as np -from sklearn.base import BaseEstimator, RegressorMixin - # The classical engine. algorithms/pir/install.sh pip-installs this from the # public repo at tag v3.4.1. from physics_engine.sklearn_adapter import PIRRegressor - +from sklearn.base import BaseEstimator, RegressorMixin # --- Vanilla config that produced the verified blind Tier A result ------------ _PIR_VANILLA_CONFIG = dict( - enforce_dimensions=False, # blind sweep ran with dim-filter OFF - allowed_powers=[1, 2], # powers 1,2 only (structural cap) + enforce_dimensions=False, # blind sweep ran with dim-filter OFF + allowed_powers=[1, 2], # powers 1,2 only (structural cap) include_pairwise_products=True, # pairwise on; no 3-var assembly use_ransac=True, use_residual=True, @@ -122,20 +120,18 @@ def model(est, X=None): unchanged. Otherwise generic positional names (x_0, x0, X0, ...) are remapped to the dataset columns. """ - expr = ( - est.model() - if hasattr(est, "model") - else str(getattr(est, "expr_", "0.0")) - ) + expr = est.model() if hasattr(est, "model") else str(getattr(est, "expr_", "0.0")) if X is None or not hasattr(X, "columns"): return expr cols = list(X.columns) + # Word-boundary match, not substring: a column literally named "co" # must not match inside "cos(x_0)" (per gAldeia's review on PR #210). def _col_appears(col_name, s): return re.search(r"\b" + re.escape(str(col_name)) + r"\b", s) is not None + # If any real column name already appears, assume names are correct. if any(_col_appears(c, expr) for c in cols): return expr diff --git a/experiment/methods/ps-tree/regressor.py b/experiment/methods/ps-tree/regressor.py index b47c17d6..5c9e8d4c 100644 --- a/experiment/methods/ps-tree/regressor.py +++ b/experiment/methods/ps-tree/regressor.py @@ -1,10 +1,10 @@ -from sklearn.tree import DecisionTreeRegressor -from deap.tools import selRandom -from pstree.cluster_gp_sklearn import selTournamentDCD -from pstree.cluster_gp_sklearn import GPRegressor, PSTreeRegressor -from pstree.complexity_utils import tree_gp_regressor_complexity +from __future__ import annotations + +from typing import Any -from sklearn.model_selection import ParameterSampler +from pstree.cluster_gp_sklearn import GPRegressor, PSTreeRegressor, selTournamentDCD +from pstree.complexity_utils import tree_gp_regressor_complexity +from sklearn.tree import DecisionTreeRegressor # hyper_params = list(ParameterSampler({ # 'basic_primitive': ['optimal', 'log,sqrt,sin,tanh'], @@ -14,41 +14,56 @@ # I ran the code above and got the hyper-parameters below. # I am hardcoding it now so it is explicit here. -# I also had to wrap the parameters in lists, and it seems that there is a +# I also had to wrap the parameters in lists, and it seems that there is a # missconfiguration in initial_height hyper_params = [ - {'max_leaf_nodes': [4], - # 'initial_height': [None], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [8], - # 'initial_height': ['0-6'], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [6], - # 'initial_height': ['0-6'], - 'basic_primitive': ['optimal']}, - - {'max_leaf_nodes': [6], - # 'initial_height': ['0-6'], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [8], - # 'initial_height': [None], - 'basic_primitive': ['optimal']}, - - {'max_leaf_nodes': [8], - # 'initial_height': [None], - 'basic_primitive': ['log,sqrt,sin,tanh']} + { + "max_leaf_nodes": [4], + # 'initial_height': [None], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': ['0-6'], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [6], + # 'initial_height': ['0-6'], + "basic_primitive": ["optimal"], + }, + { + "max_leaf_nodes": [6], + # 'initial_height': ['0-6'], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': [None], + "basic_primitive": ["optimal"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': [None], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, ] # for g in hyper_params: # g['select'] = [selRandom] if g['initial_height'] is None else [selTournamentDCD] -est = PSTreeRegressor(regr_class=GPRegressor, tree_class=DecisionTreeRegressor, - height_limit=6, n_pop=25, n_gen=1_000, - normalize=True, basic_primitive='optimal', - select=selTournamentDCD, size_objective=False, afp=True) +est = PSTreeRegressor( + regr_class=GPRegressor, + tree_class=DecisionTreeRegressor, + height_limit=6, + n_pop=25, + n_gen=1_000, + normalize=True, + basic_primitive="optimal", + select=selTournamentDCD, + size_objective=False, + afp=True, +) def complexity(est): @@ -60,4 +75,4 @@ def model(e, X=None): return e.model() -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/pysr/regressor.py b/experiment/methods/pysr/regressor.py index 9067021c..2cda0e39 100644 --- a/experiment/methods/pysr/regressor.py +++ b/experiment/methods/pysr/regressor.py @@ -1,30 +1,30 @@ from pysr import PySRRegressor -from multiprocessing import cpu_count hyper_params = [ { - 'niterations' : [1_000_000_000], - 'maxsize' : [20], - 'maxdepth' : [20], + "niterations": [1_000_000_000], + "maxsize": [20], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [30], - 'maxdepth' : [20], + "niterations": [1_000_000], + "maxsize": [30], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [50], - 'maxdepth' : [20], + "niterations": [1_000_000], + "maxsize": [50], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [30], - 'maxdepth' : [20], - 'population_size' : [200], + "niterations": [1_000_000], + "maxsize": [30], + "maxdepth": [20], + "population_size": [200], }, ] + def complexity(est): return int(est.get_best()["complexity"]) @@ -37,11 +37,11 @@ def model(est, X=None): niterations=1_000_000_000, ncyclesperiteration=2_500, population_size=100, - populations=2, # originally max(15, 2*n_cpu), but srbench runs on a single core + populations=2, # originally max(15, 2*n_cpu), but srbench runs on a single core verbosity=0, # budget 10 minutes for compile time, # ensuring we can finish within 2 hours: - timeout_in_seconds=60*60 - 10*60, + timeout_in_seconds=60 * 60 - 10 * 60, maxsize=30, maxdepth=20, binary_operators=["+", "-", "*", "/"], @@ -53,7 +53,7 @@ def model(est, X=None): log=9, sqrt=9, ), - **{"/": (-1, 9)} + **{"/": (-1, 9)}, }, nested_constraints=dict( sin=dict( @@ -72,10 +72,10 @@ def model(est, X=None): ), sqrt=dict( sqrt=0, - ) + ), ), # prefer multiprocessing: - procs=1, # cpu_count(), + procs=1, # cpu_count(), multithreading=False, batching=True, batch_size=50, diff --git a/experiment/methods/qlattice/regressor.py b/experiment/methods/qlattice/regressor.py index 8d7dd7b8..13ab18f6 100644 --- a/experiment/methods/qlattice/regressor.py +++ b/experiment/methods/qlattice/regressor.py @@ -4,7 +4,12 @@ import numpy as np import pandas as pd from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_X_y, check_array, check_is_fitted, _check_sample_weight +from sklearn.utils.validation import ( + _check_sample_weight, + check_array, + check_is_fitted, + check_X_y, +) from sympy.printing.printer import Printer @@ -13,26 +18,27 @@ class InternalTimeOutException(Exception): def alarm_handler(signum, frame): - print(f"raising InternalTimeOutException") + print("raising InternalTimeOutException") raise InternalTimeOutException -def auto_run_time(ql, - data, - output_name, - kind="regression", - stypes=None, - n_epochs=200, - threads="auto", - max_complexity=10, - query_string=None, - loss_function=None, - criterion="wide_parsimony", - sample_weights=None, - function_names=None, - starting_models=None, - max_time=None - ): +def auto_run_time( + ql, + data, + output_name, + kind="regression", + stypes=None, + n_epochs=200, + threads="auto", + max_complexity=10, + query_string=None, + loss_function=None, + criterion="wide_parsimony", + sample_weights=None, + function_names=None, + starting_models=None, + max_time=None, +): if max_time: signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(max_time) @@ -84,7 +90,7 @@ def auto_run_time(ql, best = feyn.get_diverse_models(models) except InternalTimeOutException: - print('InternalTimeOutException raised') + print("InternalTimeOutException raised") best = feyn.get_diverse_models(models) return best @@ -92,21 +98,21 @@ def auto_run_time(ql, def numpy_check_to_DataFrame(X, y=None): if isinstance(X, np.ndarray): - columns = [f'x__{i}' for i in range(X.shape[1])] + columns = [f"x__{i}" for i in range(X.shape[1])] X = pd.DataFrame(X, columns=columns) if y is None: return X elif isinstance(y, np.ndarray): if y.ndim == 1: - y = pd.Series(y, name='target') + y = pd.Series(y, name="target") elif y.ndim == 2: - y = pd.DataFrame(y, columns=['target']) + y = pd.DataFrame(y, columns=["target"]) return X, y def feature_names(X, y=None): - if hasattr(X, 'columns'): + if hasattr(X, "columns"): input_names = X.columns else: input_names = None @@ -114,7 +120,7 @@ def feature_names(X, y=None): if y is None: return input_names else: - if hasattr(y, 'name'): + if hasattr(y, "name"): output_name = y.name else: output_name = None @@ -122,21 +128,22 @@ def feature_names(X, y=None): class QLatticeRegressor(BaseEstimator, RegressorMixin): - def __init__(self, - kind='regression', - stypes=None, - n_epochs=200, - threads='auto', - max_complexity=10, - query_string=None, - loss_function=None, - criterion='wide_parsimony', - sample_weights=None, - function_names=None, - starting_models=None, - random_state=None, - max_time=None - ): + def __init__( + self, + kind="regression", + stypes=None, + n_epochs=200, + threads="auto", + max_complexity=10, + query_string=None, + loss_function=None, + criterion="wide_parsimony", + sample_weights=None, + function_names=None, + starting_models=None, + random_state=None, + max_time=None, + ): self.kind = kind self.stypes = stypes self.n_epochs = n_epochs @@ -177,22 +184,23 @@ def fit(self, X, y, sample_weight=None): sample_weight = _check_sample_weight(sample_weight, X) ql = feyn.QLattice(random_seed=rseed) - self.models_ = auto_run_time(ql=ql, - data=data, - output_name=data.columns[-1], - kind=self.kind, - stypes=self.stypes, - n_epochs=self.n_epochs, - threads=self.threads, - max_complexity=self.max_complexity, - query_string=self.query_string, - loss_function=self.loss_function, - criterion=self.criterion, - sample_weights=sample_weight, - function_names=self.function_names, - starting_models=self.starting_models, - max_time=self.max_time - ) + self.models_ = auto_run_time( + ql=ql, + data=data, + output_name=data.columns[-1], + kind=self.kind, + stypes=self.stypes, + n_epochs=self.n_epochs, + threads=self.threads, + max_complexity=self.max_complexity, + query_string=self.query_string, + loss_function=self.loss_function, + criterion=self.criterion, + sample_weights=sample_weight, + function_names=self.function_names, + starting_models=self.starting_models, + max_time=self.max_time, + ) return self def predict(self, X, n=0): @@ -225,10 +233,10 @@ def score(self, X, y, n=0): est = QLatticeRegressor( - kind='regression', + kind="regression", n_epochs=200, max_complexity=10, - criterion='wide_parsimony', + criterion="wide_parsimony", ) @@ -241,7 +249,7 @@ def model(est, X): mapping = {feyn.tools._sympy.get_sanitized_name(col): col for col in X.columns} for k, v in reversed(mapping.items()): string_model = string_model.replace(k, v) - except: + except Exception: pass return string_model @@ -291,6 +299,7 @@ def pre_train_fn(est, X, y): # define eval_kwargs. eval_kwargs = dict( pre_train=pre_train_fn, - test_params={'n_epochs': 2, - } + test_params={ + "n_epochs": 2, + }, ) diff --git a/experiment/methods/rils-rols/regressor.py b/experiment/methods/rils-rols/regressor.py index 87cc92e2..7eff29ad 100644 --- a/experiment/methods/rils-rols/regressor.py +++ b/experiment/methods/rils-rols/regressor.py @@ -1,43 +1,50 @@ +import sympy as sp from rils_rols.rils_rols import RILSROLSRegressor +from sklearn.base import RegressorMixin from symbolic_utils import complexity as cplx -from sklearn.base import BaseEstimator, RegressorMixin -import sympy as sp -eval_kwargs = {'scale_x': False, 'scale_y': False} +eval_kwargs = {"scale_x": False, "scale_y": False} complexity_penalty = [0.001, 0.01, 0.1] -max_complexity = [25, 50, 100] +max_complexity = [25, 50, 100] hyper_params = [] for c in complexity_penalty: for max_c in max_complexity: - hyper_params.append({ - 'complexity_penalty' : [c], - 'max_complexity' : [max_c], - }) - -est:RegressorMixin = RILSROLSRegressor( - max_time=60*60, - max_fit_calls=1000*1000, + hyper_params.append( + { + "complexity_penalty": [c], + "max_complexity": [max_c], + } + ) + +est: RegressorMixin = RILSROLSRegressor( + max_time=60 * 60, + max_fit_calls=1000 * 1000, max_complexity=50, sample_size=0, - verbose=False) + verbose=False, +) + def model(est, X=None) -> str: if X is None: return str(est.model_string()) else: - mapping = {'x'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x" + str(i): k for i, k in enumerate(X.columns)} new_model = str(est.model_string()) - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) return new_model + def complexity(est): return cplx(sp.parse_expr(str(est.model_string()))) - + + def get_population(est) -> list[RegressorMixin]: return [est] + def get_best_solution(est) -> RegressorMixin: - return est \ No newline at end of file + return est diff --git a/experiment/methods/sklearn/regressor.py b/experiment/methods/sklearn/regressor.py index 303f1e66..30ae1faf 100644 --- a/experiment/methods/sklearn/regressor.py +++ b/experiment/methods/sklearn/regressor.py @@ -2,7 +2,6 @@ from sklearn.linear_model import LinearRegression - est = LinearRegression() hyper_params = [{"fit_intercept": (True, False)}] diff --git a/experiment/methods/sklearn_adaboost/regressor.py b/experiment/methods/sklearn_adaboost/regressor.py index 6e6de625..667c760b 100644 --- a/experiment/methods/sklearn_adaboost/regressor.py +++ b/experiment/methods/sklearn_adaboost/regressor.py @@ -1,16 +1,29 @@ from sklearn import ensemble -hyper_params = [{ - 'learning_rate' : (0.01, 0.1, 1.0, 10.0,), - 'n_estimators' : (10, 100, 1000,), -}] +hyper_params = [ + { + "learning_rate": ( + 0.01, + 0.1, + 1.0, + 10.0, + ), + "n_estimators": ( + 10, + 100, + 1000, + ), + } +] + +est = ensemble.AdaBoostRegressor() -est=ensemble.AdaBoostRegressor() def complexity(est): - size = 0 + size = 0 for i in est.estimators_: size += i.tree_.node_count return size + model = None diff --git a/experiment/methods/sklearn_lasso/regressor.py b/experiment/methods/sklearn_lasso/regressor.py index f47db4df..477fc056 100644 --- a/experiment/methods/sklearn_lasso/regressor.py +++ b/experiment/methods/sklearn_lasso/regressor.py @@ -2,11 +2,17 @@ hyper_params = [ { - 'alpha': (1e-04,0.001,0.01,0.1,1,), + "alpha": ( + 1e-04, + 0.001, + 0.01, + 0.1, + 1, + ), }, ] -est=linear_model.LassoLars() +est = linear_model.LassoLars() complexity = None model = None diff --git a/experiment/methods/sklearn_linear/regressor.py b/experiment/methods/sklearn_linear/regressor.py index c35b2f59..354b0106 100644 --- a/experiment/methods/sklearn_linear/regressor.py +++ b/experiment/methods/sklearn_linear/regressor.py @@ -1,12 +1,12 @@ from sklearn import linear_model hyper_params = [ - { - 'fit_intercept': (True,), - }, + { + "fit_intercept": (True,), + }, ] -est=linear_model.LinearRegression() +est = linear_model.LinearRegression() complexity = None model = None diff --git a/experiment/methods/sklearn_mlp/regressor.py b/experiment/methods/sklearn_mlp/regressor.py index 99268507..7107724b 100644 --- a/experiment/methods/sklearn_mlp/regressor.py +++ b/experiment/methods/sklearn_mlp/regressor.py @@ -1,17 +1,33 @@ -from sklearn.neural_network import MLPRegressor import numpy as np +from sklearn.neural_network import MLPRegressor hyper_params = [ { - 'activation' : ('logistic', 'tanh', 'relu',), - 'solver' : ('lbfgs','adam','sgd',), - 'learning_rate' : ('constant', 'invscaling', 'adaptive',), + "activation": ( + "logistic", + "tanh", + "relu", + ), + "solver": ( + "lbfgs", + "adam", + "sgd", + ), + "learning_rate": ( + "constant", + "invscaling", + "adaptive", + ), }, ] -est=MLPRegressor() +est = MLPRegressor() + def complexity(est): - return np.sum([c.size*2 for c in est.coefs_] - + [c.size*2 for c in est.intercepts_]) + return np.sum( + [c.size * 2 for c in est.coefs_] + [c.size * 2 for c in est.intercepts_] + ) + + model = None diff --git a/experiment/methods/sklearn_randomforest/regressor.py b/experiment/methods/sklearn_randomforest/regressor.py index 24041dde..809b53aa 100644 --- a/experiment/methods/sklearn_randomforest/regressor.py +++ b/experiment/methods/sklearn_randomforest/regressor.py @@ -1,15 +1,20 @@ -from sklearn import ensemble import numpy as np +from sklearn import ensemble + +hyper_params = [ + { + "n_estimators": (10, 100, 1000), + "min_weight_fraction_leaf": (0.0, 0.25, 0.5), + "max_features": ("sqrt", "log2", None), + } +] -hyper_params = [{ - 'n_estimators': (10, 100, 1000), - 'min_weight_fraction_leaf': (0.0, 0.25, 0.5), - 'max_features': ('sqrt','log2',None), -}] +est = ensemble.RandomForestRegressor() -est=ensemble.RandomForestRegressor() def complexity(est): return np.sum([e.tree_.node_count for e in est.estimators_]) + + model = None diff --git a/experiment/methods/sklearn_ridge/regressor.py b/experiment/methods/sklearn_ridge/regressor.py index 5ee9515f..4467474b 100644 --- a/experiment/methods/sklearn_ridge/regressor.py +++ b/experiment/methods/sklearn_ridge/regressor.py @@ -1,21 +1,40 @@ from sklearn import kernel_ridge -hyper_params = [{ - 'kernel': ('linear', 'poly','rbf','sigmoid',), - 'alpha': (1e-4,1e-2,0.1,1,), - 'gamma': (0.01,0.1,1,10,), -}] +hyper_params = [ + { + "kernel": ( + "linear", + "poly", + "rbf", + "sigmoid", + ), + "alpha": ( + 1e-4, + 1e-2, + 0.1, + 1, + ), + "gamma": ( + 0.01, + 0.1, + 1, + 10, + ), + } +] + +est = kernel_ridge.KernelRidge() -est=kernel_ridge.KernelRidge() def complexity(est): # this is a loose lower bound on the model complexity, basically jsut - # capturing the number of parameters in the model. Could make this + # capturing the number of parameters in the model. Could make this # kernel-specific and more accurate. - return est.dual_coef_.size + return est.dual_coef_.size + model = None eval_kwargs = { - 'scale_y': False, - 'scale_x': True, - } + "scale_y": False, + "scale_x": True, +} diff --git a/experiment/methods/sklearn_sgd/regressor.py b/experiment/methods/sklearn_sgd/regressor.py index ff273edb..0e2104b8 100644 --- a/experiment/methods/sklearn_sgd/regressor.py +++ b/experiment/methods/sklearn_sgd/regressor.py @@ -1,14 +1,22 @@ from sklearn import linear_model - hyper_params = [ { - 'alpha': (1e-06,1e-04,0.01,1,), - 'penalty': ('l2','l1','elasticnet',), + "alpha": ( + 1e-06, + 1e-04, + 0.01, + 1, + ), + "penalty": ( + "l2", + "l1", + "elasticnet", + ), }, ] -est=linear_model.SGDRegressor() +est = linear_model.SGDRegressor() complexity = None model = None diff --git a/experiment/methods/tir/regressor.py b/experiment/methods/tir/regressor.py index 29c5295a..e6fd2cbb 100644 --- a/experiment/methods/tir/regressor.py +++ b/experiment/methods/tir/regressor.py @@ -1,58 +1,63 @@ -import sys -import os import pyTIR as tir -from itertools import product -#os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + +# os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" hyper_params = [ { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),), - 'ngens' : (1000,), + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), + "ngens": (1000,), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),), - 'ngens' : (1000,), + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), + "ngens": (1000,), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), }, ] # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = tir.TIRRegressor(npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5,5), error="R^2", alg="MOO") +eval_kwargs = {"scale_x": False, "scale_y": False} +est = tir.TIRRegressor( + npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5, 5), error="R^2", alg="MOO" +) + def pre_train(est, X, y): """Adjust settings based on data before training""" - if X.shape[0]*X.shape[1] <= 1000: + if X.shape[0] * X.shape[1] <= 1000: est.penalty = 0.01 + def complexity(e): return e.len + def model(e, X=None): - new_model = e.sympy.replace("^","**") + new_model = e.sympy.replace("^", "**") if X is not None: - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) return new_model + def get_population(est): pop = [] - for i in range(min(100, len(est.front))-1,-1,-1): + for i in range(min(100, len(est.front)) - 1, -1, -1): pop.append(est.create_model_from(i)) - return pop + return pop + def get_best_solution(est): return est diff --git a/experiment/methods/tpsr/regressor.py b/experiment/methods/tpsr/regressor.py index bbae170e..9f29c565 100644 --- a/experiment/methods/tpsr/regressor.py +++ b/experiment/methods/tpsr/regressor.py @@ -1,28 +1,34 @@ +# handling hardcored paths and no package setup +import sys import time -import json -import torch + import numpy as np import sympy as sp -from sklearn import feature_selection +import torch +from sklearn import feature_selection from sklearn.base import BaseEstimator, RegressorMixin -# handling hardcored paths and no package setup -import sys; sys.path.append("/opt/conda/bin/tpsr") +sys.path.append("/opt/conda/bin/tpsr") # TPSR stuff import symbolicregression -from symbolicregression.envs import build_env -from symbolicregression.model import build_modules -from symbolicregression.e2e_model import Transformer, pred_for_sample_no_refine, refine_for_sample, refine_for_sample_test +from default_pi import E2EHeuristic +from dyna_gym.agents.mcts import update_root from dyna_gym.agents.uct import UCT -from dyna_gym.agents.mcts import update_root, convert_to_json from rl_env import RLEnv -from default_pi import E2EHeuristic +from symbolicregression.e2e_model import ( + Transformer, + refine_for_sample, + refine_for_sample_test, +) +from symbolicregression.envs import build_env +from symbolicregression.model import build_modules + def get_top_k_features(X, y, k=10): - if y.ndim==2: - y=y[:,0] - if X.shape[1]<=k: + if y.ndim == 2: + y = y[:, 0] + if X.shape[1] <= k: return [i for i in range(X.shape[1])] else: kbest = feature_selection.SelectKBest(feature_selection.r_regression, k=k) @@ -32,256 +38,271 @@ def get_top_k_features(X, y, k=10): print("keeping only the top-{} features. Order was {}".format(k, top_features)) return list(top_features[:k]) -class TPRS_params(): + +class TPRS_params: def __init__(self): - self.ablation_to_keep=None - self.accumulate_gradients=1 - self.actor_lr=1e-06 - self.alg='mcts' - self.amp=-1 - self.attention_dropout=0 - self.backbone_model='e2e' - self.batch_load=False - self.batch_size=1 - self.batch_size_eval=64 - self.beam_early_stopping=True - self.beam_eval=True - self.beam_eval_train=0 - self.beam_length_penalty=1 - self.beam_selection_metrics=1 - self.beam_size=1 - self.beam_temperature=0.1 - self.beam_type='sampling' - self.clip_grad_norm=0.5 - self.collate_queue_size=2000 - self.cpu=True - self.critic_lr=1e-05 - self.debug=False - self.debug_slurm=False - self.debug_train_statistics=False - self.dec_emb_dim=512 - self.dec_positional_embeddings='learnable' - self.device='gpu' - self.dropout=0 - self.dump_path='' - self.emb_emb_dim=64 - self.emb_expansion_factor=1 - self.embedder_type='LinearPoint' - self.enc_emb_dim=512 - self.enc_positional_embeddings=None - self.enforce_dim=True - self.entropy_coef=0.01 - self.entropy_weighted_strategy='none' - self.env_base_seed=0 - self.env_name='functions' - self.eval_data='' - self.eval_dump_path=None - self.eval_from_exp='' - self.eval_in_domain=False - self.eval_input_length_modulo=-1 - self.eval_mcts_in_domain=False - self.eval_mcts_on_pmlb=False - self.eval_noise=0 - self.eval_noise_gamma=0.0 - self.eval_noise_type='additive' - self.eval_on_pmlb=False - self.eval_only=True - self.eval_size=10000 - self.eval_verbose=0 - self.eval_verbose_print=False - self.exp_id='' - self.exp_name='debug' - self.export_data=False - self.extra_binary_operators='' - self.extra_constants=None - self.extra_unary_operators='' - self.float_precision=3 - self.fp16=False - self.gpu_to_use='0' - self.horizon=200 - self.kl_coef=0.01 - self.kl_regularizer=0.001 - self.lam=0.1 - self.local_rank=-1 - self.lr=1e-05 - self.lr_patience=100 - self.mantissa_len=1 - self.master_port=-1 - self.max_binary_ops_offset=4 - self.max_binary_ops_per_dim=1 - self.max_centroids=10 - self.max_epoch=100000 - self.max_exponent=100 - self.max_exponent_prefactor=1 - self.max_generated_output_len=200 - self.max_input_dimension=10 - self.max_input_points=200 - self.max_int=10 - self.max_len=200 - self.max_number_bags=10 - self.max_output_dimension=1 - self.max_src_len=200 - self.max_target_len=200 - self.max_token_len=0 - self.max_trials=1 - self.max_unary_depth=6 - self.max_unary_ops=4 - self.min_binary_ops_per_dim=0 - self.min_input_dimension=1 - self.min_len_per_dim=5 - self.min_op_prob=0.01 - self.min_output_dimension=1 - self.min_unary_ops=0 - self.n_dec_heads=16 - self.n_dec_hidden_layers=1 - self.n_dec_layers=16 - self.n_emb_layers=1 - self.n_enc_heads=16 - self.n_enc_hidden_layers=1 - self.n_enc_layers=2 - self.n_prediction_points=200 - self.n_steps_per_epoch=3000 - self.n_trees_to_refine=10 - self.n_words=10292 - self.no_prefix_cache=True # setting these will disable cache - self.no_seq_cache=True - self.norm_attention=False - self.num_beams=1 - self.num_workers=10 - self.nvidia_apex=False - self.operators_to_downsample='div_0,arcsin_0,arccos_0,tan_0.2,arctan_0.2,sqrt_5,pow2_3,inv_3' - self.operators_to_not_repeat='' - self.optimizer='adam_inverse_sqrt,warmup_updates=10000' - self.pad_to_max_dim=True - self.pmlb_data_type='feynman' - self.prediction_sigmas='1,2,4,8,16' - self.print_freq=100 - self.prob_const=0.0 - self.prob_rand=0.0 - self.queue_strategy=None - self.reduce_num_constants=True - self.refinements_types='method=BFGS_batchsize=256_metric=/_mse' - self.reload_checkpoint='' - self.reload_data='' - self.reload_model='' - self.reload_size=-1 - self.required_operators='' - self.rescale=True - self.reward_coef=1 - self.reward_type='nmse' - self.rl_alg='ppo' - self.rollout=3 - self.run_id=1 - self.sample_only=False - self.save_eval_dic='./eval_result' - self.save_model=False - self.save_periodic=25 - self.save_results=False - self.seed=23 - self.share_inout_emb=True - self.simplify=False - self.stopping_criterion='' - self.target_kl=1 - self.target_noise=0.0 - self.tasks='functions' - self.test_env_seed=1 - self.tokens_per_batch=10000 - self.train_noise_gamma=0.0 - self.train_value=False - self.ts_mode='best' - self.ucb_base=10.0 - self.ucb_constant=1.0 - self.uct_alg='uct' - self.update_modules='all' - self.use_abs=False - self.use_controller=True - self.use_skeleton=False - self.use_sympy=False - self.validation_metrics='r2_zero,r2,accuracy_l1_biggio,accuracy_l1_1e-3,accuracy_l1_1e-2,accuracy_l1_1e-1,_complexity' - self.vf_coef=0.0001 - self.warmup_epoch=5 - self.width=3 - self.windows=False - + self.cache = "" + self.ablation_to_keep = None + self.accumulate_gradients = 1 + self.actor_lr = 1e-06 + self.alg = "mcts" + self.amp = -1 + self.attention_dropout = 0 + self.backbone_model = "e2e" + self.batch_load = False + self.batch_size = 1 + self.batch_size_eval = 64 + self.beam_early_stopping = True + self.beam_eval = True + self.beam_eval_train = 0 + self.beam_length_penalty = 1 + self.beam_selection_metrics = 1 + self.beam_size = 1 + self.beam_temperature = 0.1 + self.beam_type = "sampling" + self.clip_grad_norm = 0.5 + self.collate_queue_size = 2000 + self.cpu = True + self.critic_lr = 1e-05 + self.debug = False + self.debug_slurm = False + self.debug_train_statistics = False + self.dec_emb_dim = 512 + self.dec_positional_embeddings = "learnable" + self.device = "gpu" + self.dropout = 0 + self.dump_path = "" + self.emb_emb_dim = 64 + self.emb_expansion_factor = 1 + self.embedder_type = "LinearPoint" + self.enc_emb_dim = 512 + self.enc_positional_embeddings = None + self.enforce_dim = True + self.entropy_coef = 0.01 + self.entropy_weighted_strategy = "none" + self.env_base_seed = 0 + self.env_name = "functions" + self.eval_data = "" + self.eval_dump_path = None + self.eval_from_exp = "" + self.eval_in_domain = False + self.eval_input_length_modulo = -1 + self.eval_mcts_in_domain = False + self.eval_mcts_on_pmlb = False + self.eval_noise = 0 + self.eval_noise_gamma = 0.0 + self.eval_noise_type = "additive" + self.eval_on_pmlb = False + self.eval_only = True + self.eval_size = 10000 + self.eval_verbose = 0 + self.eval_verbose_print = False + self.exp_id = "" + self.exp_name = "debug" + self.export_data = False + self.extra_binary_operators = "" + self.extra_constants = None + self.extra_unary_operators = "" + self.float_precision = 3 + self.fp16 = False + self.gpu_to_use = "0" + self.horizon = 200 + self.kl_coef = 0.01 + self.kl_regularizer = 0.001 + self.lam = 0.1 + self.local_rank = -1 + self.lr = 1e-05 + self.lr_patience = 100 + self.mantissa_len = 1 + self.master_port = -1 + self.max_binary_ops_offset = 4 + self.max_binary_ops_per_dim = 1 + self.max_centroids = 10 + self.max_epoch = 100000 + self.max_exponent = 100 + self.max_exponent_prefactor = 1 + self.max_generated_output_len = 200 + self.max_input_dimension = 10 + self.max_input_points = 200 + self.max_int = 10 + self.max_len = 200 + self.max_number_bags = 10 + self.max_output_dimension = 1 + self.max_src_len = 200 + self.max_target_len = 200 + self.max_token_len = 0 + self.max_trials = 1 + self.max_unary_depth = 6 + self.max_unary_ops = 4 + self.min_binary_ops_per_dim = 0 + self.min_input_dimension = 1 + self.min_len_per_dim = 5 + self.min_op_prob = 0.01 + self.min_output_dimension = 1 + self.min_unary_ops = 0 + self.n_dec_heads = 16 + self.n_dec_hidden_layers = 1 + self.n_dec_layers = 16 + self.n_emb_layers = 1 + self.n_enc_heads = 16 + self.n_enc_hidden_layers = 1 + self.n_enc_layers = 2 + self.n_prediction_points = 200 + self.n_steps_per_epoch = 3000 + self.n_trees_to_refine = 10 + self.n_words = 10292 + self.no_prefix_cache = True # setting these will disable cache + self.no_seq_cache = True + self.norm_attention = False + self.num_beams = 1 + self.num_workers = 10 + self.nvidia_apex = False + self.operators_to_downsample = ( + "div_0,arcsin_0,arccos_0,tan_0.2,arctan_0.2,sqrt_5,pow2_3,inv_3" + ) + self.operators_to_not_repeat = "" + self.optimizer = "adam_inverse_sqrt,warmup_updates=10000" + self.pad_to_max_dim = True + self.pmlb_data_type = "feynman" + self.prediction_sigmas = "1,2,4,8,16" + self.print_freq = 100 + self.prob_const = 0.0 + self.prob_rand = 0.0 + self.queue_strategy = None + self.reduce_num_constants = True + self.refinements_types = "method=BFGS_batchsize=256_metric=/_mse" + self.reload_checkpoint = "" + self.reload_data = "" + self.reload_model = "" + self.reload_size = -1 + self.required_operators = "" + self.rescale = True + self.reward_coef = 1 + self.reward_type = "nmse" + self.rl_alg = "ppo" + self.rollout = 3 + self.run_id = 1 + self.sample_only = False + self.save_eval_dic = "./eval_result" + self.save_model = False + self.save_periodic = 25 + self.save_results = False + self.seed = 23 + self.share_inout_emb = True + self.simplify = False + self.stopping_criterion = "" + self.target_kl = 1 + self.target_noise = 0.0 + self.tasks = "functions" + self.test_env_seed = 1 + self.tokens_per_batch = 10000 + self.train_noise_gamma = 0.0 + self.train_value = False + self.ts_mode = "best" + self.ucb_base = 10.0 + self.ucb_constant = 1.0 + self.uct_alg = "uct" + self.update_modules = "all" + self.use_abs = False + self.use_controller = True + self.use_skeleton = False + self.use_sympy = False + self.validation_metrics = "r2_zero,r2,accuracy_l1_biggio,accuracy_l1_1e-3,accuracy_l1_1e-2,accuracy_l1_1e-1,_complexity" + self.vf_coef = 0.0001 + self.warmup_epoch = 5 + self.width = 3 + self.windows = False + + class TPSRRegressor(BaseEstimator, RegressorMixin): def __init__(self, random_state=42): - + self.tpsr_params = TPRS_params() self.random_state = random_state self.tpsr_params.seed = self.random_state - + def fit(self, X, y): - + np.random.seed(self.tpsr_params.seed) torch.manual_seed(self.tpsr_params.seed) torch.cuda.manual_seed(self.tpsr_params.seed) # manually setting the hyperparameters - self.tpsr_params.cpu = False if torch.cuda.is_available() else True - self.tpsr_params.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.tpsr_params.cpu = False if torch.cuda.is_available() else True + self.tpsr_params.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) symbolicregression.utils.CUDA = self.tpsr_params.device - if (self.tpsr_params.no_prefix_cache==False) and (self.tpsr_params.no_seq_cache == False): - self.tpsr_params.cache = 'b' #both cachings are arctivated - elif (self.tpsr_params.no_prefix_cache==True) and (self.tpsr_params.no_seq_cache ==False): - self.tpsr_params.cache = 's' #only sequence caching - elif (self.tpsr_params.no_prefix_cache==False) and (self.tpsr_params.no_seq_cache==True): - self.tpsr_params.cache = 'k' #only top-k caching + if (not self.tpsr_params.no_prefix_cache) and ( + not self.tpsr_params.no_seq_cache + ): + self.tpsr_params.cache = "b" # both cachings are arctivated + elif (self.tpsr_params.no_prefix_cache) and (not self.tpsr_params.no_seq_cache): + self.tpsr_params.cache = "s" # only sequence caching + elif (not self.tpsr_params.no_prefix_cache) and (self.tpsr_params.no_seq_cache): + self.tpsr_params.cache = "k" # only top-k caching else: - self.tpsr_params.cache = 'n' # no caching + self.tpsr_params.cache = "n" # no caching - self.tpsr_params.backbone_model="e2e" + self.tpsr_params.backbone_model = "e2e" self.equation_env = build_env(self.tpsr_params) - modules = build_modules(self.equation_env, self.tpsr_params) + build_modules(self.equation_env, self.tpsr_params) # print(self.tpsr_params) # pre-processing the data to handle up to 10 features - - top_k_features = get_top_k_features(X, y, k=self.tpsr_params.max_input_dimension) + + top_k_features = get_top_k_features( + X, y, k=self.tpsr_params.max_input_dimension + ) X = X[:, top_k_features] - samples = {'x_to_fit': [X], 'y_to_fit':[y], - 'x_to_pred':0, 'y_to_pred':0} + samples = {"x_to_fit": [X], "y_to_fit": [y], "x_to_pred": 0, "y_to_pred": 0} # --- main --- # Transformer loads the pre trained - model = Transformer(params = self.tpsr_params, env=self.equation_env, samples=samples) - model.to(self.tpsr_params.device) + model = Transformer( + params=self.tpsr_params, env=self.equation_env, samples=samples + ) + model.to(self.tpsr_params.device) rl_env = RLEnv( - params = self.tpsr_params, - samples = samples, - equation_env = self.equation_env, - model = model) + params=self.tpsr_params, + samples=samples, + equation_env=self.equation_env, + model=model, + ) dp = E2EHeuristic( - equation_env=self.equation_env, - rl_env=rl_env, - model=model, - k=self.tpsr_params.width, - num_beams=self.tpsr_params.num_beams, - horizon=self.tpsr_params.horizon, - device=self.tpsr_params.device, - use_seq_cache=not self.tpsr_params.no_seq_cache, - use_prefix_cache=not self.tpsr_params.no_prefix_cache, - length_penalty = self.tpsr_params.beam_length_penalty, - train_value_mode=self.tpsr_params.train_value, - debug=self.tpsr_params.debug) + equation_env=self.equation_env, + rl_env=rl_env, + model=model, + k=self.tpsr_params.width, + num_beams=self.tpsr_params.num_beams, + horizon=self.tpsr_params.horizon, + device=self.tpsr_params.device, + use_seq_cache=not self.tpsr_params.no_seq_cache, + use_prefix_cache=not self.tpsr_params.no_prefix_cache, + length_penalty=self.tpsr_params.beam_length_penalty, + train_value_mode=self.tpsr_params.train_value, + debug=self.tpsr_params.debug, + ) start = time.time() agent = UCT( - action_space=[], # this will not be used as we have a default policy - gamma=1., # no discounting - ucb_constant=1., + action_space=[], # this will not be used as we have a default policy + gamma=1.0, # no discounting + ucb_constant=1.0, horizon=self.tpsr_params.horizon, rollouts=self.tpsr_params.rollout, dp=dp, width=self.tpsr_params.width, reuse_tree=True, alg=self.tpsr_params.uct_alg, - ucb_base=self.tpsr_params.ucb_base) + ucb_base=self.tpsr_params.ucb_base, + ) agent.display() if self.tpsr_params.sample_only: @@ -289,16 +310,17 @@ def fit(self, X, y): # programs are saved in samples json file horizon = 1 else: - horizon = 200 - + horizon = 200 + done = False s = rl_env.state - ret_all = [] for t in range(horizon): if len(s) >= self.tpsr_params.horizon: - print(f'Cannot process programs longer than {self.tpsr_params.horizon}. Stop here.') + print( + f"Cannot process programs longer than {self.tpsr_params.horizon}. Stop here." + ) break if done: @@ -307,42 +329,43 @@ def fit(self, X, y): act = agent.act(rl_env, done) s, r, done, _ = rl_env.step(act) - if t ==0: - real_root = agent.root - update_root(agent, act, s) dp.update_cache(s) - + time_elapsed = time.time() - start - print('TPSR+E2E Time Elapsed:', time_elapsed) + print("TPSR+E2E Time Elapsed:", time_elapsed) # TPSR+E2E NMSE # y_mcts , mcts_str , mcts_tree = pred_for_sample_no_refine( # model, self.equation_env, s ,samples['x_to_fit']) - + # TPSR+E2E NMSE after Refine - y_mcts , mcts_str , mcts_tree = refine_for_sample( - self.tpsr_params, model, self.equation_env, s, - samples['x_to_fit'], samples['y_to_fit']) + y_mcts, mcts_str, mcts_tree = refine_for_sample( + self.tpsr_params, + model, + self.equation_env, + s, + samples["x_to_fit"], + samples["y_to_fit"], + ) self.mcts_str_ = mcts_str - self.tree_ = mcts_tree - self.model_ = model - self.X_ = X - self.y_ = y - self.s_ = s - - return self + self.tree_ = mcts_tree + self.model_ = model + self.X_ = X + self.y_ = y + self.s_ = s + return self def predict(self, X): # y_mcts , _ , _ = pred_for_sample_no_refine( # self.model_, self.equation_env, self.s_ ,[X]) - - _, y_mcts , _ , _ = refine_for_sample_test( - self.model_, self.equation_env, self.s_, - [self.X_], [self.y_], [X]) - + + _, y_mcts, _, _ = refine_for_sample_test( + self.model_, self.equation_env, self.s_, [self.X_], [self.y_], [X] + ) + return y_mcts @@ -351,10 +374,10 @@ def predict(self, X): def model(est, X=None): mcts_str = est.mcts_str_ - + replace_ops = {"add": "+", "mul": "*", "sub": "-", "pow": "**", "inv": "1/"} for op, replace_op in replace_ops.items(): - mcts_str = mcts_str.replace(op,replace_op) + mcts_str = mcts_str.replace(op, replace_op) mcts_eq = sp.parse_expr(mcts_str) @@ -364,4 +387,4 @@ def model(est, X=None): # python regressor.py # define eval_kwargs. -eval_kwargs = dict(use_dataframe=False) \ No newline at end of file +eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/tuned/AFPRegressor.py b/experiment/methods/tuned/AFPRegressor.py index bfecfa19..c67c4c57 100644 --- a/experiment/methods/tuned/AFPRegressor.py +++ b/experiment/methods/tuned/AFPRegressor.py @@ -1,11 +1,25 @@ -from ..AFPRegressor import complexity,model, est -from ellyn import ellyn +from ..AFPRegressor import complexity as complexity +from ..AFPRegressor import est as est +from ..AFPRegressor import model as model from .params._afpregressor import params - est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) -est.time_limit = 8*60*60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/AIFeynman.py b/experiment/methods/tuned/AIFeynman.py index c1664d96..660c807b 100644 --- a/experiment/methods/tuned/AIFeynman.py +++ b/experiment/methods/tuned/AIFeynman.py @@ -1,7 +1,8 @@ -from ..AIFeynman import complexity, model, est +from ..AIFeynman import complexity as complexity +from ..AIFeynman import est as est +from ..AIFeynman import model as model from .params._aifeynman import params est.set_params(**params) # 8 hours -est.max_time = 8*60*60 - +est.max_time = 8 * 60 * 60 diff --git a/experiment/methods/tuned/BSRRegressor.py b/experiment/methods/tuned/BSRRegressor.py index d20dae49..2e0c3693 100644 --- a/experiment/methods/tuned/BSRRegressor.py +++ b/experiment/methods/tuned/BSRRegressor.py @@ -1,9 +1,11 @@ -from ..BSRRegressor import complexity,model,est +from ..BSRRegressor import complexity as complexity +from ..BSRRegressor import est as est +from ..BSRRegressor import model as model from .params._bsrregressor import params est.set_params(**params) -est.max_time = 8*60*60 +est.max_time = 8 * 60 * 60 # double the evals -est.itrNum = int(est.itrNum*2**0.5) -est.val = int(est.val*2**0.5) +est.itrNum = int(est.itrNum * 2**0.5) +est.val = int(est.val * 2**0.5) diff --git a/experiment/methods/tuned/DSRRegressor.py b/experiment/methods/tuned/DSRRegressor.py index eb6aaf38..32d2a77a 100644 --- a/experiment/methods/tuned/DSRRegressor.py +++ b/experiment/methods/tuned/DSRRegressor.py @@ -1,12 +1,11 @@ -from ..DSRRegressor import complexity,model,base_config from dsr import DeepSymbolicRegressor -import numpy as np -import copy -from .params._dsrregressor import params -#double the evals -base_config['training']['n_samples'] = 1000000 +from ..DSRRegressor import base_config +from ..DSRRegressor import complexity as complexity +from ..DSRRegressor import model as model + +# double the evals +base_config["training"]["n_samples"] = 1000000 # Create the model est = DeepSymbolicRegressor(base_config) - diff --git a/experiment/methods/tuned/EHCRegressor.py b/experiment/methods/tuned/EHCRegressor.py index e269dcae..b01a1953 100644 --- a/experiment/methods/tuned/EHCRegressor.py +++ b/experiment/methods/tuned/EHCRegressor.py @@ -1,10 +1,24 @@ -from ..EHCRegressor import complexity, model, est -from ellyn import ellyn +from ..EHCRegressor import complexity as complexity +from ..EHCRegressor import est as est +from ..EHCRegressor import model as model - -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.eHC_its=5 +est.eHC_its = 5 est.g = 143 est.popsize = 1000 -est.time_limit = 8*60*60 +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/EPLEXRegressor.py b/experiment/methods/tuned/EPLEXRegressor.py index f4927f77..d7ad4a6d 100644 --- a/experiment/methods/tuned/EPLEXRegressor.py +++ b/experiment/methods/tuned/EPLEXRegressor.py @@ -1,10 +1,26 @@ -from ..EPLEXRegressor import complexity,model, est +from ..EPLEXRegressor import complexity as complexity +from ..EPLEXRegressor import est as est +from ..EPLEXRegressor import model as model from .params._eplexregressor import params est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) -est.time_limit = 8*60*60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/FEATRegressor.py b/experiment/methods/tuned/FEATRegressor.py index 8c6e93a7..5f1d47f1 100644 --- a/experiment/methods/tuned/FEATRegressor.py +++ b/experiment/methods/tuned/FEATRegressor.py @@ -1,12 +1,14 @@ -from ..FEATRegressor import complexity,model,pre_train, eval_kwargs, est -from feat import FeatRegressor +from ..FEATRegressor import complexity as complexity +from ..FEATRegressor import est as est +from ..FEATRegressor import eval_kwargs as eval_kwargs +from ..FEATRegressor import model as model from .params._featregressor import params est.set_params(**params) -est.functions = '+,-,*,/,^2,^3,sqrt,sin,cos,exp,log' -est.otype = 'f' +est.functions = "+,-,*,/,^2,^3,sqrt,sin,cos,exp,log" +est.otype = "f" # double the evals -est.max_time=int(8*60*60) # 8 hrs -est.gens = int(est.gens*2**0.5) -est.pop_size = int(est.pop_size*2**0.5) +est.max_time = int(8 * 60 * 60) # 8 hrs +est.gens = int(est.gens * 2**0.5) +est.pop_size = int(est.pop_size * 2**0.5) diff --git a/experiment/methods/tuned/FE_AFPRegressor.py b/experiment/methods/tuned/FE_AFPRegressor.py index c452bbeb..6dbfa53c 100644 --- a/experiment/methods/tuned/FE_AFPRegressor.py +++ b/experiment/methods/tuned/FE_AFPRegressor.py @@ -1,9 +1,26 @@ -from ..FE_AFPRegressor import complexity,model, pre_train, eval_kwargs, est +from ..FE_AFPRegressor import complexity as complexity +from ..FE_AFPRegressor import est as est +from ..FE_AFPRegressor import eval_kwargs as eval_kwargs +from ..FE_AFPRegressor import model as model from .params._fe_afpregressor import params est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] -est.time_limit = 8*60*60 -# doubling evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] +est.time_limit = 8 * 60 * 60 +# doubling evals +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) diff --git a/experiment/methods/tuned/FFXRegressor.py b/experiment/methods/tuned/FFXRegressor.py index 5823d291..1d3cb5a4 100644 --- a/experiment/methods/tuned/FFXRegressor.py +++ b/experiment/methods/tuned/FFXRegressor.py @@ -1,2 +1,3 @@ -from ..FFXRegressor import complexity,model,est -from ffx import FFXRegressor +from ..FFXRegressor import complexity as complexity +from ..FFXRegressor import est as est +from ..FFXRegressor import model as model diff --git a/experiment/methods/tuned/GPGOMEARegressor.py b/experiment/methods/tuned/GPGOMEARegressor.py index 2af62b2c..2b738efa 100644 --- a/experiment/methods/tuned/GPGOMEARegressor.py +++ b/experiment/methods/tuned/GPGOMEARegressor.py @@ -1,11 +1,12 @@ -from ..GPGOMEARegressor import complexity, model, est -from pyGPGOMEA import GPGOMEARegressor as GPG +from ..GPGOMEARegressor import complexity as complexity +from ..GPGOMEARegressor import est as est +from ..GPGOMEARegressor import model as model from .params._gpgomearegressor import params est.set_params(**params) -est.functions = '+_-_*_p/_plog_sqrt_sin_cos' +est.functions = "+_-_*_p/_plog_sqrt_sin_cos" # double the evals est.evaluations = 1000000 -est.time=8*60*60 +est.time = 8 * 60 * 60 diff --git a/experiment/methods/tuned/ITEARegressor.py b/experiment/methods/tuned/ITEARegressor.py index b34bce4e..42e5612f 100644 --- a/experiment/methods/tuned/ITEARegressor.py +++ b/experiment/methods/tuned/ITEARegressor.py @@ -1,9 +1,12 @@ -from ..ITEARegressor import complexity,model,eval_kwargs, est +from ..ITEARegressor import complexity as complexity +from ..ITEARegressor import est as est +from ..ITEARegressor import eval_kwargs as eval_kwargs +from ..ITEARegressor import model as model from .params._itearegressor import params est.set_params(**params) -est.transfunctions = '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]' +est.transfunctions = "[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]" # double the evals -est.npop = int(est.npop*2**0.5) -est.ngens = int(est.ngens*2**0.5) +est.npop = int(est.npop * 2**0.5) +est.ngens = int(est.ngens * 2**0.5) diff --git a/experiment/methods/tuned/MRGPRegressor.py b/experiment/methods/tuned/MRGPRegressor.py index 41d77d3d..8c228eb2 100644 --- a/experiment/methods/tuned/MRGPRegressor.py +++ b/experiment/methods/tuned/MRGPRegressor.py @@ -1,10 +1,11 @@ -from ..MRGPRegressor import complexity, model, est -from ..src.mrgp import MRGPRegressor +from ..MRGPRegressor import complexity as complexity +from ..MRGPRegressor import est as est +from ..MRGPRegressor import model as model from .params._mrgpregressor import params est.set_params(**params) # double the evals -est.time_out=8*60*60 -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) +est.time_out = 8 * 60 * 60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) diff --git a/experiment/methods/tuned/OperonRegressor.py b/experiment/methods/tuned/OperonRegressor.py index 24d46fe6..3f76844e 100644 --- a/experiment/methods/tuned/OperonRegressor.py +++ b/experiment/methods/tuned/OperonRegressor.py @@ -1,11 +1,12 @@ -from ..OperonRegressor import complexity,model,est -from operon.sklearn import SymbolicRegressor +from ..OperonRegressor import complexity as complexity +from ..OperonRegressor import est as est +from ..OperonRegressor import model as model from .params._operonregressor import params est.set_params(**params) -est.allowed_symbols = 'add,sub,mul,div,exp,log,sin,cos,sqrt,square,constant,variable' +est.allowed_symbols = "add,sub,mul,div,exp,log,sin,cos,sqrt,square,constant,variable" # double the evals est.max_evaluations = 1000000 -est.generations=100000 # just large enough since we have an evaluation budget -est.time_limit=8*60*60 # 8 hours +est.generations = 100000 # just large enough since we have an evaluation budget +est.time_limit = 8 * 60 * 60 # 8 hours diff --git a/experiment/methods/tuned/gplearn.py b/experiment/methods/tuned/gplearn.py index 39e3fe3b..869a1d28 100644 --- a/experiment/methods/tuned/gplearn.py +++ b/experiment/methods/tuned/gplearn.py @@ -1,9 +1,11 @@ -from ..gplearn import complexity,model, est +from ..gplearn.regressor import complexity as complexity +from ..gplearn.regressor import est as est +from ..gplearn.regressor import model as model from .params._gplearn import params est.set_params(**params) -est.function_set = ('add', 'sub', 'mul', 'div', 'log','sqrt', 'sin','cos') +est.function_set = ("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos") # double the evals -est.population_size = int(est.population_size*2**0.5) -est.generations = int(est.generations*2**0.5) +est.population_size = int(est.population_size * 2**0.5) +est.generations = int(est.generations * 2**0.5) diff --git a/experiment/methods/tuned/params/_afpregressor.py b/experiment/methods/tuned/params/_afpregressor.py index 779c59ae..8d8da237 100644 --- a/experiment/methods/tuned/params/_afpregressor.py +++ b/experiment/methods/tuned/params/_afpregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': False, 'FE_ind_size': 0, 'FE_pop_size': 0, 'FE_rank': False, 'FE_train_gens': 0, 'FE_train_size': 0, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 22118, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'afp', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 1, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": False, + "FE_ind_size": 0, + "FE_pop_size": 0, + "FE_rank": False, + "FE_train_gens": 0, + "FE_train_size": 0, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 22118, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "afp", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 1, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_aifeynman.py b/experiment/methods/tuned/params/_aifeynman.py index 00c2f998..dc5add99 100644 --- a/experiment/methods/tuned/params/_aifeynman.py +++ b/experiment/methods/tuned/params/_aifeynman.py @@ -1 +1,9 @@ -params = {'BF_ops_file_type': '14ops.txt', 'BF_try_time': 60, 'NN_epochs': 4000, 'max_time': 7200, 'polyfit_deg': 4, 'random_state': 11284, 'test_percentage': 20} +params = { + "BF_ops_file_type": "14ops.txt", + "BF_try_time": 60, + "NN_epochs": 4000, + "max_time": 7200, + "polyfit_deg": 4, + "random_state": 11284, + "test_percentage": 20, +} diff --git a/experiment/methods/tuned/params/_bsrregressor.py b/experiment/methods/tuned/params/_bsrregressor.py index 21a3d19c..ad66e031 100644 --- a/experiment/methods/tuned/params/_bsrregressor.py +++ b/experiment/methods/tuned/params/_bsrregressor.py @@ -1 +1,11 @@ -params = {'Ops': ['inv', 'ln', 'neg', 'sin', 'cos', 'exp', 'square', 'cubic', '+', '*'], 'alpha1': 0.4, 'alpha2': 0.4, 'beta': -1, 'disp': False, 'itrNum': 5000, 'max_time': 7200, 'treeNum': 3, 'val': 100} +params = { + "Ops": ["inv", "ln", "neg", "sin", "cos", "exp", "square", "cubic", "+", "*"], + "alpha1": 0.4, + "alpha2": 0.4, + "beta": -1, + "disp": False, + "itrNum": 5000, + "max_time": 7200, + "treeNum": 3, + "val": 100, +} diff --git a/experiment/methods/tuned/params/_dsrregressor.py b/experiment/methods/tuned/params/_dsrregressor.py index deefb909..8ce4ba20 100644 --- a/experiment/methods/tuned/params/_dsrregressor.py +++ b/experiment/methods/tuned/params/_dsrregressor.py @@ -1 +1,5 @@ -params = {} +from __future__ import annotations + +from typing import Any + +params: dict[str, Any] = {} diff --git a/experiment/methods/tuned/params/_eplexregressor.py b/experiment/methods/tuned/params/_eplexregressor.py index 03f79cf8..42f55d50 100644 --- a/experiment/methods/tuned/params/_eplexregressor.py +++ b/experiment/methods/tuned/params/_eplexregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': False, 'FE_ind_size': 0, 'FE_pop_size': 0, 'FE_rank': False, 'FE_train_gens': 0, 'FE_train_size': 0, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 6265, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'epsilon_lexicase', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 0, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": False, + "FE_ind_size": 0, + "FE_pop_size": 0, + "FE_rank": False, + "FE_train_gens": 0, + "FE_train_size": 0, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 6265, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "epsilon_lexicase", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 0, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_fe_afpregressor.py b/experiment/methods/tuned/params/_fe_afpregressor.py index 15bfb78f..7bb54e96 100644 --- a/experiment/methods/tuned/params/_fe_afpregressor.py +++ b/experiment/methods/tuned/params/_fe_afpregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': True, 'FE_ind_size': 10, 'FE_pop_size': 100, 'FE_rank': True, 'FE_train_gens': 10, 'FE_train_size': 10, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 29802, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'afp', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 0, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": True, + "FE_ind_size": 10, + "FE_pop_size": 100, + "FE_rank": True, + "FE_train_gens": 10, + "FE_train_size": 10, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 29802, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "afp", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 0, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_featregressor.py b/experiment/methods/tuned/params/_featregressor.py index 5bd3cd25..71acca7e 100644 --- a/experiment/methods/tuned/params/_featregressor.py +++ b/experiment/methods/tuned/params/_featregressor.py @@ -1 +1,45 @@ -params = {'pop_size': 1000, 'gens': 250, 'ml': 'LinearRidgeRegression', 'classification': False, 'verbosity': 0, 'max_stall': 100, 'sel': 'lexicase', 'surv': 'nsga2', 'cross_rate': 0.75, 'root_xo_rate': 0.5, 'otype': 'a', 'functions': '', 'max_depth': 6, 'max_dim': 10, 'random_state': 29802, 'erc': False, 'obj': 'fitness,complexity', 'shuffle': True, 'split': 0.75, 'fb': 0.5, 'scorer': '', 'feature_names': '', 'backprop': True, 'iters': 1, 'lr': 0.1, 'batch_size': 100, 'n_jobs': 1, 'hillclimb': False, 'logfile': '', 'max_time': 7200, 'residual_xo': False, 'stagewise_xo': False, 'stagewise_xo_tol': False, 'softmax_norm': False, 'save_pop': 0, 'normalize': True, 'val_from_arch': True, 'corr_delete_mutate': True, 'simplify': 0.005, 'protected_groups': '', 'tune_initial': False, 'tune_final': True, 'starting_pop': ''} +params = { + "pop_size": 1000, + "gens": 250, + "ml": "LinearRidgeRegression", + "classification": False, + "verbosity": 0, + "max_stall": 100, + "sel": "lexicase", + "surv": "nsga2", + "cross_rate": 0.75, + "root_xo_rate": 0.5, + "otype": "a", + "functions": "", + "max_depth": 6, + "max_dim": 10, + "random_state": 29802, + "erc": False, + "obj": "fitness,complexity", + "shuffle": True, + "split": 0.75, + "fb": 0.5, + "scorer": "", + "feature_names": "", + "backprop": True, + "iters": 1, + "lr": 0.1, + "batch_size": 100, + "n_jobs": 1, + "hillclimb": False, + "logfile": "", + "max_time": 7200, + "residual_xo": False, + "stagewise_xo": False, + "stagewise_xo_tol": False, + "softmax_norm": False, + "save_pop": 0, + "normalize": True, + "val_from_arch": True, + "corr_delete_mutate": True, + "simplify": 0.005, + "protected_groups": "", + "tune_initial": False, + "tune_final": True, + "starting_pop": "", +} diff --git a/experiment/methods/tuned/params/_ffxregressor.py b/experiment/methods/tuned/params/_ffxregressor.py index deefb909..8ce4ba20 100644 --- a/experiment/methods/tuned/params/_ffxregressor.py +++ b/experiment/methods/tuned/params/_ffxregressor.py @@ -1 +1,5 @@ -params = {} +from __future__ import annotations + +from typing import Any + +params: dict[str, Any] = {} diff --git a/experiment/methods/tuned/params/_gpgomearegressor.py b/experiment/methods/tuned/params/_gpgomearegressor.py index d7674986..fa581219 100644 --- a/experiment/methods/tuned/params/_gpgomearegressor.py +++ b/experiment/methods/tuned/params/_gpgomearegressor.py @@ -1 +1,31 @@ -params = {'caching': False, 'classweights': False, 'elitism': 1, 'erc': True, 'evaluations': 500000, 'functions': '+_-_*_p/_plog_sqrt_sin_cos', 'generations': -1, 'gomea': True, 'gomfos': 'LT', 'ims': False, 'initmaxtreeheight': 4, 'linearscaling': True, 'maxsize': 1000, 'maxtreeheight': 17, 'parallel': False, 'popsize': 1000, 'prob': 'symbreg', 'reproduction': 0.0, 'sbagx': 0.0, 'sblibtype': False, 'sbrdo': 0.0, 'seed': -1, 'silent': True, 'subcross': 0.5, 'submut': 0.5, 'syntuniqinit': 1000, 'time': 7200, 'tournament': 4, 'unifdepthvar': True} +params = { + "caching": False, + "classweights": False, + "elitism": 1, + "erc": True, + "evaluations": 500000, + "functions": "+_-_*_p/_plog_sqrt_sin_cos", + "generations": -1, + "gomea": True, + "gomfos": "LT", + "ims": False, + "initmaxtreeheight": 4, + "linearscaling": True, + "maxsize": 1000, + "maxtreeheight": 17, + "parallel": False, + "popsize": 1000, + "prob": "symbreg", + "reproduction": 0.0, + "sbagx": 0.0, + "sblibtype": False, + "sbrdo": 0.0, + "seed": -1, + "silent": True, + "subcross": 0.5, + "submut": 0.5, + "syntuniqinit": 1000, + "time": 7200, + "tournament": 4, + "unifdepthvar": True, +} diff --git a/experiment/methods/tuned/params/_gplearn.py b/experiment/methods/tuned/params/_gplearn.py index a527d307..f2a1f264 100644 --- a/experiment/methods/tuned/params/_gplearn.py +++ b/experiment/methods/tuned/params/_gplearn.py @@ -1 +1,20 @@ -params = {'generations': 500, 'init_method': 'half and half', 'low_memory': False, 'max_samples': 1.0, 'metric': 'mean absolute error', 'n_jobs': 1, 'p_crossover': 0.9, 'p_hoist_mutation': 0.01, 'p_point_mutation': 0.01, 'p_point_replace': 0.05, 'p_subtree_mutation': 0.01, 'parsimony_coefficient': 0.001, 'population_size': 1000, 'random_state': 11964, 'stopping_criteria': 0.0, 'tournament_size': 20, 'verbose': 0, 'warm_start': False} +params = { + "generations": 500, + "init_method": "half and half", + "low_memory": False, + "max_samples": 1.0, + "metric": "mean absolute error", + "n_jobs": 1, + "p_crossover": 0.9, + "p_hoist_mutation": 0.01, + "p_point_mutation": 0.01, + "p_point_replace": 0.05, + "p_subtree_mutation": 0.01, + "parsimony_coefficient": 0.001, + "population_size": 1000, + "random_state": 11964, + "stopping_criteria": 0.0, + "tournament_size": 20, + "verbose": 0, + "warm_start": False, +} diff --git a/experiment/methods/tuned/params/_itearegressor.py b/experiment/methods/tuned/params/_itearegressor.py index 8f421647..a7cd84f8 100644 --- a/experiment/methods/tuned/params/_itearegressor.py +++ b/experiment/methods/tuned/params/_itearegressor.py @@ -1 +1,6 @@ -params = {'ngens': 500, 'nonzeroexps': 1, 'npop': 1000, 'transfunctions': '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]'} +params = { + "ngens": 500, + "nonzeroexps": 1, + "npop": 1000, + "transfunctions": "[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]", +} diff --git a/experiment/methods/tuned/params/_mrgpregressor.py b/experiment/methods/tuned/params/_mrgpregressor.py index 81da6136..4eab31c1 100644 --- a/experiment/methods/tuned/params/_mrgpregressor.py +++ b/experiment/methods/tuned/params/_mrgpregressor.py @@ -1 +1,9 @@ -params = {'g': 250, 'max_len': 6, 'popsize': 1000, 'rt_cross': 0.5, 'rt_mut': 0.5, 'time_out': 7200, 'tmp_dir': None} +params = { + "g": 250, + "max_len": 6, + "popsize": 1000, + "rt_cross": 0.5, + "rt_mut": 0.5, + "time_out": 7200, + "tmp_dir": None, +} diff --git a/experiment/methods/tuned/params/_operonregressor.py b/experiment/methods/tuned/params/_operonregressor.py index bf0c0bcd..542593f3 100644 --- a/experiment/methods/tuned/params/_operonregressor.py +++ b/experiment/methods/tuned/params/_operonregressor.py @@ -1 +1,34 @@ -params = {'allowed_symbols': 'add,mul,aq,constant,variable', 'brood_size': 10, 'comparison_factor': 0, 'crossover_internal_probability': 0.9, 'crossover_probability': 1.0, 'objectives': ['r2'], 'female_selector': 'tournament', 'generations': 10000, 'initialization_method': 'btc', 'local_iterations': 5, 'male_selector': 'tournament', 'max_depth': 10, 'max_evaluations': 500000, 'max_length': 50, 'max_selection_pressure': 100, 'mutation': {'onepoint': 1.0, 'changevar': 1.0, 'changefunc': 1.0, 'insertsubtree': 1.0, 'replacesubtree': 1.0, 'removesubtree': 1.0}, 'mutation_probability': 0.25, 'n_threads': 1, 'offspring_generator': 'basic', 'pool_size': 500, 'population_size': 500, 'random_state': 22118, 'reinserter': 'keep-best', 'time_limit': 7200, 'tournament_size': 5} +params = { + "allowed_symbols": "add,mul,aq,constant,variable", + "brood_size": 10, + "comparison_factor": 0, + "crossover_internal_probability": 0.9, + "crossover_probability": 1.0, + "objectives": ["r2"], + "female_selector": "tournament", + "generations": 10000, + "initialization_method": "btc", + "local_iterations": 5, + "male_selector": "tournament", + "max_depth": 10, + "max_evaluations": 500000, + "max_length": 50, + "max_selection_pressure": 100, + "mutation": { + "onepoint": 1.0, + "changevar": 1.0, + "changefunc": 1.0, + "insertsubtree": 1.0, + "replacesubtree": 1.0, + "removesubtree": 1.0, + }, + "mutation_probability": 0.25, + "n_threads": 1, + "offspring_generator": "basic", + "pool_size": 500, + "population_size": 500, + "random_state": 22118, + "reinserter": "keep-best", + "time_limit": 7200, + "tournament_size": 5, +} diff --git a/experiment/methods/tuned/params/_sembackpropgp.py b/experiment/methods/tuned/params/_sembackpropgp.py index f367548a..d1dbf3e1 100644 --- a/experiment/methods/tuned/params/_sembackpropgp.py +++ b/experiment/methods/tuned/params/_sembackpropgp.py @@ -1 +1,31 @@ -params = {'caching': False, 'classweights': False, 'elitism': 1, 'erc': True, 'evaluations': 500000, 'functions': '+_-_*_aq_plog_sin_cos', 'generations': -1, 'gomea': False, 'gomfos': 'LT', 'ims': False, 'initmaxtreeheight': 6, 'linearscaling': True, 'maxsize': 1000, 'maxtreeheight': 20, 'parallel': False, 'popsize': 1000, 'prob': 'symbreg', 'reproduction': 0.0, 'sbagx': False, 'sblibtype': 'p_10_9999_l_n', 'sbrdo': 0.9, 'seed': -1, 'silent': True, 'subcross': 0.0, 'submut': 0.1, 'syntuniqinit': 1000, 'time': 7200, 'tournament': 4, 'unifdepthvar': True} +params = { + "caching": False, + "classweights": False, + "elitism": 1, + "erc": True, + "evaluations": 500000, + "functions": "+_-_*_aq_plog_sin_cos", + "generations": -1, + "gomea": False, + "gomfos": "LT", + "ims": False, + "initmaxtreeheight": 6, + "linearscaling": True, + "maxsize": 1000, + "maxtreeheight": 20, + "parallel": False, + "popsize": 1000, + "prob": "symbreg", + "reproduction": 0.0, + "sbagx": False, + "sblibtype": "p_10_9999_l_n", + "sbrdo": 0.9, + "seed": -1, + "silent": True, + "subcross": 0.0, + "submut": 0.1, + "syntuniqinit": 1000, + "time": 7200, + "tournament": 4, + "unifdepthvar": True, +} diff --git a/experiment/methods/tuned/sembackpropgp.py b/experiment/methods/tuned/sembackpropgp.py index c95cec90..38183207 100644 --- a/experiment/methods/tuned/sembackpropgp.py +++ b/experiment/methods/tuned/sembackpropgp.py @@ -1,10 +1,11 @@ -from ..sembackpropgp import complexity, model, est -from pyGPGOMEA import GPGOMEARegressor as GPG +from ..sembackpropgp import complexity as complexity +from ..sembackpropgp import est as est +from ..sembackpropgp import model as model from .params._sembackpropgp import params est.set_params(**params) -est.functions = '+_-_*_aq_plog_sin_cos' +est.functions = "+_-_*_aq_plog_sin_cos" -#double the evals -est.evaluations=1000000 -est.time = 8*60*60 +# double the evals +est.evaluations = 1000000 +est.time = 8 * 60 * 60 diff --git a/experiment/methods/udsr/regressor.py b/experiment/methods/udsr/regressor.py index f7b0e4a5..167ffc9f 100644 --- a/experiment/methods/udsr/regressor.py +++ b/experiment/methods/udsr/regressor.py @@ -1,19 +1,13 @@ -"""Borrows gpzgd's example of how to wrap a CLI learner with a sklearn interface -""" +"""Borrows gpzgd's example of how to wrap a CLI learner with a sklearn interface""" -from dso import DeepSymbolicRegressor -from dso import DeepSymbolicOptimizer - -import sys -import os -from tempfile import TemporaryDirectory import json -import numpy as np - -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_is_fitted, check_X_y, check_array +from tempfile import TemporaryDirectory +import numpy as np import tensorflow as tf +from dso import DeepSymbolicOptimizer +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.utils.validation import check_array, check_is_fitted, check_X_y print("Num GPUs Available: ", tf.test.is_gpu_available()) print("CUDA Available: ", tf.test.is_built_with_cuda()) @@ -29,22 +23,34 @@ hyper_params = [] for f in function_set: - hyper_params.append({ - 'function_set' : [f], - 'degree' : [2], - 'run_gp_meld' : [True] - }) - + hyper_params.append({"function_set": [f], "degree": [2], "run_gp_meld": [True]}) + class uDSRRegressor(BaseEstimator, RegressorMixin): - def __init__(self, random_state=42, - function_set=["add","sub","mul","div","sin","cos","exp","log","sqrt","const","poly"], - degree=3, run_gp_meld=True): - + def __init__( + self, + random_state=42, + function_set=[ + "add", + "sub", + "mul", + "div", + "sin", + "cos", + "exp", + "log", + "sqrt", + "const", + "poly", + ], + degree=3, + run_gp_meld=True, + ): + self.random_state = random_state self.function_set = function_set - self.degree = degree - self.run_gp_meld = run_gp_meld + self.degree = degree + self.run_gp_meld = run_gp_meld def fit(self, X, y): # 1. create a temporary directory to store the training data set @@ -52,58 +58,50 @@ def fit(self, X, y): # 3. create a temp file and store the data X, y = check_X_y(X, y, accept_sparse=False) if len(y.shape) == 1: - Z = np.hstack((X, y[:,None])) + Z = np.hstack((X, y[:, None])) else: Z = np.hstack((X, y)) - fname = temp_dir + "/tmpdata.csv" + fname = temp_dir + "/tmpdata.csv" np.savetxt(f"{fname}", Z, delimiter=",") - + # 4. create a temp config file config = { - 'task' : { - 'task_type' : 'regression', - 'dataset' : fname, - + "task": { + "task_type": "regression", + "dataset": fname, # To customize a function set, edit this! See functions.py for a list of # supported functions. Note "const" will add placeholder constants that # will be optimized within the training loop. This will considerably # increase runtime. "function_set": self.function_set, - # Metric to be used for the reward function. See regression.py for # supported metrics. - "metric" : "inv_nrmse", - "metric_params" : [1.0], - + "metric": "inv_nrmse", + "metric_params": [1.0], # Optional alternate metric to be used at evaluation time. - "extra_metric_test" : None, - "extra_metric_test_params" : [], - + "extra_metric_test": None, + "extra_metric_test_params": [], # NRMSE threshold for early stopping. This is useful for noiseless # benchmark problems when DSO discovers the True solution. - "threshold" : 1e-12, - + "threshold": 1e-12, # With protected=False, floating-point errors (e.g. log of negative # number) will simply returns a minimal reward. With protected=True, # "protected" functions will prevent floating-point errors, but may - # introduce discontinuities in the learned functions. - "protected" : True, - + # introduce discontinuities in the learned functions. + "protected": True, # You can add artificial reward noise directly to the reward function. # Note this does NOT add noise to the dataset. - "reward_noise" : 0.0, - "reward_noise_type" : "r", - "normalize_variance" : False, - + "reward_noise": 0.0, + "reward_noise_type": "r", + "normalize_variance": False, # Set of thresholds (shared by all input variables) for building # decision trees. Note that no StateChecker will be added to Library # if decision_tree_threshold_set is an empty list or null. - "decision_tree_threshold_set" : [], - + "decision_tree_threshold_set": [], # Parameters for optimizing the "poly" token. # Note: poly_optimizer is turned on if and only if "poly" is in function_set. - "poly_optimizer_params" : { + "poly_optimizer_params": { # The (maximal) degree of the polynomials used to fit the data "degree": self.degree, # Cutoff value for the coefficients of polynomials. Coefficients @@ -114,7 +112,7 @@ def fit(self, X, y): # of least squares regressor "dso_least_squares". "regressor": "dso_least_squares", "regressor_params": { - # Cutoff value for p-value of coefficients. Coefficients with + # Cutoff value for p-value of coefficients. Coefficients with # larger p-values are forced to zero. "cutoff_p_value": 1.0, # Maximum number of terms in the polynomial. If more coefficients are nonzero, @@ -122,93 +120,68 @@ def fit(self, X, y): "n_max_terms": None, # Cutoff value for the coefficients of polynomials. Coefficients # with magnitude less than this value will be regarded as 0. - "coef_tol": 1e-6 - } - } + "coef_tol": 1e-6, + }, + }, }, - # Hyperparameters related to genetic programming hybrid methods. - "gp_meld" : { - "run_gp_meld" : self.run_gp_meld, - "population_size" : 25, - "generations" : 25, - "crossover_operator" : "cxOnePoint", - "p_crossover" : 0.5, - "mutation_operator" : "multi_mutate", - "p_mutate" : 0.5, - "tournament_size" : 5, - "train_n" : 50, - "mutate_tree_max" : 3, - "verbose" : True, + "gp_meld": { + "run_gp_meld": self.run_gp_meld, + "population_size": 25, + "generations": 25, + "crossover_operator": "cxOnePoint", + "p_crossover": 0.5, + "mutation_operator": "multi_mutate", + "p_mutate": 0.5, + "tournament_size": 5, + "train_n": 50, + "mutate_tree_max": 3, + "verbose": True, # Speeds up processing when doing expensive evaluations. - "parallel_eval" : False + "parallel_eval": False, }, - # Only the key training hyperparameters are listed here. See # config_common.json for the full list. - "training" : { - "n_samples" : 10000, - "batch_size" : 500, - "epsilon" : 0.02, + "training": { + "n_samples": 10000, + "batch_size": 500, + "epsilon": 0.02, # Recommended to set this to as many cores as you can use! Especially if # using the "const" token. - "n_cores_batch" : -1 + "n_cores_batch": -1, }, - # Only the key RNN controller hyperparameters are listed here. See # config_common.json for the full list. - "controller" : { + "controller": { "learning_rate": 0.0025, - "entropy_weight" : 0.03, - "entropy_gamma" : 0.7, - + "entropy_weight": 0.03, + "entropy_gamma": 0.7, # EXPERIMENTAL: Priority queue training hyperparameters. - "pqt" : True, - "pqt_k" : 10, - "pqt_batch_size" : 1, - "pqt_weight" : 200.0, - "pqt_use_pg" : False + "pqt": True, + "pqt_k": 10, + "pqt_batch_size": 1, + "pqt_weight": 200.0, + "pqt_use_pg": False, }, - # Hyperparameters related to including in situ priors and constraints. Each # prior must explicitly be turned "on" or it will not be used. See # config_common.json for descriptions of each prior. "prior": { - "length" : { - "min_" : 4, - "max_" : 100, - "on" : True - }, - "inverse" : { - "on" : True - }, - "trig" : { - "on" : True - }, - "const" : { - "on" : True - }, - "no_inputs" : { - "on" : True - }, - "uniform_arity" : { - "on" : True - }, - "soft_length" : { - "loc" : 10, - "scale" : 5, - "on" : True - }, - "domain_range" : { - "on" : True - } + "length": {"min_": 4, "max_": 100, "on": True}, + "inverse": {"on": True}, + "trig": {"on": True}, + "const": {"on": True}, + "no_inputs": {"on": True}, + "uniform_arity": {"on": True}, + "soft_length": {"loc": 10, "scale": 5, "on": True}, + "domain_range": {"on": True}, + }, + "experiment": { + "logdir": None, }, - 'experiment' : { - 'logdir' : None, - } } - - cname = temp_dir + "/config.json" + + cname = temp_dir + "/config.json" with open(cname, "w") as config_file: json.dump(config, config_file, indent=4) @@ -231,24 +204,25 @@ def predict(self, X): est = uDSRRegressor() -def model(est, X=None): # Should work for both methods +def model(est, X=None): # Should work for both methods # clean_pred_model from assess_symbolic model has fixes to the string rep return str(est.program_.sympy_expr) + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) - + if __name__ == "__main__": - import numpy as np - + import numpy as np + # Generate some data np.random.seed(0) X = np.random.random((10, 2)) - y = np.sin(X[:,0]) + X[:,1] ** 2 + y = np.sin(X[:, 0]) + X[:, 1] ** 2 # Fit the model - est.fit(X, y) # Should solve in ~10 seconds + est.fit(X, y) # Should solve in ~10 seconds # View the best expression print(est.program_.pretty()) diff --git a/experiment/methods/xgboost/regressor.py b/experiment/methods/xgboost/regressor.py index caa7d220..e8a72d5a 100644 --- a/experiment/methods/xgboost/regressor.py +++ b/experiment/methods/xgboost/regressor.py @@ -1,19 +1,21 @@ -import xgboost import numpy as np +import xgboost hyper_params = [ { - 'n_estimators' : (10, 100, 1000), - 'learning_rate' : (0.0001,0.01, 0.1), - 'gamma' : (0,0.1,0.2,0.3,0.4), - 'subsample' : (0.5, 0.75), + "n_estimators": (10, 100, 1000), + "learning_rate": (0.0001, 0.01, 0.1), + "gamma": (0, 0.1, 0.2, 0.3, 0.4), + "subsample": (0.5, 0.75), }, ] -est=xgboost.XGBRegressor(max_depth=6) +est = xgboost.XGBRegressor(max_depth=6) + def complexity(est): - c = np.sum([m.count(':') for m in est._Booster.get_dump()]) + c = np.sum([m.count(":") for m in est._Booster.get_dump()]) return int(c) + model = None diff --git a/experiment/methods_table.ipynb b/experiment/methods_table.ipynb index 7accd340..cfed3715 100644 --- a/experiment/methods_table.ipynb +++ b/experiment/methods_table.ipynb @@ -6,61 +6,60 @@ "metadata": {}, "outputs": [], "source": [ - "from glob import glob\n", "import importlib\n", - "from sklearn.model_selection import ParameterGrid\n", + "from glob import glob\n", + "\n", "from symbolic_algs import symbolic_algs\n", "\n", "sr_frames = []\n", "ml_frames = []\n", "\n", - "for m in [n.split('/')[-1][:-3] for n in glob('methods/*.py')]:\n", - " if m.startswith('_'): continue\n", - " if 'EHC' in m: continue\n", - " if 'FFX' in m: continue\n", - " \n", - " m_short = m.replace('Regressor','')\n", - " \n", - " alg = importlib.__import__('methods.'+m,\n", - " globals(),locals(), \n", - " ['*']\n", - " ) \n", - " if m_short == 'DSR':\n", - " hp = {'batch_size':alg.batch_sizes}\n", + "for m in [n.split(\"/\")[-1][:-3] for n in glob(\"methods/*.py\")]:\n", + " if m.startswith(\"_\"):\n", + " continue\n", + " if \"EHC\" in m:\n", + " continue\n", + " if \"FFX\" in m:\n", + " continue\n", + "\n", + " m_short = m.replace(\"Regressor\", \"\")\n", + "\n", + " alg = importlib.__import__(\"methods.\" + m, globals(), locals(), [\"*\"])\n", + " if m_short == \"DSR\":\n", + " hp = {\"batch_size\": alg.batch_sizes}\n", " else:\n", " hp = alg.hyper_params\n", " if isinstance(hp, list):\n", " tuneables = hp[0].keys()\n", " else:\n", " tuneables = hp.keys()\n", - " \n", - "# print(m)\n", - "# print(hp)\n", - " \n", + "\n", + " # print(m)\n", + " # print(hp)\n", + "\n", " # get tuned for symbolic_algs\n", " if m not in symbolic_algs:\n", " if isinstance(hp, list):\n", " for p in hp:\n", " ml_frames.append([m_short, p])\n", " else:\n", - " ml_frames.append([m_short, hp]) # hp_tuned])\n", + " ml_frames.append([m_short, hp]) # hp_tuned])\n", " continue\n", " else:\n", - " alg_tuned = importlib.__import__('methods.tuned.params._'+m.lower(),\n", - " globals(),locals(), \n", - " ['params']\n", - " ) \n", - " hp_tuned = {k:v for k,v in alg_tuned.params.items() if k in tuneables}\n", + " alg_tuned = importlib.__import__(\n", + " \"methods.tuned.params._\" + m.lower(), globals(), locals(), [\"params\"]\n", + " )\n", + " hp_tuned = {k: v for k, v in alg_tuned.params.items() if k in tuneables}\n", " if isinstance(hp, list):\n", " for p in hp:\n", " sr_frames.append([m_short, p])\n", " else:\n", - " sr_frames.append([m_short, hp]) # hp_tuned])\n", - " \n", + " sr_frames.append([m_short, hp]) # hp_tuned])\n", + "\n", "# hp_pg = ParameterGrid(hp)\n", "# print('tuned:',hp_tuned)\n", "# print(list(hp_pg))\n", - "print('ML')\n", + "print(\"ML\")\n", "print(ml_frames)" ] }, @@ -70,21 +69,20 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", - "import json\n", - "from utils import jsonify\n", "def hp_format(p):\n", - "# pdb.set_trace()\n", + " # pdb.set_trace()\n", " if isinstance(p, dict):\n", - " for k,v in p.items():\n", - " if isinstance(v, list) and len(v)==1:\n", + " for k, v in p.items():\n", + " if isinstance(v, list) and len(v) == 1:\n", " p[k] = v[0]\n", " elif isinstance(v, float):\n", - " p[k] = round(v,2)\n", - " elif isinstance(p,list):\n", + " p[k] = round(v, 2)\n", + " elif isinstance(p, list):\n", " return [hp_format(i) for i in p]\n", - " \n", + "\n", " return p\n", + "\n", + "\n", "# args = dict(sort_keys=True, indent=4)\n", "# # if isinstance(p, list):\n", "# # return '\\\\\\\\'.join([json.dumps(jsonify(i), **args)\n", @@ -92,16 +90,17 @@ "# # else:\n", "# return json.dumps(jsonify(p), **args)\n", "\n", + "\n", "def add_midrules(ltx):\n", " ltx2 = ltx.splitlines()\n", " size = len(ltx2) - 1\n", - " for i,line in enumerate(ltx.splitlines()[::-1]):\n", - " method = line.split('&')[0].strip().replace('\\\\','')\n", - " if method in methods: \n", - "# pdb.set_trace()\n", - " ltx2.insert(size-i,'\\midrule')\n", - " \n", - " return '\\n'.join(ltx2)" + " for i, line in enumerate(ltx.splitlines()[::-1]):\n", + " method = line.split(\"&\")[0].strip().replace(\"\\\\\", \"\")\n", + " if method in methods:\n", + " # pdb.set_trace()\n", + " ltx2.insert(size - i, \"\\midrule\")\n", + "\n", + " return \"\\n\".join(ltx2)" ] }, { @@ -110,7 +109,7 @@ "metadata": {}, "outputs": [], "source": [ - "methods" + "# Methods are defined from sr_df in the following cell." ] }, { @@ -119,50 +118,54 @@ "metadata": {}, "outputs": [], "source": [ + "from math import ceil\n", + "\n", "import pandas as pd\n", - "from math import floor, ceil\n", - "column_format = 'l p{37em}'\n", - "pd.set_option('display.max_colwidth', 1000)\n", - "sr_df = pd.DataFrame(sr_frames, columns = ['Method','Hyperparameters']).sort_values(by='Method')\n", + "\n", + "column_format = \"l p{37em}\"\n", + "pd.set_option(\"display.max_colwidth\", 1000)\n", + "sr_df = pd.DataFrame(sr_frames, columns=[\"Method\", \"Hyperparameters\"]).sort_values(\n", + " by=\"Method\"\n", + ")\n", "methods = sr_df.Method.unique()\n", - "for i in [1,2]:\n", - " print_methods = methods[:ceil(len(methods)/2)] if i == 1 else methods[ceil(len(methods)/2):]\n", + "for i in [1, 2]:\n", + " print_methods = (\n", + " methods[: ceil(len(methods) / 2)]\n", + " if i == 1\n", + " else methods[ceil(len(methods) / 2) :]\n", + " )\n", " df = sr_df.loc[sr_df.Method.isin(print_methods)]\n", - " df.loc[df['Method'] == df['Method'].shift(1), 'Method'] = ''\n", + " df.loc[df[\"Method\"] == df[\"Method\"].shift(1), \"Method\"] = \"\"\n", " sr_ltx = df.to_latex(\n", - " index=False,\n", - " column_format=column_format,\n", - " float_format='{:0.1f}',\n", - " formatters={\n", - " 'Hyperparameters': hp_format\n", - " }\n", - " )\n", + " index=False,\n", + " column_format=column_format,\n", + " float_format=\"{:0.1f}\",\n", + " formatters={\"Hyperparameters\": hp_format},\n", + " )\n", " # insert midrule between methods\n", " sr_ltx = add_midrules(sr_ltx)\n", - "# for s in sr_ltx.splitlines(): print(s[:80])\n", - " with open('../../paper/tables/sr_methods_hp_table'+str(i)+'.tex','w') as f:\n", + " # for s in sr_ltx.splitlines(): print(s[:80])\n", + " with open(\"../../paper/tables/sr_methods_hp_table\" + str(i) + \".tex\", \"w\") as f:\n", " f.write(sr_ltx)\n", - " \n", + "\n", "##########\n", "# ML table\n", "##########\n", - "ml_df = pd.DataFrame(ml_frames, columns = ['Method','Hyperparameters'])\n", - "ml_df.loc[ml_df['Method'] == ml_df['Method'].shift(1), 'Method'] = ''\n", + "ml_df = pd.DataFrame(ml_frames, columns=[\"Method\", \"Hyperparameters\"])\n", + "ml_df.loc[ml_df[\"Method\"] == ml_df[\"Method\"].shift(1), \"Method\"] = \"\"\n", "ml_ltx = ml_df.to_latex(\n", - " index=False,\n", - " column_format=column_format,\n", - " float_format='{:0.1f}',\n", - " formatters={\n", - " 'Hyperparameters': hp_format\n", - " }\n", - " )\n", + " index=False,\n", + " column_format=column_format,\n", + " float_format=\"{:0.1f}\",\n", + " formatters={\"Hyperparameters\": hp_format},\n", + ")\n", "# insert midrule between methods\n", "ml_ltx2 = ml_ltx.splitlines()\n", - "for i in range(len(ml_ltx),0):\n", - " ml_ltx2.insert(i,'\\midrule') \n", - "ml_ltx = '\\n'.join(ml_ltx2)\n", - " #add_midrules(sr_ltx)\n", - "with open('../../paper/tables/ml_methods_hp_table.tex','w') as f:\n", + "for i in range(len(ml_ltx), 0):\n", + " ml_ltx2.insert(i, \"\\midrule\")\n", + "ml_ltx = \"\\n\".join(ml_ltx2)\n", + "# add_midrules(sr_ltx)\n", + "with open(\"../../paper/tables/ml_methods_hp_table.tex\", \"w\") as f:\n", " f.write(ml_ltx)" ] }, @@ -172,7 +175,8 @@ "metadata": {}, "outputs": [], "source": [ - "for s in ml_ltx.splitlines(): print(s[:80])" + "for s in ml_ltx.splitlines():\n", + " print(s[:80])" ] }, { @@ -181,8 +185,7 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "add_midrules(sr_ltx)\n" + "add_midrules(sr_ltx)" ] }, { diff --git a/experiment/metrics/evaluation.py b/experiment/metrics/evaluation.py index 75c7a203..36fdf977 100644 --- a/experiment/metrics/evaluation.py +++ b/experiment/metrics/evaluation.py @@ -1,7 +1,8 @@ +import signal + import numpy as np -from sklearn.metrics import accuracy_score, mean_squared_error import sympy as sp -from sympy.parsing.sympy_parser import parse_expr +from sklearn.metrics import mean_squared_error """ Timeout handling @@ -9,30 +10,34 @@ MAXTIME = 60 -import signal + class SimplifyTimeOutException(Exception): pass + def alarm_handler(signum, frame): - print(f"raising SimplifyTimeOutException") + print("raising SimplifyTimeOutException") raise SimplifyTimeOutException """ For all of these, higher is better """ + + def accuracy(est, X, y): pred = est.predict(X) mse = mean_squared_error(y, pred) y_var = np.var(y) - if(y_var == 0.0): + if y_var == 0.0: y_var = 1e-9 r2 = 1 - mse / y_var r2 = np.round(r2, 3) return r2 + """ Utilities """ @@ -49,79 +54,77 @@ def round_floats(ex1): return ex2 - def get_symbolic_model(pred_model, local_dict): # TODO: update namespace for exact_formula runs sp_model = sp.parse_expr(pred_model, local_dict=local_dict) sp_model = round_floats(sp_model) signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: sp_model = sp.simplify(sp_model) except Exception as e: - print('Warning: simplify failed. Msg:',e) + print("Warning: simplify failed. Msg:", e) pass return sp_model + def simplicity(pred_model, feature_names): - local_dict = {f:sp.Symbol(f) for f in feature_names} + local_dict = {f: sp.Symbol(f) for f in feature_names} sp_model = get_symbolic_model(pred_model, local_dict) # compute num. components num_components = 0 for _ in sp.preorder_traversal(sp_model): num_components += 1 # compute simplicity as per juding criteria - simplicity = -np.round(np.log(num_components)/np.log(5), 1) + simplicity = -np.round(np.log(num_components) / np.log(5), 1) return simplicity + """ Problem specific """ + + def symbolic_equivalence(true_model, pred_model, local_dict): """Check whether symbolic model is equivalent to the ground truth model.""" sp_model = get_symbolic_model(pred_model, local_dict) sym_diff = round_floats(true_model - sp_model) - sym_frac = round_floats(sp_model/true_model) - print('true_model:',true_model, '; \npred_model:',sp_model) + sym_frac = round_floats(sp_model / true_model) + print("true_model:", true_model, "; \npred_model:", sp_model) try: - diff_const=sym_diff.is_constant(simplify=False) - frac_const=sym_frac.is_constant(simplify=False) + diff_const = sym_diff.is_constant(simplify=False) + frac_const = sym_frac.is_constant(simplify=False) # check if we can skip simplification if not diff_const and not frac_const: signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: if not diff_const: sym_diff = sp.simplify(sym_diff) - diff_const=sym_diff.is_constant() + diff_const = sym_diff.is_constant() if not frac_const: sym_frac = sp.simplify(sym_frac) - frac_const=sym_frac.is_constant() + frac_const = sym_frac.is_constant() except Exception as e: - print('Warning: simplify failed. Msg:',e) + print("Warning: simplify failed. Msg:", e) pass - except Exception as e: - print('const checking failed.') - diff_const=False - frac_const=False + except Exception: + print("const checking failed.") + diff_const = False + frac_const = False pass - result = dict( - equivalent = ( - str(sym_diff) == '0' - or diff_const - or frac_const - ), - sym_diff = str(sym_diff), - sym_frac = str(sym_frac), - true_model = str(true_model), - pred_model = str(sp_model) - ) - - print('result:',result) + equivalent=(str(sym_diff) == "0" or diff_const or frac_const), + sym_diff=str(sym_diff), + sym_frac=str(sym_frac), + true_model=str(true_model), + pred_model=str(sp_model), + ) + + print("result:", result) return result diff --git a/experiment/metrics/test_evaluation.py b/experiment/metrics/test_evaluation.py index 4917b17e..b080f0c9 100644 --- a/experiment/metrics/test_evaluation.py +++ b/experiment/metrics/test_evaluation.py @@ -1,81 +1,101 @@ -from .evaluation import * +import sympy as sp + +from .evaluation import get_symbolic_model, simplicity, symbolic_equivalence # define local namespace dictionary SEIR_VARS = {} -for s in ['dS', 'dE', 'dI', 'dR', - 'x_S', 'x_E', 'x_I', 'x_R', 'x_rzero', 'x_gamma', - 'x_sigma', 'x_c', 'x_N', 'x_Aux']: - SEIR_VARS.update({ s: sp.Symbol(s) }) +for s in [ + "dS", + "dE", + "dI", + "dR", + "x_S", + "x_E", + "x_I", + "x_R", + "x_rzero", + "x_gamma", + "x_sigma", + "x_c", + "x_N", + "x_Aux", +]: + SEIR_VARS.update({s: sp.Symbol(s)}) seir_gts = { - "dS": get_symbolic_model('-x_gamma*x_rzero*x_S*x_I/x_N',SEIR_VARS), - "dE": get_symbolic_model('x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E', SEIR_VARS), - "dI": get_symbolic_model('x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N', SEIR_VARS), - "dR": get_symbolic_model('x_gamma*x_I-x_c*x_R*x_I/x_N', SEIR_VARS) + "dS": get_symbolic_model("-x_gamma*x_rzero*x_S*x_I/x_N", SEIR_VARS), + "dE": get_symbolic_model("x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E", SEIR_VARS), + "dI": get_symbolic_model("x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N", SEIR_VARS), + "dR": get_symbolic_model("x_gamma*x_I-x_c*x_R*x_I/x_N", SEIR_VARS), } + def test_seir_true(): """SEIR models evaluate as solutions""" pred_models = [ - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N'), - ("dE", 'x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E'), - ("dI", 'x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N'), - ("dR", 'x_gamma*x_I-x_c*x_R*x_I/x_N'), - ("dE", 'x_E*(-x_sigma) + x_gamma*x_S*x_I/(x_N/x_rzero)'), - ("dE", '(-x_E*x_sigma**2 + x_gamma*x_sigma*x_rzero*x_S*x_I/x_N)/x_sigma') + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N"), + ("dE", "x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E"), + ("dI", "x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N"), + ("dR", "x_gamma*x_I-x_c*x_R*x_I/x_N"), + ("dE", "x_E*(-x_sigma) + x_gamma*x_S*x_I/(x_N/x_rzero)"), + ("dE", "(-x_E*x_sigma**2 + x_gamma*x_sigma*x_rzero*x_S*x_I/x_N)/x_sigma"), ] - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_offset(): """SEIR models evaluate as solutions when offset""" pred_models = [ - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N+1.80'), - ("dE", 'x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E - 3.12'), - ("dI", 'x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N + 678.0'), - ("dR", 'x_gamma*x_I-x_c*x_R*x_I/x_N-0.043'), + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N+1.80"), + ("dE", "x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E - 3.12"), + ("dI", "x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N + 678.0"), + ("dR", "x_gamma*x_I-x_c*x_R*x_I/x_N-0.043"), ] - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_scale(): """SEIR models evaluate as solutions when scaled""" pred_models = { - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N*1.80'), - ("dE", '(x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E) * 3.12'), - ("dI", '(x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N)/678.0'), - ("dR", '(x_gamma*x_I-x_c*x_R*x_I/x_N)*0.043'), - ("dE", 'x_E*(-3*x_sigma) + x_gamma*x_S*x_I/(0.33333*x_N/x_rzero)'), + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N*1.80"), + ("dE", "(x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E) * 3.12"), + ("dI", "(x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N)/678.0"), + ("dR", "(x_gamma*x_I-x_c*x_R*x_I/x_N)*0.043"), + ("dE", "x_E*(-3*x_sigma) + x_gamma*x_S*x_I/(0.33333*x_N/x_rzero)"), } - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_round(): """SEIR models evaluate as solutions when rounded""" pred_models = { - "dE": '(x_gamma*x_rzero*x_S*x_I/x_N*1.0003 - x_sigma*x_E*0.9999)', - "dI": 'x_sigma*x_E*0.9999-x_gamma*x_I+x_c*x_R*x_I/x_N*1.0001', - "dR": '(x_gamma*x_I*1.0004-x_c*x_R*x_I/x_N)*0.043', + "dE": "(x_gamma*x_rzero*x_S*x_I/x_N*1.0003 - x_sigma*x_E*0.9999)", + "dI": "x_sigma*x_E*0.9999-x_gamma*x_I+x_c*x_R*x_I/x_N*1.0001", + "dR": "(x_gamma*x_I*1.0004-x_c*x_R*x_I/x_N)*0.043", } - for k,v in pred_models.items(): + for k, v in pred_models.items(): result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_simplicity_comparator(): """Test comparison has the right sign""" - big = '2.35*( 0.4 * x1 * x2 - 1.5 * x1 + 2.5 * x2 + 1 + log(30 * x3**2))' - small = '2.35*( 0.4 * x1 * x2 - 1.5 * x1 )' - features = ['x1','x2','x3','x4','x5'] - assert simplicity(big, features) < simplicity(small, features) \ No newline at end of file + big = "2.35*( 0.4 * x1 * x2 - 1.5 * x1 + 2.5 * x2 + 1 + log(30 * x3**2))" + small = "2.35*( 0.4 * x1 * x2 - 1.5 * x1 )" + features = ["x1", "x2", "x3", "x4", "x5"] + assert simplicity(big, features) < simplicity(small, features) diff --git a/experiment/read_file.py b/experiment/read_file.py index da901c5a..b9f7e780 100755 --- a/experiment/read_file.py +++ b/experiment/read_file.py @@ -1,30 +1,31 @@ -import pandas as pd import numpy as np +import pandas as pd -def read_file(filename, label='target', use_dataframe=True, sep=None): + +def read_file(filename, label="target", use_dataframe=True, sep=None): if filename is None: - raise ValueError('filename is required') - - if filename.endswith('gz'): - compression = 'gzip' + raise ValueError("filename is required") + + if filename.endswith("gz"): + compression = "gzip" else: compression = None - - print('compression:',compression) - print('filename:',filename) + + print("compression:", compression) + print("filename:", filename) if sep is None: - if filename.endswith(('.tsv', '.tsv.gz')): - sep = '\t' - elif filename.endswith(('.csv', '.csv.gz')): - sep = ',' + if filename.endswith((".tsv", ".tsv.gz")): + sep = "\t" + elif filename.endswith((".csv", ".csv.gz")): + sep = "," else: - sep = ',' + sep = "," input_data = pd.read_csv(filename, sep=sep, compression=compression) - + # clean up column names - clean_names = {k:k.strip().replace('.','_') for k in input_data.columns} + clean_names = {k: k.strip().replace(".", "_") for k in input_data.columns} input_data = input_data.rename(columns=clean_names) feature_names = [x for x in input_data.columns.values if x != label] @@ -35,8 +36,6 @@ def read_file(filename, label='target', use_dataframe=True, sep=None): X = X.values y = input_data[label].values - assert(X.shape[1] == feature_names.shape[0]) + assert X.shape[1] == feature_names.shape[0] return X, y, feature_names - - diff --git a/experiment/seeds.py b/experiment/seeds.py index 385ad33a..915b0cce 100755 --- a/experiment/seeds.py +++ b/experiment/seeds.py @@ -1,13 +1,103 @@ # fixed seeds for the experiment SEEDS = [ - 23654, 15795, 860, 5390, 16850, 29910, 4426, 21962, 14423, 28020, - 29802, 21575, 11964, 11284, 22118, 6265, 11363, 27495, 16023, 8322, - 1685, 32052, 769, 26967, 30187, 32157, 23333, 2433, 5311, 5051, - 6420, 17568, 20939, 19769, 28693, 6396, 29419, 27480, 32304, 8666, - 25658, 18942, 24233, 18431, 32219, 2747, 25551, 26382, 189, 31677, - 19118, 3005, 21042, 1899, 24118, 1267, 31551, 17912, 11394, 3556, - 3890, 8838, 30740, 27464, 14502, 21777, 10627, 8792, 10555, 10253, - 8433, 10233, 11016, 23897, 2612, 23425, 25939, 22619, 21870, 23483, - 26054, 15787, 27132, 17159, 12206, 8226, 14541, 3152, 26531, 1585, - 3943, 23939, 19457, 1021, 11653, 10805, 13417, 20227, 7989, 9692 + 23654, + 15795, + 860, + 5390, + 16850, + 29910, + 4426, + 21962, + 14423, + 28020, + 29802, + 21575, + 11964, + 11284, + 22118, + 6265, + 11363, + 27495, + 16023, + 8322, + 1685, + 32052, + 769, + 26967, + 30187, + 32157, + 23333, + 2433, + 5311, + 5051, + 6420, + 17568, + 20939, + 19769, + 28693, + 6396, + 29419, + 27480, + 32304, + 8666, + 25658, + 18942, + 24233, + 18431, + 32219, + 2747, + 25551, + 26382, + 189, + 31677, + 19118, + 3005, + 21042, + 1899, + 24118, + 1267, + 31551, + 17912, + 11394, + 3556, + 3890, + 8838, + 30740, + 27464, + 14502, + 21777, + 10627, + 8792, + 10555, + 10253, + 8433, + 10233, + 11016, + 23897, + 2612, + 23425, + 25939, + 22619, + 21870, + 23483, + 26054, + 15787, + 27132, + 17159, + 12206, + 8226, + 14541, + 3152, + 26531, + 1585, + 3943, + 23939, + 19457, + 1021, + 11653, + 10805, + 13417, + 20227, + 7989, + 9692, ] diff --git a/experiment/submit_jobs.py b/experiment/submit_jobs.py index c923722d..f92a5bcd 100755 --- a/experiment/submit_jobs.py +++ b/experiment/submit_jobs.py @@ -1,42 +1,42 @@ -from glob import glob -import os -import sys import argparse +import os +from glob import glob + from pmlb import regression_dataset_names -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Submit long jobs.", - add_help=False) - parser.add_argument('DATA_PATH',type=str) - parser.add_argument('-ml',action='store',dest='mls', type=str, - default='') - parser.add_argument('-q',action='store',dest='Q', type=str, - default='moore_normal') - parser.add_argument('-n_trials',action='store',dest='TRIALS', default=1) - parser.add_argument('-results',action='store',dest='RDIR', - default='results',type=str,help='Results directory') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Submit long jobs.", add_help=False) + parser.add_argument("DATA_PATH", type=str) + parser.add_argument("-ml", action="store", dest="mls", type=str, default="") + parser.add_argument( + "-q", action="store", dest="Q", type=str, default="moore_normal" + ) + parser.add_argument("-n_trials", action="store", dest="TRIALS", default=1) + parser.add_argument( + "-results", + action="store", + dest="RDIR", + default="results", + type=str, + help="Results directory", + ) args = parser.parse_args() -datapath = args.DATA_PATH +datapath = args.DATA_PATH -if args.mls == '': - ml_files = glob('./methods/*.py') - mls = ','.join([m[10:-3] for m in ml_files if '__' not in m]) - print('No MLs passed. Running everything in methods/*.py:', - mls) +if args.mls == "": + ml_files = glob("./methods/*.py") + mls = ",".join([m[10:-3] for m in ml_files if "__" not in m]) + print("No MLs passed. Running everything in methods/*.py:", mls) else: mls = args.mls -lpc_options = '-q {Q} -m 12000 -n_jobs 1'.format(Q=args.Q) +lpc_options = "-q {Q} -m 12000 -n_jobs 1".format(Q=args.Q) for dataset in regression_dataset_names: - f = args.DATA_PATH + '/' + dataset + '/' + dataset + '.tsv.gz' - jobline = ('python analyze.py {DATA} ' - '-ml {ML} ' - '-results {RDIR} -n_trials {NT} {LPC}').format(DATA=f, - LPC=lpc_options, - ML=mls, - RDIR=args.RDIR, - NT=args.TRIALS) + f = args.DATA_PATH + "/" + dataset + "/" + dataset + ".tsv.gz" + jobline = ( + "python analyze.py {DATA} -ml {ML} -results {RDIR} -n_trials {NT} {LPC}" + ).format(DATA=f, LPC=lpc_options, ML=mls, RDIR=args.RDIR, NT=args.TRIALS) print(jobline) os.system(jobline) diff --git a/experiment/symbolic_utils.py b/experiment/symbolic_utils.py index daec4aed..c46c49ed 100644 --- a/experiment/symbolic_utils.py +++ b/experiment/symbolic_utils.py @@ -1,19 +1,22 @@ +import ast import pdb +import re + import pandas as pd -from yaml import load, Loader import sympy +from read_file import read_file + # from sympy import * -from sympy import Symbol, simplify, factor, Float, preorder_traversal, Integer +from sympy import Float, Integer, Symbol, preorder_traversal, simplify from sympy.parsing.sympy_parser import parse_expr -from read_file import read_file -import re -import ast +from yaml import Loader, load ############################################################################### # fn definitions from the algorithms, written in sympy operators -def sub(x,y): - return sympy.Add(x,-y) +def sub(x, y): + return sympy.Add(x, -y) + # def division(x,y): # print('division') @@ -26,30 +29,37 @@ def sub(x,y): # # return result # return sympy.Mul(x,1/y) -#TODO: handle protected division -def div(x,y): - return sympy.Mul(x,1/y) + +# TODO: handle protected division +def div(x, y): + return sympy.Mul(x, 1 / y) + def square(x): - return sympy.Pow(x,2) + return sympy.Pow(x, 2) + def cube(x): - return sympy.Pow(x,3) + return sympy.Pow(x, 3) + def quart(x): - return sympy.Pow(x,4) + return sympy.Pow(x, 4) + def PLOG(x, base=None): if isinstance(x, sympy.Float): if x < 0: x = sympy.Abs(x) if base: - return sympy.log(x,base) + return sympy.log(x, base) else: return sympy.log(x) + def PLOG10(x): - return PLOG(x,10) + return PLOG(x, 10) + def PSQRT(x): if isinstance(x, sympy.Float): @@ -57,209 +67,216 @@ def PSQRT(x): return sympy.sqrt(sympy.Abs(x)) return sympy.sqrt(x) + ############################################################################### + def complexity(expr): - c=0 + c = 0 for arg in preorder_traversal(expr): c += 1 return c - + + def round_floats(ex1): ex2 = ex1 for a in preorder_traversal(ex1): if isinstance(a, Float): if abs(a) < 0.0001: - ex2 = ex2.subs(a,Integer(0)) + ex2 = ex2.subs(a, Integer(0)) else: - ex2 = ex2.subs(a, Float(round(a, 3),3)) + ex2 = ex2.subs(a, Float(round(a, 3), 3)) return ex2 + ################################################################################ # currently the MRGP model is put together incorrectly. this set of functions # corrects the MRGP model form so that it can be fed to sympy and simplified. ################################################################################ def add_commas(model): - return ''.join([m + ',' if not m.endswith('(') else m - for m in model.split()])[:-1] + return "".join([m + "," if not m.endswith("(") else m for m in model.split()])[:-1] + def decompose_mrgp_model(model_str): """split mrgp model into its betas and model parts""" - new_model=[] # get betas - betas = [float(b[0]) for b in re.findall( - r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*', - model_str)] - print('betas:',betas) + betas = [ + float(b[0]) + for b in re.findall(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*", model_str) + ] + print("betas:", betas) # get form - submodel = re.sub(pattern=r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*', - repl=r'', - string=model_str) - return betas, submodel #new_model + submodel = re.sub( + pattern=r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*", repl=r"", string=model_str + ) + return betas, submodel # new_model + def print_model(node): - if hasattr(node, 'func'): - model_str = node.func.id + '(' - elif hasattr(node, 'id'): - # model_str = node.id + if hasattr(node, "func"): + model_str = node.func.id + "(" + elif hasattr(node, "id"): + # model_str = node.id return node.id else: pdb.set_trace() - if hasattr(node, 'args'): + if hasattr(node, "args"): i = 0 for arg in node.args: model_str += print_model(arg) i += 1 if i < len(node.args): - model_str += ',' - model_str += ')' + model_str += "," + model_str += ")" # print('print_model::',model_str) return model_str + def add_betas(node, betas): beta = betas[0] betas.pop(0) if float(beta) > 0: - model_str = str(beta) + '*' + print_model(node) + model_str = str(beta) + "*" + print_model(node) i = 1 else: # print('filtering fn w beta=',beta) - model_str = '' + model_str = "" i = 0 - if hasattr(node, 'args'): + if hasattr(node, "args"): for arg in node.args: submodel = add_betas(arg, betas) - if submodel != '': - model_str += '+' if i != 0 else '' - model_str += submodel + if submodel != "": + model_str += "+" if i != 0 else "" + model_str += submodel i += 1 # print('add_betas::',model_str) return model_str + + ################################################################################ + def clean_pred_model(model_str, dataset, est_name): - mrgp = 'MRGP' in est_name - - model_str = model_str.strip() + mrgp = "MRGP" in est_name + + model_str = model_str.strip() if mrgp: - model_str = model_str.replace('+','add') + model_str = model_str.replace("+", "add") model_str = add_commas(model_str) betas, model_str = decompose_mrgp_model(model_str) - X, labels, features = read_file(dataset) - - local_dict = {k:Symbol(k) for k in features} + + local_dict = {k: Symbol(k) for k in features} new_model_str = model_str # rename features - for i,f in reversed(list(enumerate(features))): - print('replacing feature',i,'with',f) - if any([n in est_name.lower() for n in ['mrgp','operon','dsr']]): + for i, f in reversed(list(enumerate(features))): + print("replacing feature", i, "with", f) + if any([n in est_name.lower() for n in ["mrgp", "operon", "dsr"]]): i = i + 1 - new_model_str = new_model_str.replace('x'+str(i),f) - new_model_str = new_model_str.replace('x_'+str(i),f) - new_model_str = new_model_str.replace('X_'+str(i),f) - new_model_str = new_model_str.replace('X'+str(i),f) - new_model_str = new_model_str.replace('x[:,{}]'.format(i),f) - new_model_str = new_model_str.replace('x[{}]'.format(i),f) + new_model_str = new_model_str.replace("x" + str(i), f) + new_model_str = new_model_str.replace("x_" + str(i), f) + new_model_str = new_model_str.replace("X_" + str(i), f) + new_model_str = new_model_str.replace("X" + str(i), f) + new_model_str = new_model_str.replace("x[:,{}]".format(i), f) + new_model_str = new_model_str.replace("x[{}]".format(i), f) # operators - new_model_str = new_model_str.replace('^','**') - #GP-GOMEA - new_model_str = new_model_str.replace('p/','/') - new_model_str = new_model_str.replace('plog','PLOG') - new_model_str = new_model_str.replace('aq','/') + new_model_str = new_model_str.replace("^", "**") + # GP-GOMEA + new_model_str = new_model_str.replace("p/", "/") + new_model_str = new_model_str.replace("plog", "PLOG") + new_model_str = new_model_str.replace("aq", "/") # MRGP - new_model_str = new_model_str.replace('mylog','PLOG') + new_model_str = new_model_str.replace("mylog", "PLOG") # ITEA - new_model_str = new_model_str.replace('sqrtAbs','PSQRT') + new_model_str = new_model_str.replace("sqrtAbs", "PSQRT") # new_model_str = re.sub(pattern=r'sqrtAbs\((.*?)\)', # repl=r'sqrt(abs(\1))', # string=new_model_str # ) - new_model_str = new_model_str.replace('np.','') + new_model_str = new_model_str.replace("np.", "") # ellyn & FEAT - new_model_str = new_model_str.replace('|','') - new_model_str = new_model_str.replace('log','PLOG') - new_model_str = new_model_str.replace('sqrt','PSQRT') + new_model_str = new_model_str.replace("|", "") + new_model_str = new_model_str.replace("log", "PLOG") + new_model_str = new_model_str.replace("sqrt", "PSQRT") # AIFeynman - new_model_str = new_model_str.replace('pi','3.1415926535') - - local_dict.update({ - 'add':sympy.Add, - 'mul':sympy.Mul, - 'max':sympy.Max, - 'min':sympy.Min, - 'sub':sub, - 'div':div, - 'square':square, - 'cube':cube, - 'quart':quart, - 'PLOG':PLOG, - 'PLOG10':PLOG, - 'PSQRT':PSQRT - }) + new_model_str = new_model_str.replace("pi", "3.1415926535") + + local_dict.update( + { + "add": sympy.Add, + "mul": sympy.Mul, + "max": sympy.Max, + "min": sympy.Min, + "sub": sub, + "div": div, + "square": square, + "cube": cube, + "quart": quart, + "PLOG": PLOG, + "PLOG10": PLOG, + "PSQRT": PSQRT, + } + ) # BSR # get rid of square brackets - new_model_str = new_model_str.replace('[','').replace(']','') + new_model_str = new_model_str.replace("[", "").replace("]", "") - print('parsing',new_model_str) + print("parsing", new_model_str) if mrgp: - mrgp_ast = ast.parse(new_model_str, "","eval") - new_model_str = add_betas(mrgp_ast.body,betas) - assert(len(betas)==0) + mrgp_ast = ast.parse(new_model_str, "", "eval") + new_model_str = add_betas(mrgp_ast.body, betas) + assert len(betas) == 0 print(local_dict) - model_sym = parse_expr(new_model_str, local_dict = local_dict) - print('round_floats') + model_sym = parse_expr(new_model_str, local_dict=local_dict) + print("round_floats") model_sym = round_floats(model_sym) - print('rounded:',model_sym) - print('simplify...') + print("rounded:", model_sym) + print("simplify...") model_sym = simplify(model_sym, ratio=1) - print('simplified:',model_sym) + print("simplified:", model_sym) return model_sym def get_sym_model(dataset, return_str=True): """return sympy model from dataset metadata""" metadata = load( - open('/'.join(dataset.split('/')[:-1])+'/metadata.yaml','r'), - Loader=Loader + open("/".join(dataset.split("/")[:-1]) + "/metadata.yaml", "r"), Loader=Loader ) - df = pd.read_csv(dataset,sep='\t') - features = [c for c in df.columns if c != 'target'] -# print('features:',df.columns) - description = metadata['description'].split('\n') - model_str = [ms for ms in description if '=' in ms][0].split('=')[-1] - model_str = model_str.replace('pi','3.1415926535') + df = pd.read_csv(dataset, sep="\t") + features = [c for c in df.columns if c != "target"] + # print('features:',df.columns) + description = metadata["description"].split("\n") + model_str = [ms for ms in description if "=" in ms][0].split("=")[-1] + model_str = model_str.replace("pi", "3.1415926535") if return_str: return model_str # pdb.set_trace() # handle feynman problem constants -# print('model:',model_str) - model_sym = parse_expr(model_str, - local_dict = {k:Symbol(k) for k in features}) + # print('model:',model_str) + model_sym = parse_expr(model_str, local_dict={k: Symbol(k) for k in features}) model_sym = round_floats(model_sym) -# print('sym model:',model_sym) + # print('sym model:',model_sym) return model_sym + def get_sympy_model(model_str, dataset): - model_str = model_str.replace('pi', '3.1415926535') + model_str = model_str.replace("pi", "3.1415926535") - df = pd.read_csv(dataset, sep='\t') - features = [c for c in df.columns if c != 'target'] + df = pd.read_csv(dataset, sep="\t") + features = [c for c in df.columns if c != "target"] - model_sym = parse_expr( - model_str, - local_dict={k: Symbol(k) for k in features} - ) + model_sym = parse_expr(model_str, local_dict={k: Symbol(k) for k in features}) model_sym = round_floats(model_sym) return model_sym + def rewrite_AIFeynman_model_size(model_str): """AIFeynman complexity was incorrect prior to version , update it here""" return complexity(parse_expr(model_str)) diff --git a/experiment/test_algorithm.py b/experiment/test_algorithm.py index 089674a2..ebae2bf3 100644 --- a/experiment/test_algorithm.py +++ b/experiment/test_algorithm.py @@ -1,92 +1,99 @@ -import pytest -import sys -import os +import importlib import json +import os +import sys import types -from os.path import dirname as d from os.path import abspath -root_dir = d(abspath(__file__)) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') +from os.path import dirname as d + +import pytest + +sys.path.append(d(abspath(__file__))) + from evaluate_model import evaluate_model -import importlib +from read_file import read_file +from symbolic_utils import ( + complexity, + round_floats, +) +from sympy import Symbol + # symbolic model stuff from sympy.parsing.sympy_parser import parse_expr -from symbolic_utils import (complexity, round_floats, - sub, div, square, cube, quart, - PLOG, PLOG10, PSQRT) -from read_file import read_file -from sympy import Symbol -if 'OMP_NUM_THREADS' not in os.environ.keys(): - os.environ['OMP_NUM_THREADS'] = '1' -if 'OPENBLAS_NUM_THREADS' not in os.environ.keys(): - os.environ['OPENBLAS_NUM_THREADS'] = '1' -if 'MKL_NUM_THREADS' not in os.environ.keys(): - os.environ['MKL_NUM_THREADS'] = '1' +if "OMP_NUM_THREADS" not in os.environ.keys(): + os.environ["OMP_NUM_THREADS"] = "1" +if "OPENBLAS_NUM_THREADS" not in os.environ.keys(): + os.environ["OPENBLAS_NUM_THREADS"] = "1" +if "MKL_NUM_THREADS" not in os.environ.keys(): + os.environ["MKL_NUM_THREADS"] = "1" # shared variables -dataset = 'test/192_vineyard_small.tsv.gz' -results_path = 'tmp_results' +dataset = "test/192_vineyard_small.tsv.gz" +results_path = "tmp_results" random_state = 42 + def get_algorithm(ml): - algorithm = importlib.__import__('methods.'+ml+'.regressor',globals(), - locals(), - ['*']) - print('algorithm imported:',algorithm) + algorithm = importlib.__import__( + "methods." + ml + ".regressor", globals(), locals(), ["*"] + ) + print("algorithm imported:", algorithm) return algorithm + def test_import(ml): """Check algorithm imports""" algorithm = get_algorithm(ml) - assert 'est' in dir(algorithm) - assert 'model' in dir(algorithm) + assert "est" in dir(algorithm) + assert "model" in dir(algorithm) eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs eval_kwarg_types = { - 'test_params':dict, - 'max_train_samples':int, - 'scale_x':bool, - 'scale_y':bool, - 'pre_train':types.FunctionType, - 'use_dataframe':bool + "test_params": dict, + "max_train_samples": int, + "scale_x": bool, + "scale_y": bool, + "pre_train": types.FunctionType, + "use_dataframe": bool, } - for k,v in eval_kwargs.items(): - assert k in ['test_params', - 'max_train_samples', - 'scale_x', - 'scale_y', - 'pre_train', - 'use_dataframe' - ] + for k, v in eval_kwargs.items(): + assert k in [ + "test_params", + "max_train_samples", + "scale_x", + "scale_y", + "pre_train", + "use_dataframe", + ] assert isinstance(v, eval_kwarg_types[k]) + # TODO: need to update evaluate model to handle this format def test_evaluate(ml): """Dataset evaluation""" algorithm = get_algorithm(ml) - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs else: eval_kwargs = {} - print(eval_kwargs) + print(eval_kwargs) json_file = evaluate_model( - dataset=dataset, # input file - results_path=results_path, # results directory - random_state=random_state, # random state seed - est_name=ml, # estimator name - est=algorithm.est, # estimator class + dataset=dataset, # input file + results_path=results_path, # results directory + random_state=random_state, # random state seed + est_name=ml, # estimator name + est=algorithm.est, # estimator class algorithm=algorithm, - model=algorithm.model, # method to extract the model from the estimator - test=True, # testing - **eval_kwargs + model=algorithm.model, # method to extract the model from the estimator + test=True, # testing + **eval_kwargs, ) print(json_file) print("hello") @@ -98,35 +105,36 @@ def test_sympy(ml): algorithm = get_algorithm(ml) - dataset_name = dataset.split('/')[-1][:-7] - json_file = (results_path + '/' + dataset_name + '_' + ml + '_' - + str(random_state) + '.json') - + dataset_name = dataset.split("/")[-1][:-7] + json_file = ( + results_path + "/" + dataset_name + "_" + ml + "_" + str(random_state) + ".json" + ) + if os.path.exists(json_file): - r = json.load(open(json_file, 'r')) + r = json.load(open(json_file, "r")) else: - raise FileNotFoundError(json_file+' not found') + raise FileNotFoundError(json_file + " not found") - est_name = r['algorithm'] + r["algorithm"] - raw_model = r['symbolic_model'] - print('raw_model:',raw_model) + raw_model = r["symbolic_model"] + print("raw_model:", raw_model) X, labels, features = read_file(dataset) - local_dict = {k:Symbol(k) for k in features} + local_dict = {k: Symbol(k) for k in features} - model_sym = parse_expr(raw_model, local_dict = local_dict) + model_sym = parse_expr(raw_model, local_dict=local_dict) model_sym = round_floats(model_sym) - print('sym model:',model_sym) + print("sym model:", model_sym) # Should pass - if hasattr(algorithm, 'complexity'): + if hasattr(algorithm, "complexity"): try: model_complexity = algorithm.complexity(model_sym) - print('model complexity (author provided):', model_complexity) + print("model complexity (author provided):", model_complexity) except Exception as exc: - print('algorithm complexity failed; using fallback:', exc) + print("algorithm complexity failed; using fallback:", exc) model_complexity = complexity(model_sym) - print('model complexity (using our implementation):', model_complexity) - else: # Given that the string is sympy compatible, this should work + print("model complexity (using our implementation):", model_complexity) + else: # Given that the string is sympy compatible, this should work model_complexity = complexity(model_sym) - print('model complexity (using our implementation):', model_complexity) + print("model complexity (using our implementation):", model_complexity) diff --git a/experiment/test_evaluate_model.py b/experiment/test_evaluate_model.py index d4fd00c1..1d8bcdd5 100755 --- a/experiment/test_evaluate_model.py +++ b/experiment/test_evaluate_model.py @@ -1,36 +1,40 @@ +import importlib import sys -from os.path import dirname as d from os.path import abspath -root_dir = d(d(abspath(__file__))) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') +from os.path import dirname as d + +sys.path.append(d(d(abspath(__file__)))) + from evaluate_model import evaluate_model -import importlib + def test_evaluate_model(ml): - print('running test_evaluate_model with ml=',ml) - dataset = 'test/192_vineyard_small.tsv.gz' - results_path = 'tmp_results' + print("running test_evaluate_model with ml=", ml) + dataset = "test/192_vineyard_small.tsv.gz" + results_path = "tmp_results" random_state = 42 - algorithm = importlib.__import__(f'methods.{ml}.regressor',globals(), - locals(), - ['est','hyper_params','complexity']) + algorithm = importlib.__import__( + f"methods.{ml}.regressor", + globals(), + locals(), + ["est", "hyper_params", "complexity"], + ) - print('algorithm imported:',algorithm) + print("algorithm imported:", algorithm) eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs evaluate_model( - dataset=dataset, - results_path=results_path, - random_state=random_state, + dataset=dataset, + results_path=results_path, + random_state=random_state, est_name=ml, est=algorithm.est, model=algorithm.model, algorithm=algorithm, - test=True, # testing - **eval_kwargs + test=True, # testing + **eval_kwargs, ) diff --git a/experiment/test_tuned_models.py b/experiment/test_tuned_models.py index 96b652a2..3f2515ee 100644 --- a/experiment/test_tuned_models.py +++ b/experiment/test_tuned_models.py @@ -1,54 +1,50 @@ +import importlib import sys +from os.path import abspath from os.path import dirname as d -from os.path import abspath, join -root_dir = d(d(abspath(__file__))) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') -import pytest -from glob import glob +sys.path.append(d(d(abspath(__file__)))) + + from evaluate_model import evaluate_model -import importlib + def test_tuned_models(ml): - print('running test_evaluate_model with ml=',ml) - dataset = 'test/strogatz_shearflow1.tsv.gz' - results_path = 'tmp_results' + print("running test_evaluate_model with ml=", ml) + dataset = "test/strogatz_shearflow1.tsv.gz" + results_path = "tmp_results" random_state = 42 - algorithm = importlib.__import__('methods.'+ml, - globals(), - locals(), - ['*'] - ) + algorithm = importlib.__import__("methods." + ml, globals(), locals(), ["*"]) - print('algorithm:',algorithm.est) - if 'hyper_params' not in dir(algorithm): - algorithm.hyper_params = {} - print('hyperparams:',algorithm.hyper_params) + print("algorithm:", algorithm.est) + if "hyper_params" not in dir(algorithm): + setattr(algorithm, "hyper_params", {}) + print("hyperparams:", algorithm.hyper_params) # optional keyword arguments passed to evaluate eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs # check for conflicts btw cmd line args and eval_kwargs # if args.SYM_DATA: - eval_kwargs['scale_x'] = False - eval_kwargs['scale_y'] = False - eval_kwargs['skip_tuning'] = True - eval_kwargs['sym_data'] = True - - evaluate_model(dataset, - results_path, - random_state, - ml, - algorithm.est, - algorithm.hyper_params, - algorithm.complexity, - algorithm.model, - target_noise=0, - feature_noise=0, - test=True, # testing - **eval_kwargs - ) + eval_kwargs["scale_x"] = False + eval_kwargs["scale_y"] = False + eval_kwargs["skip_tuning"] = True + eval_kwargs["sym_data"] = True + + evaluate_model( + dataset, + results_path, + random_state, + ml, + algorithm.est, + algorithm.hyper_params, + algorithm.complexity, + algorithm.model, + target_noise=0, + feature_noise=0, + test=True, # testing + **eval_kwargs, + ) diff --git a/experiment/utils.py b/experiment/utils.py index 2b27ab69..2a6e57ce 100755 --- a/experiment/utils.py +++ b/experiment/utils.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd + def jsonify(d): """recursively formats dicts for json serialization""" if isinstance(d, list): @@ -13,18 +14,17 @@ def jsonify(d): d[k] = jsonify(d[k]) elif isinstance(d, np.ndarray): return d.tolist() - elif d.__class__.__name__.startswith('int'): + elif d.__class__.__name__.startswith("int"): return int(d) - elif d.__class__.__name__.startswith('float'): + elif d.__class__.__name__.startswith("float"): return float(d) elif isinstance(d, pd.DataFrame) or isinstance(d, pd.Series): return d.values.tolist() elif isinstance(d, bool): return d - elif d == None: + elif d is None: return None elif not isinstance(d, str): - print("WARNING: attempting to store ",d,"as a str for json") + print("WARNING: attempting to store ", d, "as a str for json") return str(d) return d - diff --git a/postprocessing/blackbox_results.ipynb b/postprocessing/blackbox_results.ipynb index 1bdf5ae3..b9257326 100644 --- a/postprocessing/blackbox_results.ipynb +++ b/postprocessing/blackbox_results.ipynb @@ -15,67 +15,66 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import json\n", + "import os\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from glob import glob\n", - "from tqdm import tqdm\n", + "import pandas as pd\n", "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib\n", - "matplotlib.rc('pdf', fonttype=42)\n", - "import os\n", - "sns.set(font_scale=1.25, style='whitegrid')\n", "\n", - "rdir = '../results/'\n", - "# figdir = 'figs/black-box' \n", - "figdir = '../../paper/figs/results_pmlb_r1/'\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", + "\n", + "sns.set(font_scale=1.25, style=\"whitegrid\")\n", + "\n", + "rdir = \"../results/\"\n", + "# figdir = 'figs/black-box'\n", + "figdir = \"../../paper/figs/results_pmlb_r1/\"\n", "# Set to path to PMLB datasets\n", - "datadir = '../../../pmlb/datasets/'\n", - "print('figdir:',figdir)\n", - "def save(name='tmp',h=None):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "datadir = \"../../../pmlb/datasets/\"\n", + "print(\"figdir:\", figdir)\n", + "\n", + "\n", + "def save(name=\"tmp\", h=None):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',figdir+'/'+name+'.pdf')\n", + " print(\"saving\", figdir + \"/\" + name + \".pdf\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.pdf', bbox_inches='tight')\n", - " \n", + " plt.savefig(figdir + \"/\" + name + \".pdf\", bbox_inches=\"tight\")\n", + "\n", + "\n", "symbolic_algs = [\n", - " 'AFP', \n", - " 'AFP_FE',\n", - " 'BSR',\n", - " 'DSR',\n", - " 'FFX',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'GP-GOMEA',\n", - " 'gplearn',\n", - " 'ITEA', \n", - " 'MRGP', \n", - " 'Operon',\n", - " 'SBP-GP',\n", - " 'AIFeynman'\n", - "]\n", - "nongp_algs = [\n", - " 'BSR',\n", - " 'DSR',\n", - " 'AIFeynman'\n", + " \"AFP\",\n", + " \"AFP_FE\",\n", + " \"BSR\",\n", + " \"DSR\",\n", + " \"FFX\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"GP-GOMEA\",\n", + " \"gplearn\",\n", + " \"ITEA\",\n", + " \"MRGP\",\n", + " \"Operon\",\n", + " \"SBP-GP\",\n", + " \"AIFeynman\",\n", "]\n", + "nongp_algs = [\"BSR\", \"DSR\", \"AIFeynman\"]\n", "gp_algs = [\n", - " 'AFP', \n", - " 'AFP_FE',\n", - " 'FFX',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'GP-GOMEA',\n", - " 'gplearn',\n", - " 'ITEA', \n", - " 'MRGP', \n", - " 'Operon',\n", - " 'SBP-GP',\n", + " \"AFP\",\n", + " \"AFP_FE\",\n", + " \"FFX\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"GP-GOMEA\",\n", + " \"gplearn\",\n", + " \"ITEA\",\n", + " \"MRGP\",\n", + " \"Operon\",\n", + " \"SBP-GP\",\n", "]" ] }, @@ -92,7 +91,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_results = pd.read_feather('../results/black-box_results.feather')" + "df_results = pd.read_feather(\"../results/black-box_results.feather\")" ] }, { @@ -108,33 +107,40 @@ "metadata": {}, "outputs": [], "source": [ - "df_results2 = df_results.merge(df_results.groupby('dataset')['algorithm'].nunique().reset_index(),\n", - " on='dataset',suffixes=('','_count'))\n", + "df_results2 = df_results.merge(\n", + " df_results.groupby(\"dataset\")[\"algorithm\"].nunique().reset_index(),\n", + " on=\"dataset\",\n", + " suffixes=(\"\", \"_count\"),\n", + ")\n", + "\n", "\n", - " \n", "# rankings per trial per dataset\n", - "for col in [c for c in df_results2.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_results2[col+'_rank_per_trial']=df_results2.groupby(['dataset','random_state'])[col].apply(lambda x: \n", - " round(x,3).rank(\n", - " ascending=ascending))\n", - " \n", - "df_sum = df_results2.groupby(['algorithm','dataset'],as_index=False).median()\n", - "df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt)\n", - "df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x))\n", + "for col in [c for c in df_results2.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_results2[col + \"_rank_per_trial\"] = df_results2.groupby(\n", + " [\"dataset\", \"random_state\"]\n", + " )[col].apply(lambda x: round(x, 3).rank(ascending=ascending))\n", + "\n", + "df_sum = df_results2.groupby([\"algorithm\", \"dataset\"], as_index=False).median()\n", + "df_sum[\"rmse_test\"] = df_sum[\"mse_test\"].apply(np.sqrt)\n", + "df_sum[\"log_mse_test\"] = df_sum[\"mse_test\"].apply(lambda x: np.log(1 + x))\n", "df_results = df_results2\n", "\n", "# rankings and normalized scores per dataset\n", - "for col in [c for c in df_sum.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_sum[col+'_rank']=df_sum.groupby(['dataset'])[col].apply(lambda x: \n", - " round(x,3).rank(ascending=ascending)\n", - " )\n", - " df_sum[col+'_norm'] = df_sum.groupby('dataset')[col].apply(lambda x: (x-x.min())/(x.max()-x.min()))\n", + "for col in [c for c in df_sum.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_sum[col + \"_rank\"] = df_sum.groupby([\"dataset\"])[col].apply(\n", + " lambda x: round(x, 3).rank(ascending=ascending)\n", + " )\n", + " df_sum[col + \"_norm\"] = df_sum.groupby(\"dataset\")[col].apply(\n", + " lambda x: (x - x.min()) / (x.max() - x.min())\n", + " )\n", "###################\n", "# save summary data\n", "###################\n", - "df_sum.to_csv(rdir+'results_black-box_summary.csv.gz',compression='gzip',index=False)\n", + "df_sum.to_csv(\n", + " rdir + \"results_black-box_summary.csv.gz\", compression=\"gzip\", index=False\n", + ")\n", "\n", "df_sum" ] @@ -152,39 +158,48 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "def compare(df_compare, x='r2_test',y='algorithm', row=None, col=None, scale=None, xlim=[],**kwargs):\n", + "def compare(\n", + " df_compare,\n", + " x=\"r2_test\",\n", + " y=\"algorithm\",\n", + " row=None,\n", + " col=None,\n", + " scale=None,\n", + " xlim=[],\n", + " **kwargs,\n", + "):\n", " plt.figure()\n", " order = df_compare.groupby(y)[x].median().sort_values(ascending=False).index\n", - " if scale=='log' and len(xlim)>0 and xlim[0] == 0:\n", - " df_compare.loc[:,x] += 1\n", + " if scale == \"log\" and len(xlim) > 0 and xlim[0] == 0:\n", + " df_compare.loc[:, x] += 1\n", " xlim[0] = 1\n", - " xnew = '1 + '+x\n", - " df_compare=df_compare.rename(columns={x:xnew})\n", + " xnew = \"1 + \" + x\n", + " df_compare = df_compare.rename(columns={x: xnew})\n", " x = xnew\n", - " \n", - " sns.catplot(data=df_compare, \n", - " kind='point',\n", - "# color='w',\n", - " y=y,\n", - " x=x,\n", - " order=order,\n", - " fliersize=0,\n", - "# notch=True,\n", - " row=row,\n", - " col=col,\n", - " palette='flare_r',\n", - " **kwargs\n", - " )\n", - " plt.ylabel('')\n", - " if len(xlim)>0:\n", - " plt.xlim(xlim[0],xlim[1])\n", + "\n", + " sns.catplot(\n", + " data=df_compare,\n", + " kind=\"point\",\n", + " # color='w',\n", + " y=y,\n", + " x=x,\n", + " order=order,\n", + " fliersize=0,\n", + " # notch=True,\n", + " row=row,\n", + " col=col,\n", + " palette=\"flare_r\",\n", + " **kwargs,\n", + " )\n", + " plt.ylabel(\"\")\n", + " if len(xlim) > 0:\n", + " plt.xlim(xlim[0], xlim[1])\n", " if scale:\n", " plt.gca().set_xscale(scale)\n", - " \n", - " save(name='_'.join(['cat-boxplot',x + '-by-'+ y]) )\n", + "\n", + " save(name=\"_\".join([\"cat-boxplot\", x + \"-by-\" + y]))\n", " if col:\n", - " save(name='_'.join(['cat-boxplot',x + '-by-'+ y] + [col]) )" + " save(name=\"_\".join([\"cat-boxplot\", x + \"-by-\" + y] + [col]))" ] }, { @@ -202,65 +217,75 @@ "source": [ "# Make the PairGrid\n", "sns.set_style(\"whitegrid\")\n", - "df_plot=df_sum.copy()\n", + "df_plot = df_sum.copy()\n", "\n", - "df_plot['*algorithm*'] = df_plot['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x)\n", + "df_plot[\"*algorithm*\"] = df_plot[\"algorithm\"].apply(\n", + " lambda x: \"*\" + x if x in symbolic_algs else x\n", + ")\n", "# different options\n", - "x_vars=[\n", - "# 'rmse_test',\n", - "# 'log_mse_test',\n", - "# 'r2_test_norm',\n", - " 'r2_test',\n", - "# 'r2_test_rank',\n", - " 'model_size',\n", - "# 'model_size_rank',\n", - " 'training time (s)',\n", + "x_vars = [\n", + " # 'rmse_test',\n", + " # 'log_mse_test',\n", + " # 'r2_test_norm',\n", + " \"r2_test\",\n", + " # 'r2_test_rank',\n", + " \"model_size\",\n", + " # 'model_size_rank',\n", + " \"training time (s)\",\n", "]\n", - "order = df_plot.groupby('*algorithm*')[x_vars[0]].median().sort_values(\n", - " ascending='r2' not in x_vars[0] or 'rank' in x_vars[0]).index\n", - "g = sns.PairGrid(df_plot, \n", - " x_vars=x_vars,\n", - " y_vars=['*algorithm*'],\n", - " height=6.5, \n", - " aspect=0.6,\n", - "# hue='symbolic_dataset'\n", - " )\n", - "\n", - "# Draw a dot plot \n", - "g.map(sns.pointplot, \n", - " size=10,\n", - " orient=\"h\",\n", - " jitter=False,\n", - " order=order,\n", - " palette=\"flare_r\",\n", - " linewidth=1,\n", - " markeredgecolor='w',\n", - " join=False,\n", - " estimator=np.median,\n", - " n_boot=1000,\n", - " ci=95\n", - " )\n", + "order = (\n", + " df_plot.groupby(\"*algorithm*\")[x_vars[0]]\n", + " .median()\n", + " .sort_values(ascending=\"r2\" not in x_vars[0] or \"rank\" in x_vars[0])\n", + " .index\n", + ")\n", + "g = sns.PairGrid(\n", + " df_plot,\n", + " x_vars=x_vars,\n", + " y_vars=[\"*algorithm*\"],\n", + " height=6.5,\n", + " aspect=0.6,\n", + " # hue='symbolic_dataset'\n", + ")\n", + "\n", + "# Draw a dot plot\n", + "g.map(\n", + " sns.pointplot,\n", + " size=10,\n", + " orient=\"h\",\n", + " jitter=False,\n", + " order=order,\n", + " palette=\"flare_r\",\n", + " linewidth=1,\n", + " markeredgecolor=\"w\",\n", + " join=False,\n", + " estimator=np.median,\n", + " n_boot=1000,\n", + " ci=95,\n", + ")\n", "# # Use semantically meaningful titles for the columns\n", - "titles = [x.replace('_',' ').title().replace('(S)','(s)').replace('R2','$R^2$') for x in x_vars]\n", - "g.axes.flat[0].set_ylabel('')\n", + "titles = [\n", + " x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\").replace(\"R2\", \"$R^2$\")\n", + " for x in x_vars\n", + "]\n", + "g.axes.flat[0].set_ylabel(\"\")\n", "for ax, title in zip(g.axes.flat, titles):\n", - "\n", " # Set a different title for each axes\n", " ax.set(title=title)\n", - " ax.set_xlabel('')\n", - " \n", - " if any([n in title.lower() for n in ['size','time']]):\n", - " ax.set_xscale('log')\n", - " \n", - " if title == '$R^2$ Test':\n", - " ax.set_xlim([-.25,1])\n", + " ax.set_xlabel(\"\")\n", + "\n", + " if any([n in title.lower() for n in [\"size\", \"time\"]]):\n", + " ax.set_xscale(\"log\")\n", + "\n", + " if title == \"$R^2$ Test\":\n", + " ax.set_xlim([-0.25, 1])\n", "\n", " # Make the grid horizontal instead of vertical\n", " ax.yaxis.grid(True)\n", "\n", "sns.despine(left=True, bottom=True)\n", "\n", - "save(name='_'.join(['pairgrid-pointplot']+x_vars))" + "save(name=\"_\".join([\"pairgrid-pointplot\"] + x_vars))" ] }, { @@ -278,76 +303,89 @@ "source": [ "# Make the PairGrid\n", "sns.set_style(\"whitegrid\")\n", - "df_plot=df_sum.copy()\n", - "df_plot['*algorithm*'] = df_plot['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x)\n", - "df_plot = df_plot[['*algorithm*','r2_test','r2_test_norm','r2_test_rank','friedman_dataset']]\n", - "\n", - "x='r2_test'\n", - "markers = ['o','s','x']\n", - "colors = sns.color_palette('flare',n_colors=3)\n", - "for x in ['r2_test_norm']:\n", - " h = plt.figure(figsize=(12,6))\n", + "df_plot = df_sum.copy()\n", + "df_plot[\"*algorithm*\"] = df_plot[\"algorithm\"].apply(\n", + " lambda x: \"*\" + x if x in symbolic_algs else x\n", + ")\n", + "df_plot = df_plot[\n", + " [\"*algorithm*\", \"r2_test\", \"r2_test_norm\", \"r2_test_rank\", \"friedman_dataset\"]\n", + "]\n", + "\n", + "x = \"r2_test\"\n", + "markers = [\"o\", \"s\", \"x\"]\n", + "colors = sns.color_palette(\"flare\", n_colors=3)\n", + "for x in [\"r2_test_norm\"]:\n", + " h = plt.figure(figsize=(12, 6))\n", " ax = plt.gca()\n", - " order = df_plot.groupby('*algorithm*')[x].median().sort_values(\n", - " ascending='r2' not in x or 'rank' in x).index\n", + " order = (\n", + " df_plot.groupby(\"*algorithm*\")[x]\n", + " .median()\n", + " .sort_values(ascending=\"r2\" not in x or \"rank\" in x)\n", + " .index\n", + " )\n", " i = 0\n", " axes = []\n", - " g = sns.pointplot(ax=ax,\n", - " data=df_plot,\n", - " x=x,\n", - " y='*algorithm*',\n", - " hue='friedman_dataset',\n", - " size=10,\n", - " orient=\"h\",\n", - " order=order,\n", - " markers=['s','x'],\n", - " palette=\"flare_r\",\n", - " join=False,\n", - " estimator=np.median,\n", - " legend_out=True\n", - " )\n", + " g = sns.pointplot(\n", + " ax=ax,\n", + " data=df_plot,\n", + " x=x,\n", + " y=\"*algorithm*\",\n", + " hue=\"friedman_dataset\",\n", + " size=10,\n", + " orient=\"h\",\n", + " order=order,\n", + " markers=[\"s\", \"x\"],\n", + " palette=\"flare_r\",\n", + " join=False,\n", + " estimator=np.median,\n", + " legend_out=True,\n", + " )\n", " # all PMLB\n", - " g2 = sns.pointplot(ax=ax,\n", - " data=df_plot,\n", - " x=x,\n", - " y='*algorithm*',\n", - "# hue='friedman_dataset',\n", - " size=10,\n", - " orient=\"h\",\n", - " order=order,\n", - " color='r',\n", - "# palette=\"flare_r\",\n", - " join=False,\n", - " estimator=np.median,\n", - " markers = ['o'],\n", - " legend_out=True\n", - " )\n", + " g2 = sns.pointplot(\n", + " ax=ax,\n", + " data=df_plot,\n", + " x=x,\n", + " y=\"*algorithm*\",\n", + " # hue='friedman_dataset',\n", + " size=10,\n", + " orient=\"h\",\n", + " order=order,\n", + " color=\"r\",\n", + " # palette=\"flare_r\",\n", + " join=False,\n", + " estimator=np.median,\n", + " markers=[\"o\"],\n", + " legend_out=True,\n", + " )\n", " # add all PMLB to legend\n", " handles, labels = ax.get_legend_handles_labels()\n", - " #Create custom artists\n", - " both = plt.Line2D((0,1),(0,0), color='r', marker='o', linestyle='-', label='All')\n", + " # Create custom artists\n", + " both = plt.Line2D((0, 1), (0, 0), color=\"r\", marker=\"o\", linestyle=\"-\", label=\"All\")\n", " handles.append(both)\n", - " plt.legend(title='Data Subset',\n", - " handles=handles, \n", - " labels=['No Friedman','Only Friedman','All'])\n", - " \n", - " if x=='r2_test':\n", - " plt.xlim([-.5,1.0])\n", - " if 'norm' in x:\n", - " x='normalized_R2_test'\n", + " plt.legend(\n", + " title=\"Data Subset\",\n", + " handles=handles,\n", + " labels=[\"No Friedman\", \"Only Friedman\", \"All\"],\n", + " )\n", + "\n", + " if x == \"r2_test\":\n", + " plt.xlim([-0.5, 1.0])\n", + " if \"norm\" in x:\n", + " x = \"normalized_R2_test\"\n", " sns.despine(left=True, bottom=True)\n", - "# plt.title(prefix.replace('_',' ').title())\n", + " # plt.title(prefix.replace('_',' ').title())\n", " axes.append(ax)\n", - " i+=1\n", + " i += 1\n", " plt.tight_layout()\n", " for ax in axes:\n", - " ax.set_ylabel('')\n", - " ax.set_xlabel(x.replace('_',' ').title().replace('(S)','(s)').replace('R2','$R^2$'))\n", + " ax.set_ylabel(\"\")\n", + " ax.set_xlabel(\n", + " x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\").replace(\"R2\", \"$R^2$\")\n", + " )\n", " ax.yaxis.grid(True)\n", " ax.xaxis.grid(True)\n", "\n", - "\n", - " save(name='friedman_comparison_'+'_'.join(['pairgrid-pointplot']+[x]))" + " save(name=\"friedman_comparison_\" + \"_\".join([\"pairgrid-pointplot\"] + [x]))" ] }, { @@ -364,14 +402,22 @@ "outputs": [], "source": [ "import pandas as pd\n", + "\n", + "\n", "# for dataset, dfg in df_sum.groupby('dataset'):\n", - "def get_size(dataset,i):\n", - " df = pd.read_csv(datadir+dataset+'/'+dataset+'.tsv.gz',sep='\\t') \n", + "def get_size(dataset, i):\n", + " df = pd.read_csv(datadir + dataset + \"/\" + dataset + \".tsv.gz\", sep=\"\\t\")\n", " return df.shape[i]\n", - "df_sum.loc[:,'dataset_nsamples'] = df_sum['dataset'].apply(lambda x: get_size(x,0))\n", - "df_sum.loc[:,'dataset_nsamples_train'] = df_sum['dataset'].apply(lambda x: min(get_size(x,0),10000))\n", - "df_sum.loc[:,'dataset_nfeatures'] = df_sum['dataset'].apply(lambda x: get_size(x,1))\n", - "df_sum.loc[:,'dataset_npoints'] = df_sum['dataset_nsamples']*df_sum['dataset_nfeatures']\n", + "\n", + "\n", + "df_sum.loc[:, \"dataset_nsamples\"] = df_sum[\"dataset\"].apply(lambda x: get_size(x, 0))\n", + "df_sum.loc[:, \"dataset_nsamples_train\"] = df_sum[\"dataset\"].apply(\n", + " lambda x: min(get_size(x, 0), 10000)\n", + ")\n", + "df_sum.loc[:, \"dataset_nfeatures\"] = df_sum[\"dataset\"].apply(lambda x: get_size(x, 1))\n", + "df_sum.loc[:, \"dataset_npoints\"] = (\n", + " df_sum[\"dataset_nsamples\"] * df_sum[\"dataset_nfeatures\"]\n", + ")\n", "df_sum" ] }, @@ -381,39 +427,61 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum['Method Family'] = df_sum['algorithm'].apply(lambda x: 'Operon' if x=='Operon' else 'XGB' if x=='XGB' else 'Other Non-GP SR' if x in nongp_algs else 'Other GP SR' if x in gp_algs else 'Other ML')\n", - "sns.set_style('whitegrid')\n", - "for col in ['dataset_nsamples_train','dataset_nsamples','dataset_nfeatures','dataset_npoints']:\n", - " plt.figure(figsize=(8,8))\n", + "df_sum[\"Method Family\"] = df_sum[\"algorithm\"].apply(\n", + " lambda x: (\n", + " \"Operon\"\n", + " if x == \"Operon\"\n", + " else \"XGB\"\n", + " if x == \"XGB\"\n", + " else \"Other Non-GP SR\"\n", + " if x in nongp_algs\n", + " else \"Other GP SR\"\n", + " if x in gp_algs\n", + " else \"Other ML\"\n", + " )\n", + ")\n", + "sns.set_style(\"whitegrid\")\n", + "for col in [\n", + " \"dataset_nsamples_train\",\n", + " \"dataset_nsamples\",\n", + " \"dataset_nfeatures\",\n", + " \"dataset_npoints\",\n", + "]:\n", + " plt.figure(figsize=(8, 8))\n", " ax = sns.lineplot(\n", - " data=df_sum.loc[df_sum.algorithm.isin(['Operon',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'SBP-GP',\n", - " 'XGB',\n", - " 'DSR',\n", - " ])],\n", - " x = col,\n", - " y = 'training time (hr)',\n", - "# hue='Method Family',\n", - "# style='Method Family',\n", - " hue='algorithm',\n", - " style='algorithm',\n", + " data=df_sum.loc[\n", + " df_sum.algorithm.isin(\n", + " [\n", + " \"Operon\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"SBP-GP\",\n", + " \"XGB\",\n", + " \"DSR\",\n", + " ]\n", + " )\n", + " ],\n", + " x=col,\n", + " y=\"training time (hr)\",\n", + " # hue='Method Family',\n", + " # style='Method Family',\n", + " hue=\"algorithm\",\n", + " style=\"algorithm\",\n", " markers=True,\n", - " palette='cividis',\n", + " palette=\"cividis\",\n", " markersize=10,\n", " markerfacecolor=None,\n", - " err_style='bars',\n", - " alpha=0.85\n", - "# facecolors=None\n", - "# scatter_kws = {'facecolors':'none'}\n", + " err_style=\"bars\",\n", + " alpha=0.85,\n", + " # facecolors=None\n", + " # scatter_kws = {'facecolors':'none'}\n", " )\n", - " ax.set_yscale('log')\n", - " ax.set_xscale('log')\n", - "# plt.legend(loc=[1.01,0])\n", - " plt.xlabel(col.replace('_',' ').title())\n", - " plt.ylabel('Training Time (hr)')\n", - " save('training_time_'+col)" + " ax.set_yscale(\"log\")\n", + " ax.set_xscale(\"log\")\n", + " # plt.legend(loc=[1.01,0])\n", + " plt.xlabel(col.replace(\"_\", \" \").title())\n", + " plt.ylabel(\"Training Time (hr)\")\n", + " save(\"training_time_\" + col)" ] }, { @@ -439,16 +507,18 @@ "outputs": [], "source": [ "np.random.seed(42)\n", - "def bootstrap(val, n = 1000, fn=np.mean):\n", + "\n", + "\n", + "def bootstrap(val, n=1000, fn=np.mean):\n", " val_samples = []\n", " for i in range(n):\n", - " sample = np.random.randint(0,len(val)-1, size=len(val))\n", - " val_samples.append( fn(val[sample]) )\n", + " sample = np.random.randint(0, len(val) - 1, size=len(val))\n", + " val_samples.append(fn(val[sample]))\n", " m = np.mean(val_samples)\n", " sd = np.std(val_samples)\n", - " ci_upper = np.quantile(val_samples,0.95)\n", - " ci_lower = np.quantile(val_samples,0.05)\n", - " return m, sd, ci_upper,ci_lower" + " ci_upper = np.quantile(val_samples, 0.95)\n", + " ci_lower = np.quantile(val_samples, 0.05)\n", + " return m, sd, ci_upper, ci_lower" ] }, { @@ -457,132 +527,137 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", "from pareto_utils import front\n", - "sns.set(font_scale=1, style='whitegrid')\n", - "labelsize=18\n", - "plt.figure(figsize=(7,7))\n", - "data = df_sum.copy() #.loc[df_sum.algorithm.isin(symbolic_algs)]\n", - "data['*algorithm*'] = data['algorithm'].apply(lambda x: x+'*' if x in symbolic_algs else x)\n", - "xcol = 'r2_test_rank'\n", + "\n", + "sns.set(font_scale=1, style=\"whitegrid\")\n", + "labelsize = 18\n", + "plt.figure(figsize=(7, 7))\n", + "data = df_sum.copy() # .loc[df_sum.algorithm.isin(symbolic_algs)]\n", + "data[\"*algorithm*\"] = data[\"algorithm\"].apply(\n", + " lambda x: x + \"*\" if x in symbolic_algs else x\n", + ")\n", + "xcol = \"r2_test_rank\"\n", "# xcol = 'r2_test'\n", - "ycol = 'model_size_rank'\n", + "ycol = \"model_size_rank\"\n", "# ycol = 'model_size'\n", - "palette = 'viridis'\n", + "palette = \"viridis\"\n", "# outline pareto front\n", - "pareto_data = data.groupby('*algorithm*').median()\n", + "pareto_data = data.groupby(\"*algorithm*\").median()\n", "\n", - "objs = pareto_data[[xcol,ycol]].values\n", + "objs = pareto_data[[xcol, ycol]].values\n", "# reverse R2 (objs are minimized)\n", "# objs[:,0] = -objs[:,0]\n", "levels = 6\n", - "styles=['-','-.','--',':', ':', ':']\n", + "styles = [\"-\", \"-.\", \"--\", \":\", \":\", \":\"]\n", "PFs = []\n", "pareto_ranks = -np.ones(len(pareto_data))\n", "for el in range(levels):\n", - "# pdb.set_trace()\n", - " PF = front(objs[:,0],objs[:,1])\n", - " if len(PF)>0:\n", - " print('PF:',PF)\n", + " # pdb.set_trace()\n", + " PF = front(objs[:, 0], objs[:, 1])\n", + " if len(PF) > 0:\n", + " print(\"PF:\", PF)\n", " pareto_ranks[PF] = el\n", - " objs[PF,:] = np.inf\n", + " objs[PF, :] = np.inf\n", " PFs.append(PF)\n", "i = 0\n", - "pareto_data.loc[:,'pareto_rank'] = pareto_ranks\n", + "pareto_data.loc[:, \"pareto_rank\"] = pareto_ranks\n", "for pfset in PFs:\n", - " xset,yset = [],[]\n", + " xset, yset = [], []\n", "\n", " for pf in pfset:\n", " xset.append(pareto_data[xcol].values[pf])\n", - " yset.append( pareto_data[ycol].values[pf] )\n", - " linestyle=styles[i]\n", - " plt.plot(xset,yset, styles[i]+'k', alpha=0.5, zorder=1)\n", - "# plt.gca().set_zorder(10)\n", - " i+=1\n", + " yset.append(pareto_data[ycol].values[pf])\n", + " linestyle = styles[i]\n", + " plt.plot(xset, yset, styles[i] + \"k\", alpha=0.5, zorder=1)\n", + " # plt.gca().set_zorder(10)\n", + " i += 1\n", "\n", - "cmap = sns.color_palette(palette=palette, \n", - " n_colors=pareto_data.pareto_rank.nunique(), \n", - " desat=None, \n", - " as_cmap=False)\n", + "cmap = sns.color_palette(\n", + " palette=palette,\n", + " n_colors=pareto_data.pareto_rank.nunique(),\n", + " desat=None,\n", + " as_cmap=False,\n", + ")\n", "\n", "ax = sns.scatterplot(\n", - " ax = plt.gca(),\n", - "# ax = g.ax_joint,\n", - "# data = data.groupby('*algorithm*').median(),\n", - " data = pareto_data,\n", - " x = xcol,\n", - " y = ycol,\n", - "# style='*algorithm*',\n", - "# style='pareto_rank',\n", - " hue = 'pareto_rank',\n", + " ax=plt.gca(),\n", + " # ax = g.ax_joint,\n", + " # data = data.groupby('*algorithm*').median(),\n", + " data=pareto_data,\n", + " x=xcol,\n", + " y=ycol,\n", + " # style='*algorithm*',\n", + " # style='pareto_rank',\n", + " hue=\"pareto_rank\",\n", " s=250,\n", - "# palette=palette,\n", + " # palette=palette,\n", " # edgecolor='k'\n", " legend=False,\n", - " palette=cmap\n", + " palette=cmap,\n", ")\n", "ax.set_zorder(2)\n", - "xoff = .5\n", + "xoff = 0.5\n", "yoff = 0.3\n", "# xoff, yoff = 0, 0\n", - "for idx,row in pareto_data.iterrows():\n", - " x=row[xcol]-xoff\n", - " y=row[ycol]-yoff\n", - " ha='right'\n", - " \n", - " if idx in ['Linear','AFP_FE*','MLP','MRGP*']:\n", - " x=row[xcol]+xoff\n", - " ha='left'\n", - " elif idx == 'Operon*':\n", - " x = row[xcol]+1\n", + "for idx, row in pareto_data.iterrows():\n", + " x = row[xcol] - xoff\n", + " y = row[ycol] - yoff\n", + " ha = \"right\"\n", + "\n", + " if idx in [\"Linear\", \"AFP_FE*\", \"MLP\", \"MRGP*\"]:\n", + " x = row[xcol] + xoff\n", + " ha = \"left\"\n", + " elif idx == \"Operon*\":\n", + " x = row[xcol] + 1\n", " y -= yoff\n", - " elif idx in ['gplearn*','FEAT*']:\n", - "# x -= xoff\n", - "# x=row[xcol]+xoff\n", - " y = row[ycol]+1\n", - "# ha='left'\n", - " \n", - " plt.text(s=idx,\n", - " x = x, \n", - " y = y,\n", - " ha=ha,\n", - " va='top',\n", - " bbox=dict(facecolor='w', edgecolor='b', boxstyle='round', alpha=1)\n", - " )\n", - " \n", + " elif idx in [\"gplearn*\", \"FEAT*\"]:\n", + " # x -= xoff\n", + " # x=row[xcol]+xoff\n", + " y = row[ycol] + 1\n", + " # ha='left'\n", + "\n", + " plt.text(\n", + " s=idx,\n", + " x=x,\n", + " y=y,\n", + " ha=ha,\n", + " va=\"top\",\n", + " bbox=dict(facecolor=\"w\", edgecolor=\"b\", boxstyle=\"round\", alpha=1),\n", + " )\n", + "\n", "# confidence intervals\n", "i = 0\n", - "for alg, dg in data.groupby('*algorithm*'):\n", - " x = dg[xcol].median() \n", + "for alg, dg in data.groupby(\"*algorithm*\"):\n", + " x = dg[xcol].median()\n", " y = dg[ycol].median()\n", " _, sdx, ciux, cilx = bootstrap(dg[xcol].values, fn=np.median, n=1000)\n", " _, sdy, ciuy, cily = bootstrap(dg[ycol].values, fn=np.median, n=1000)\n", " plt.plot(\n", - " [cilx,ciux],\n", - " [y, y],\n", - " alpha=0.5,\n", - " color=cmap[int(pareto_data.loc[alg,'pareto_rank'])]\n", - "# color='b'\n", - " )\n", + " [cilx, ciux],\n", + " [y, y],\n", + " alpha=0.5,\n", + " color=cmap[int(pareto_data.loc[alg, \"pareto_rank\"])],\n", + " # color='b'\n", + " )\n", " plt.plot(\n", - " [x, x],\n", - " [cily,ciuy],\n", - " alpha=0.5,\n", - " color=cmap[int(pareto_data.loc[alg,'pareto_rank'])]\n", - "# color='b'\n", - " )\n", + " [x, x],\n", + " [cily, ciuy],\n", + " alpha=0.5,\n", + " color=cmap[int(pareto_data.loc[alg, \"pareto_rank\"])],\n", + " # color='b'\n", + " )\n", " i += 1\n", "ax.set_aspect(1.0)\n", "ax.set_xlim(left=0)\n", "ax.set_ylim(bottom=0)\n", - "ticksize=16\n", - "plt.xticks([0,5,10,15,20],fontsize=ticksize)\n", - "plt.yticks([0,5,10,15,20],fontsize=ticksize)\n", + "ticksize = 16\n", + "plt.xticks([0, 5, 10, 15, 20], fontsize=ticksize)\n", + "plt.yticks([0, 5, 10, 15, 20], fontsize=ticksize)\n", "# ax.set_yscale('log')\n", - "plt.xlabel(xcol.replace('_',' ').replace('r2','$R^2$').title(), fontsize=labelsize)\n", - "plt.ylabel(ycol.replace('_',' ').title(), fontsize=labelsize)\n", + "plt.xlabel(xcol.replace(\"_\", \" \").replace(\"r2\", \"$R^2$\").title(), fontsize=labelsize)\n", + "plt.ylabel(ycol.replace(\"_\", \" \").title(), fontsize=labelsize)\n", "sns.despine(left=True, bottom=True)\n", - "save(name=\"pareto_plot_\"+xcol+'_'+ycol)" + "save(name=\"pareto_plot_\" + xcol + \"_\" + ycol)" ] }, { @@ -600,7 +675,7 @@ "metadata": {}, "outputs": [], "source": [ - "pareto_data['pareto_rank']" + "pareto_data[\"pareto_rank\"]" ] } ], diff --git a/postprocessing/collate_blackbox_results.py b/postprocessing/collate_blackbox_results.py index 17a78be6..045d3756 100644 --- a/postprocessing/collate_blackbox_results.py +++ b/postprocessing/collate_blackbox_results.py @@ -7,140 +7,144 @@ ################################################################################ # Black-box problems ################################################################################ -import pandas as pd import json -import numpy as np +import sys from glob import glob + +import pandas as pd from tqdm import tqdm -import os -import sys -rdir = '../results_pmlb_r1/' +rdir = "../results_pmlb_r1/" if len(sys.argv) > 1: rdir = sys.argv[1] else: - print('no rdir provided, using',rdir) -print('reading results from directory', rdir) - + print("no rdir provided, using", rdir) +print("reading results from directory", rdir) + symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', - 'AIFeynman' -] -nongp_algs = [ - 'BSR', - 'DSR', - 'AIFeynman' + "AFP", + "AFP_FE", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", + "AIFeynman", ] +nongp_algs = ["BSR", "DSR", "AIFeynman"] gp_algs = [ - 'AFP', - 'AFP_FE', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', + "AFP", + "AFP_FE", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", ] ########## # load data from json ########## frames = [] comparison_cols = [ - 'dataset', - 'algorithm', - 'random_state', - 'time_time', - 'model_size', - 'symbolic_model', - 'r2_test', - 'mse_test', - 'mae_test', - 'params' + "dataset", + "algorithm", + "random_state", + "time_time", + "model_size", + "symbolic_model", + "r2_test", + "mse_test", + "mae_test", + "params", ] fails = [] -import pdb -for f in tqdm(glob(rdir + '/*/*.json')): - if 'cv_results' in f: +for f in tqdm(glob(rdir + "/*/*.json")): + if "cv_results" in f: continue # leave out symbolic data - if 'feynman_' in f or 'strogatz_' in f: + if "feynman_" in f or "strogatz_" in f: continue # leave out LinearReg, Lasso (we have SGD with penalty) - if any([m in f for m in ['LinearRegression','Lasso','EHCRegressor']]): + if any([m in f for m in ["LinearRegression", "Lasso", "EHCRegressor"]]): continue - try: - r = json.load(open(f,'r')) - if isinstance(r['symbolic_model'],list): -# print(f) - sm = ['B'+str(i)+'*'+ri for i, ri in enumerate(r['symbolic_model'])] - sm = '+'.join(sm) - r['symbolic_model'] = sm - - sub_r = {k:v for k,v in r.items() if k in comparison_cols} - # df = pd.DataFrame(sub_r) - frames.append(sub_r) + try: + r = json.load(open(f, "r")) + if isinstance(r["symbolic_model"], list): + # print(f) + sm = ["B" + str(i) + "*" + ri for i, ri in enumerate(r["symbolic_model"])] + r["symbolic_model"] = "+".join(sm) + + sub_r = {k: v for k, v in r.items() if k in comparison_cols} + # df = pd.DataFrame(sub_r) + frames.append(sub_r) # print(f) # print(r.keys()) except Exception as e: - fails.append([f,e]) + fails.append([f, e]) pass - -print(len(fails),'fails:',fails) + +print(len(fails), "fails:", fails) # df_results = pd.concat(frames) df_results = pd.DataFrame.from_records(frames) -df_results['params_str'] = df_results['params'].apply(str) -df_results = df_results.drop(columns=['params']) +df_results["params_str"] = df_results["params"].apply(str) +df_results = df_results.drop(columns=["params"]) ########## # cleanup ########## -df_results = df_results.rename(columns={'time_time':'training time (s)'}) -df_results.loc[:,'training time (hr)'] = df_results['training time (s)']/3600 +df_results = df_results.rename(columns={"time_time": "training time (s)"}) +df_results.loc[:, "training time (hr)"] = df_results["training time (s)"] / 3600 # remove regressor from names -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('Regressor','')) -#Rename SGD to Linear -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: 'Linear' if x=='SGD' else x) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("Regressor", "") +) +# Rename SGD to Linear +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: "Linear" if x == "SGD" else x +) # rename sembackpropgp to SBP -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('sembackpropgp','SBP-GP')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("sembackpropgp", "SBP-GP") +) # rename FE_AFP to AFP_FE -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('FE_AFP','AFP_FE')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("FE_AFP", "AFP_FE") +) # rename GPGOMEA to GP-GOMEA -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('GPGOMEA','GP-GOMEA')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("GPGOMEA", "GP-GOMEA") +) # add modified R2 with 0 floor -df_results['r2_zero_test'] = df_results['r2_test'].apply(lambda x: max(x,0)) +df_results["r2_zero_test"] = df_results["r2_test"].apply(lambda x: max(x, 0)) # label friedman ddatasets -df_results.loc[:,'friedman_dataset'] = df_results['dataset'].str.contains('_fri_') -print('loaded',len(df_results),'results') +df_results.loc[:, "friedman_dataset"] = df_results["dataset"].str.contains("_fri_") +print("loaded", len(df_results), "results") # additional metadata -df_results['symbolic_alg'] = df_results['algorithm'].apply(lambda x: x in symbolic_algs) +df_results["symbolic_alg"] = df_results["algorithm"].apply(lambda x: x in symbolic_algs) -for col in ['algorithm','dataset']: - print(df_results[col].nunique(), col+'s') +for col in ["algorithm", "dataset"]: + print(df_results[col].nunique(), col + "s") ########## # save results ########## -df_results.to_feather('../results/black-box_results.feather') -print('results saved to ../results/black-box_results.feather') +df_results.to_feather("../results/black-box_results.feather") +print("results saved to ../results/black-box_results.feather") ######## -print('mean trial count:') -print(df_results.groupby('algorithm')['dataset'].count().sort_values() - / df_results.dataset.nunique()) - +print("mean trial count:") +print( + df_results.groupby("algorithm")["dataset"].count().sort_values() + / df_results.dataset.nunique() +) diff --git a/postprocessing/collate_groundtruth_results.py b/postprocessing/collate_groundtruth_results.py index 92e0885e..10a20193 100644 --- a/postprocessing/collate_groundtruth_results.py +++ b/postprocessing/collate_groundtruth_results.py @@ -7,107 +7,127 @@ ################################################################################ # Ground-truth problems ################################################################################ -import pandas as pd import json -import numpy as np -from glob import glob -from tqdm import tqdm import os import sys +from glob import glob + +import pandas as pd +from tqdm import tqdm -rdir = '../results_sym_data/' +rdir = "../results_sym_data/" if len(sys.argv) > 1: rdir = sys.argv[1] else: - print('no rdir provided, using',rdir) -print('reading results from directory', rdir) + print("no rdir provided, using", rdir) +print("reading results from directory", rdir) ########## # load data from json ########## frames = [] -excluded_datasets = [ - 'feynman_test_10', - 'feynman_I_26_2', - 'feynman_I_30_5' -] -excluded_cols = [ - 'params' -] +excluded_datasets = ["feynman_test_10", "feynman_I_26_2", "feynman_I_30_5"] +excluded_cols = ["params"] fails = [] bad_bsr = [] updated = 0 -for f in tqdm(glob(rdir + '/*/*.json')): - if os.path.exists(f+'.updated'): - f += '.updated' +for f in tqdm(glob(rdir + "/*/*.json")): + if os.path.exists(f + ".updated"): + f += ".updated" updated += 1 - if 'cv_results' in f: + if "cv_results" in f: continue - if 'EHC' in f: + if "EHC" in f: continue if any([ed in f for ed in excluded_datasets]): continue - try: - r = json.load(open(f,'r')) - if isinstance(r['symbolic_model'],list): - print('WARNING: list returned for model:',f) + try: + r = json.load(open(f, "r")) + if isinstance(r["symbolic_model"], list): + print("WARNING: list returned for model:", f) bad_bsr.append(f) - sm = ['B'+str(i)+'*'+ri for i, ri in enumerate(r['symbolic_model'])] - sm = '+'.join(sm) - r['symbolic_model'] = sm - - sub_r = {k:v for k,v in r.items() if k not in excluded_cols} - # df = pd.DataFrame(sub_r) - frames.append(sub_r) + sm = ["B" + str(i) + "*" + ri for i, ri in enumerate(r["symbolic_model"])] + r["symbolic_model"] = "+".join(sm) + + sub_r = {k: v for k, v in r.items() if k not in excluded_cols} + # df = pd.DataFrame(sub_r) + frames.append(sub_r) # print(f) # print(r.keys()) except Exception as e: - fails.append([f,e]) + fails.append([f, e]) pass - -print('{} results files loaded, {} ({:.1f}%) of which are ' - 'updated'.format(len(frames), updated, updated/len(frames)*100)) -print(len(fails),'fails:') -for f in fails: + +print( + "{} results files loaded, {} ({:.1f}%) of which are updated".format( + len(frames), updated, updated / len(frames) * 100 + ) +) +print(len(fails), "fails:") +for f in fails: print(f[0]) -print('bad bsr:',bad_bsr) +print("bad bsr:", bad_bsr) df_results = pd.DataFrame.from_records(frames) ########## # cleanup ########## -df_results = df_results.rename(columns={'time_time':'training time (s)'}) -df_results.loc[:,'training time (hr)'] = df_results['training time (s)']/3600 +df_results = df_results.rename(columns={"time_time": "training time (s)"}) +df_results.loc[:, "training time (hr)"] = df_results["training time (s)"] / 3600 # add modified R2 with 0 floor -df_results['r2_zero_test'] = df_results['r2_test'].apply(lambda x: max(x,0)) -for col in ['symbolic_error_is_zero', 'symbolic_error_is_constant', 'symbolic_fraction_is_constant']: - df_results.loc[:,col] = df_results[col].fillna(False) -print(','.join(df_results.algorithm.unique())) +df_results["r2_zero_test"] = df_results["r2_test"].apply(lambda x: max(x, 0)) +for col in [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", +]: + df_results.loc[:, col] = df_results[col].fillna(False) +print(",".join(df_results.algorithm.unique())) # remove 'Regressor' from names -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('Regressor','')) -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('tuned.','')) -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('sembackpropgp','SBP-GP')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("Regressor", "") +) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("tuned.", "") +) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("sembackpropgp", "SBP-GP") +) # rename FE_AFP to AFP_FE -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('FE_AFP','AFP_FE')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("FE_AFP", "AFP_FE") +) # rename GPGOMEA to GP-GOMEA -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('GPGOMEA','GP-GOMEA')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("GPGOMEA", "GP-GOMEA") +) # indicator of strogatz or feynman -df_results['data_group'] = df_results['dataset'].apply(lambda x: 'Feynman' if 'feynman' in x else 'Strogatz') +df_results["data_group"] = df_results["dataset"].apply( + lambda x: "Feynman" if "feynman" in x else "Strogatz" +) ########## # compute symbolic solutions ########## -df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero', - 'symbolic_error_is_constant', - 'symbolic_fraction_is_constant'] - ].apply(any,raw=True, axis=1) -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0') -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan') +df_results.loc[:, "symbolic_solution"] = df_results[ + [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", + ] +].apply(any, raw=True, axis=1) +df_results.loc[:, "symbolic_solution"] = ( + df_results["symbolic_solution"] & ~df_results["simplified_symbolic_model"].isna() +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "0" +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "nan" +) ########## # save results ########## -df_results.to_feather('../results/ground-truth_results.feather') -print('results saved to ../results/ground-truth_results.feather') - +df_results.to_feather("../results/ground-truth_results.feather") +print("results saved to ../results/ground-truth_results.feather") diff --git a/postprocessing/genCSVs.py b/postprocessing/genCSVs.py index c451aff7..daada54a 100644 --- a/postprocessing/genCSVs.py +++ b/postprocessing/genCSVs.py @@ -1,103 +1,142 @@ +from glob import glob + import numpy as np import pandas as pd -from glob import glob # CSV with datasets information -datadir = '../../../pmlb/datasets/' +datadir = "../../../pmlb/datasets/" frames = [] -for f in glob(datadir+'/*/*.tsv.gz'): - df = pd.read_csv(f,sep='\t') - group = 'feynman' if 'feynman' in f else 'strogatz' if 'strogatz' in f else 'black-box' - frames.append(dict( - name=f.split('/')[-1][:-7], - nsamples = df.shape[0], - nfeatures = df.shape[1], - npoints = df.shape[0]*df.shape[1], - Group=group - )) - +for f in glob(datadir + "/*/*.tsv.gz"): + df = pd.read_csv(f, sep="\t") + group = ( + "feynman" if "feynman" in f else "strogatz" if "strogatz" in f else "black-box" + ) + frames.append( + dict( + name=f.split("/")[-1][:-7], + nsamples=df.shape[0], + nfeatures=df.shape[1], + npoints=df.shape[0] * df.shape[1], + Group=group, + ) + ) + df = pd.DataFrame.from_records(frames) df.to_csv("../docs/csv/datasets_info.csv") # CSV for the blackbox results symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', - 'AIFeynman' + "AFP", + "AFP_FE", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", + "AIFeynman", ] -x_vars=[ 'rmse_test', - 'log_mse_test', - 'r2_test', - 'model_size', - 'training time (s)'] - -df_results = pd.read_feather('../results/black-box_results.feather') -df_blackbox = df_results.merge(df_results.groupby('dataset')['algorithm'].nunique().reset_index(), - on='dataset',suffixes=('','_count')) - -df_sum = df_blackbox.groupby(['algorithm','dataset'],as_index=False).median() -df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt) -df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x)) - -df_sum['algorithm'] = df_sum['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x) -(df_sum - .groupby(["algorithm","dataset"]) - .median()[x_vars] - .to_csv("../docs/csv/blackbox_results.csv") +x_vars = ["rmse_test", "log_mse_test", "r2_test", "model_size", "training time (s)"] + +df_results = pd.read_feather("../results/black-box_results.feather") +df_blackbox = df_results.merge( + df_results.groupby("dataset")["algorithm"].nunique().reset_index(), + on="dataset", + suffixes=("", "_count"), +) + +df_sum = df_blackbox.groupby(["algorithm", "dataset"], as_index=False).median() +df_sum["rmse_test"] = df_sum["mse_test"].apply(np.sqrt) +df_sum["log_mse_test"] = df_sum["mse_test"].apply(lambda x: np.log(1 + x)) + +df_sum["algorithm"] = df_sum["algorithm"].apply( + lambda x: "*" + x if x in symbolic_algs else x +) +( + df_sum.groupby(["algorithm", "dataset"]) + .median()[x_vars] + .to_csv("../docs/csv/blackbox_results.csv") ) # Aggregate -df_sum.merge(df, left_on="dataset", right_on="name").to_csv("../docs/csv/blackbox_results_datasets.csv") +df_sum.merge(df, left_on="dataset", right_on="name").to_csv( + "../docs/csv/blackbox_results_datasets.csv" +) # Ground Truth results -df_results = pd.read_feather('../results/ground-truth_results.feather') -df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero', - 'symbolic_error_is_constant', - 'symbolic_fraction_is_constant'] - ].apply(any,raw=True, axis=1) +df_results = pd.read_feather("../results/ground-truth_results.feather") +df_results.loc[:, "symbolic_solution"] = df_results[ + [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", + ] +].apply(any, raw=True, axis=1) # clean up any corner cases (constant models, failures) -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0') -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan') -df_results2 = df_results.merge(df_results.groupby(['dataset','target_noise'])['algorithm'].nunique().reset_index(), - on=['dataset','target_noise'],suffixes=('','_count')) +df_results.loc[:, "symbolic_solution"] = ( + df_results["symbolic_solution"] & ~df_results["simplified_symbolic_model"].isna() +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "0" +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "nan" +) +df_results2 = df_results.merge( + df_results.groupby(["dataset", "target_noise"])["algorithm"] + .nunique() + .reset_index(), + on=["dataset", "target_noise"], + suffixes=("", "_count"), +) # count repeat trials df_results2 = df_results2.merge( - df_results2.groupby(['algorithm','dataset','target_noise'])['random_state'].nunique().reset_index(), - on=['algorithm','dataset','target_noise'],suffixes=('','_repeats')) + df_results2.groupby(["algorithm", "dataset", "target_noise"])["random_state"] + .nunique() + .reset_index(), + on=["algorithm", "dataset", "target_noise"], + suffixes=("", "_repeats"), +) -# accuracy-based exact solutions -df_results2['accuracy_solution'] = df_results2['r2_test'].apply(lambda x: x > 0.999).astype(float) +# accuracy-based exact solutions +df_results2["accuracy_solution"] = ( + df_results2["r2_test"].apply(lambda x: x > 0.999).astype(float) +) -# get mean solution rates for algs on datasets at specific noise levels, averaged over trials -for soln in ['accuracy_solution','symbolic_solution']: +# get mean solution rates for algs on datasets at specific noise levels, averaged over trials +for soln in ["accuracy_solution", "symbolic_solution"]: df_results2 = df_results2.merge( - df_results2.groupby(['algorithm','dataset','target_noise'])[soln].mean().reset_index(), - on=['algorithm','dataset', 'target_noise'],suffixes=('','_rate')) - -df_sum = df_results2.groupby(['algorithm','dataset','target_noise','data_group'],as_index=False).median() - -for soln in ['accuracy_solution','symbolic_solution']: - df_sum[soln +'_rate_(%)'] = df_sum[soln+'_rate'].apply(lambda x: x*100) -df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt) -df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x)) - -columns = ['algorithm','dataset','target_noise','data_group' - ,'symbolic_solution_rate_(%)','accuracy_solution' - ,'rmse_test', 'log_mse_test' - ] -df_sum[columns].to_csv('../docs/csv/groundtruth.csv', index=False) + df_results2.groupby(["algorithm", "dataset", "target_noise"])[soln] + .mean() + .reset_index(), + on=["algorithm", "dataset", "target_noise"], + suffixes=("", "_rate"), + ) + +df_sum = df_results2.groupby( + ["algorithm", "dataset", "target_noise", "data_group"], as_index=False +).median() +for soln in ["accuracy_solution", "symbolic_solution"]: + df_sum[soln + "_rate_(%)"] = df_sum[soln + "_rate"].apply(lambda x: x * 100) +df_sum["rmse_test"] = df_sum["mse_test"].apply(np.sqrt) +df_sum["log_mse_test"] = df_sum["mse_test"].apply(lambda x: np.log(1 + x)) + +columns = [ + "algorithm", + "dataset", + "target_noise", + "data_group", + "symbolic_solution_rate_(%)", + "accuracy_solution", + "rmse_test", + "log_mse_test", +] +df_sum[columns].to_csv("../docs/csv/groundtruth.csv", index=False) diff --git a/postprocessing/groundtruth_results.ipynb b/postprocessing/groundtruth_results.ipynb index 6b60bb4b..47e63fb2 100644 --- a/postprocessing/groundtruth_results.ipynb +++ b/postprocessing/groundtruth_results.ipynb @@ -19,33 +19,33 @@ "metadata": {}, "outputs": [], "source": [ - "from tabulate import tabulate\n", - "import pandas as pd\n", - "import json\n", + "import os\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from glob import glob\n", - "from tqdm import tqdm\n", + "import pandas as pd\n", "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib\n", - "matplotlib.rc('pdf', fonttype=42)\n", - "import os\n", + "\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", + "\n", "sns.set(font_scale=1.2)\n", - "rdir = '../results/'\n", + "rdir = \"../results/\"\n", "# figdir = 'figs/ground-truth/'\n", - "figdir = '../../paper/figs/results_sym_data/'\n", + "figdir = \"../../paper/figs/results_sym_data/\"\n", + "\n", + "print(\"figdir:\", figdir)\n", "\n", - "print('figdir:',figdir)\n", - "def save(h=None,name='tmp'):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "\n", + "def save(h=None, name=\"tmp\"):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',name+'.pdf')\n", + " print(\"saving\", name + \".pdf\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.pdf', dpi=400, bbox_inches='tight')\n", - " " + " plt.savefig(figdir + \"/\" + name + \".pdf\", dpi=400, bbox_inches=\"tight\")" ] }, { @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_results = pd.read_feather(rdir+'/ground-truth_results.feather')" + "df_results = pd.read_feather(rdir + \"/ground-truth_results.feather\")" ] }, { @@ -77,17 +77,26 @@ "metadata": {}, "outputs": [], "source": [ - "df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero',\n", - " 'symbolic_error_is_constant',\n", - " 'symbolic_fraction_is_constant']\n", - " ].apply(any,raw=True, axis=1)\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\n", + " [\n", + " \"symbolic_error_is_zero\",\n", + " \"symbolic_error_is_constant\",\n", + " \"symbolic_fraction_is_constant\",\n", + " ]\n", + "].apply(any, raw=True, axis=1)\n", "# clean up any corner cases (constant models, failures)\n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() \n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0')\n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan')\n", + "df_results.loc[:, \"symbolic_solution\"] = (\n", + " df_results[\"symbolic_solution\"] & ~df_results[\"simplified_symbolic_model\"].isna()\n", + ")\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\"symbolic_solution\"] & ~(\n", + " df_results[\"simplified_symbolic_model\"] == \"0\"\n", + ")\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\"symbolic_solution\"] & ~(\n", + " df_results[\"simplified_symbolic_model\"] == \"nan\"\n", + ")\n", "\n", "# save results for detailed tabulating\n", - "df_results.to_feather(rdir+'ground-truth_solns.feather')" + "df_results.to_feather(rdir + \"ground-truth_solns.feather\")" ] }, { @@ -103,43 +112,63 @@ "metadata": {}, "outputs": [], "source": [ - "df_results2 = df_results.merge(df_results.groupby(['dataset','target_noise'])['algorithm'].nunique().reset_index(),\n", - " on=['dataset','target_noise'],suffixes=('','_count'))\n", + "df_results2 = df_results.merge(\n", + " df_results.groupby([\"dataset\", \"target_noise\"])[\"algorithm\"]\n", + " .nunique()\n", + " .reset_index(),\n", + " on=[\"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_count\"),\n", + ")\n", "# count repeat trials\n", "df_results2 = df_results2.merge(\n", - " df_results2.groupby(['algorithm','dataset','target_noise'])['random_state'].nunique().reset_index(),\n", - " on=['algorithm','dataset','target_noise'],suffixes=('','_repeats'))\n", + " df_results2.groupby([\"algorithm\", \"dataset\", \"target_noise\"])[\"random_state\"]\n", + " .nunique()\n", + " .reset_index(),\n", + " on=[\"algorithm\", \"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_repeats\"),\n", + ")\n", "\n", - "# accuracy-based exact solutions \n", - "df_results2['accuracy_solution'] = df_results2['r2_test'].apply(lambda x: x > 0.999).astype(float)\n", + "# accuracy-based exact solutions\n", + "df_results2[\"accuracy_solution\"] = (\n", + " df_results2[\"r2_test\"].apply(lambda x: x > 0.999).astype(float)\n", + ")\n", "\n", - "# get mean solution rates for algs on datasets at specific noise levels, averaged over trials \n", - "for soln in ['accuracy_solution','symbolic_solution']:\n", + "# get mean solution rates for algs on datasets at specific noise levels, averaged over trials\n", + "for soln in [\"accuracy_solution\", \"symbolic_solution\"]:\n", " df_results2 = df_results2.merge(\n", - " df_results2.groupby(['algorithm','dataset','target_noise'])[soln].mean().reset_index(),\n", - " on=['algorithm','dataset', 'target_noise'],suffixes=('','_rate'))\n", - " \n", + " df_results2.groupby([\"algorithm\", \"dataset\", \"target_noise\"])[soln]\n", + " .mean()\n", + " .reset_index(),\n", + " on=[\"algorithm\", \"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_rate\"),\n", + " )\n", + "\n", "# # rankings\n", - "for col in [c for c in df_results2.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_results2[col+'_rank_per_trial']=df_results2.groupby(\n", - " ['dataset','target_noise','random_state'])[col].apply(\n", - " lambda x: \n", - " round(x,3).rank(\n", - " ascending=ascending).astype(int))\n", - " \n", - "df_sum = df_results2.groupby(['algorithm','dataset','target_noise','data_group'],as_index=False).median()\n", + "for col in [c for c in df_results2.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_results2[col + \"_rank_per_trial\"] = df_results2.groupby(\n", + " [\"dataset\", \"target_noise\", \"random_state\"]\n", + " )[col].apply(lambda x: round(x, 3).rank(ascending=ascending).astype(int))\n", + "\n", + "df_sum = df_results2.groupby(\n", + " [\"algorithm\", \"dataset\", \"target_noise\", \"data_group\"], as_index=False\n", + ").median()\n", "# rankings and normalized scores per dataset\n", - "for col in [c for c in df_sum.columns if any([c.endswith(n) for n in ['test','size','rate']])]:\n", - " ascending = 'r2' not in col and 'solution' not in col\n", - " df_sum[col+'_rank']=df_sum.groupby(['dataset','target_noise'])[col].apply(\n", - " lambda x: round(x,3).rank(ascending=ascending).astype(int) )\n", - " df_sum[col+'_norm'] = df_sum.groupby(['dataset','target_noise'])[col].apply(lambda x: (x-x.min())/(x.max()-x.min()))\n", + "for col in [\n", + " c for c in df_sum.columns if any([c.endswith(n) for n in [\"test\", \"size\", \"rate\"]])\n", + "]:\n", + " ascending = \"r2\" not in col and \"solution\" not in col\n", + " df_sum[col + \"_rank\"] = df_sum.groupby([\"dataset\", \"target_noise\"])[col].apply(\n", + " lambda x: round(x, 3).rank(ascending=ascending).astype(int)\n", + " )\n", + " df_sum[col + \"_norm\"] = df_sum.groupby([\"dataset\", \"target_noise\"])[col].apply(\n", + " lambda x: (x - x.min()) / (x.max() - x.min())\n", + " )\n", "# df_sum['success_rate'] = df_results2.groupby(['algorithm','dataset'])['solution'].mean().reset_index()\n", - "for soln in ['accuracy_solution','symbolic_solution']:\n", - " df_sum[soln +'_rate_(%)'] = df_sum[soln+'_rate'].apply(lambda x: x*100)\n", - "df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt)\n", - "df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x))\n", + "for soln in [\"accuracy_solution\", \"symbolic_solution\"]:\n", + " df_sum[soln + \"_rate_(%)\"] = df_sum[soln + \"_rate\"].apply(lambda x: x * 100)\n", + "df_sum[\"rmse_test\"] = df_sum[\"mse_test\"].apply(np.sqrt)\n", + "df_sum[\"log_mse_test\"] = df_sum[\"mse_test\"].apply(lambda x: np.log(1 + x))\n", "df_results = df_results2\n", "df_sum" ] @@ -158,7 +187,9 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum.to_csv(rdir+'/symbolic_dataset_results_sum.csv.gz',compression='gzip', index=False)" + "df_sum.to_csv(\n", + " rdir + \"/symbolic_dataset_results_sum.csv.gz\", compression=\"gzip\", index=False\n", + ")" ] }, { @@ -174,7 +205,9 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum.groupby(['target_noise','algorithm'])['symbolic_solution_rate_(%)'].mean().round(2).unstack().transpose()" + "df_sum.groupby([\"target_noise\", \"algorithm\"])[\n", + " \"symbolic_solution_rate_(%)\"\n", + "].mean().round(2).unstack().transpose()" ] }, { @@ -190,86 +223,98 @@ "metadata": {}, "outputs": [], "source": [ - "sns.set_style('whitegrid')\n", - "def compare(df_compare=None, x='r2_test',y='algorithm', row=None, col=None, scale=None, xlim=[], est=np.mean,\n", - " orient='h', hue=None, **kwargs):\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "\n", + "def compare(\n", + " df_compare=None,\n", + " x=\"r2_test\",\n", + " y=\"algorithm\",\n", + " row=None,\n", + " col=None,\n", + " scale=None,\n", + " xlim=[],\n", + " est=np.mean,\n", + " orient=\"h\",\n", + " hue=None,\n", + " **kwargs,\n", + "):\n", " df_compare = df_compare.copy()\n", - " if row==None and col == None:\n", - " aspect=1\n", + " if row is None and col is None:\n", + " aspect = 1\n", " else:\n", - " aspect=0.55\n", - "# plt.figure(figsize=(8,7))\n", - " tmp = df_compare.groupby(['target_noise',y])[x].apply(est).unstack().mean()\n", + " aspect = 0.55\n", + " # plt.figure(figsize=(8,7))\n", + " tmp = df_compare.groupby([\"target_noise\", y])[x].apply(est).unstack().mean()\n", " order = tmp.sort_values(ascending=False).index\n", - " \n", - " for c in [x,y,row,col]:\n", + "\n", + " for c in [x, y, row, col]:\n", " if c:\n", - " df_compare = df_compare.rename(columns={c:c.replace('_',' ').title()})\n", - " \n", - " x = x.replace('_',' ').title()\n", - " y = y.replace('_',' ').title()\n", + " df_compare = df_compare.rename(columns={c: c.replace(\"_\", \" \").title()})\n", + "\n", + " x = x.replace(\"_\", \" \").title()\n", + " y = y.replace(\"_\", \" \").title()\n", " if row:\n", - " row = row.replace('_',' ').title()\n", + " row = row.replace(\"_\", \" \").title()\n", " if col:\n", - " col = col.replace('_',' ').title()\n", - " \n", - " if scale=='log' and len(xlim)>0 and xlim[0] == 0:\n", - " df_compare.loc[:,x] += 1\n", + " col = col.replace(\"_\", \" \").title()\n", + "\n", + " if scale == \"log\" and len(xlim) > 0 and xlim[0] == 0:\n", + " df_compare.loc[:, x] += 1\n", " xlim[0] = 1\n", - " xnew = '1 + '+x\n", - " df_compare=df_compare.rename(columns={x:xnew})\n", + " xnew = \"1 + \" + x\n", + " df_compare = df_compare.rename(columns={x: xnew})\n", " x = xnew\n", - " if orient=='v':\n", + " if orient == \"v\":\n", " tmp = x\n", " x = y\n", " y = tmp\n", " if col and not row:\n", - " col_wrap = min(4, df_compare[col].nunique()) \n", + " col_wrap = min(4, df_compare[col].nunique())\n", " else:\n", - " col_wrap=None\n", - " \n", + " col_wrap = None\n", + "\n", " cat_args = dict(\n", - " data=df_compare, \n", - " kind='point',\n", - " y=y,\n", - " x=x,\n", - " order=order,\n", - " row=row,\n", - " col=col,\n", - " col_wrap=col_wrap,\n", - " palette='flare_r',\n", - " margin_titles=True,\n", - " aspect=aspect,\n", - " hue=hue,\n", - " legend_out=False,\n", + " data=df_compare,\n", + " kind=\"point\",\n", + " y=y,\n", + " x=x,\n", + " order=order,\n", + " row=row,\n", + " col=col,\n", + " col_wrap=col_wrap,\n", + " palette=\"flare_r\",\n", + " margin_titles=True,\n", + " aspect=aspect,\n", + " hue=hue,\n", + " legend_out=False,\n", " )\n", " cat_args.update(kwargs)\n", - " g = sns.catplot( **cat_args )\n", + " g = sns.catplot(**cat_args)\n", " if hue:\n", - " g._legend.remove() #(title=hue.replace('_',' ').title())\n", - " g.axes.flat[-1].legend(title=hue.replace('_',' ').title(),\n", - " fontsize=10\n", - " )\n", - " for ax in g.axes.flat: \n", + " g._legend.remove() # (title=hue.replace('_',' ').title())\n", + " g.axes.flat[-1].legend(title=hue.replace(\"_\", \" \").title(), fontsize=10)\n", + " for ax in g.axes.flat:\n", " ax.yaxis.grid(True)\n", - " ax.set_ylabel('')\n", - " ax.set_xlabel(ax.get_xlabel().replace('Symbolic ',''))\n", + " ax.set_ylabel(\"\")\n", + " ax.set_xlabel(ax.get_xlabel().replace(\"Symbolic \", \"\"))\n", " if col:\n", " ttl = ax.get_title()\n", - " ax.set_title(ttl.replace(col,'').replace('=',''))\n", + " ax.set_title(ttl.replace(col, \"\").replace(\"=\", \"\"))\n", "\n", - " \n", - " if len(xlim)>0:\n", - " plt.xlim(xlim[0],xlim[1])\n", + " if len(xlim) > 0:\n", + " plt.xlim(xlim[0], xlim[1])\n", " if scale:\n", " plt.gca().set_xscale(scale)\n", "\n", " sns.despine(left=True, bottom=True)\n", - " savename = '-'.join(['cat-'+cat_args['kind']+'plot',x+ '-by-'+ y])\n", - " if row: savename += '_'+row\n", - " if col: savename += '_'+col\n", - " \n", - " save(g, savename )" + " savename = \"-\".join([\"cat-\" + cat_args[\"kind\"] + \"plot\", x + \"-by-\" + y])\n", + " if row:\n", + " savename += \"_\" + row\n", + " if col:\n", + " savename += \"_\" + col\n", + "\n", + " save(g, savename)" ] }, { @@ -278,24 +323,25 @@ "metadata": {}, "outputs": [], "source": [ - "for metric in ['symbolic_solution_rate_(%)','r2_test','accuracy_solution']:\n", - "# for metric in ['r2_test']:\n", - " for kind in ['point']: #,'strip']:\n", - " args =dict(df_compare=df_sum, x=metric, est=np.mean, orient='h',\n", - " kind=kind) \n", - " if kind=='point': \n", - " args['join'] = False \n", - " args['markers']=['o','s','x','+']\n", - " if metric == 'r2_test':\n", - " args['xlim'] = [-1, 1]\n", - " compare(**args,\n", - " hue='target_noise', \n", - " col=None,\n", - " ) \n", - " compare(**args, \n", - " hue='target_noise', \n", - " col='data_group',\n", - " ) " + "for metric in [\"symbolic_solution_rate_(%)\", \"r2_test\", \"accuracy_solution\"]:\n", + " # for metric in ['r2_test']:\n", + " for kind in [\"point\"]: # ,'strip']:\n", + " args = dict(df_compare=df_sum, x=metric, est=np.mean, orient=\"h\", kind=kind)\n", + " if kind == \"point\":\n", + " args[\"join\"] = False\n", + " args[\"markers\"] = [\"o\", \"s\", \"x\", \"+\"]\n", + " if metric == \"r2_test\":\n", + " args[\"xlim\"] = [-1, 1]\n", + " compare(\n", + " **args,\n", + " hue=\"target_noise\",\n", + " col=None,\n", + " )\n", + " compare(\n", + " **args,\n", + " hue=\"target_noise\",\n", + " col=\"data_group\",\n", + " )" ] }, { @@ -306,66 +352,72 @@ "source": [ "# Make the PairGrid\n", "df_plot = df_sum.copy()\n", - "tmp = df_plot.groupby(['target_noise','algorithm'])['symbolic_solution_rate'].mean().unstack().mean()\n", + "tmp = (\n", + " df_plot.groupby([\"target_noise\", \"algorithm\"])[\"symbolic_solution_rate\"]\n", + " .mean()\n", + " .unstack()\n", + " .mean()\n", + ")\n", "order = tmp.sort_values(ascending=False).index\n", - "df_plot['size_diff'] = df_plot['model_size']-df_plot['simplified_complexity']+1\n", - "x_vars=[\n", - "# 'accuracy_solution_rate_(%)',\n", - "# 'mse_test',\n", - "# 'r2_test_rank',\n", - "# 'r2_test_norm',\n", - " 'symbolic_solution_rate_(%)',\n", - " 'r2_test',\n", - " 'simplified_complexity',\n", - "# 'size_diff',\n", - "# 'model_size',\n", - "# 'training time (s)',\n", - "# 'solution'\n", + "df_plot[\"size_diff\"] = df_plot[\"model_size\"] - df_plot[\"simplified_complexity\"] + 1\n", + "x_vars = [\n", + " # 'accuracy_solution_rate_(%)',\n", + " # 'mse_test',\n", + " # 'r2_test_rank',\n", + " # 'r2_test_norm',\n", + " \"symbolic_solution_rate_(%)\",\n", + " \"r2_test\",\n", + " \"simplified_complexity\",\n", + " # 'size_diff',\n", + " # 'model_size',\n", + " # 'training time (s)',\n", + " # 'solution'\n", "]\n", - "g = sns.PairGrid(df_plot, \n", - " x_vars=x_vars,\n", - " y_vars=['algorithm'],\n", - " height=6.5, \n", - " aspect=0.7,\n", - " hue='target_noise',\n", - "# hue_order=[0.01,0.001,0]\n", - "# hue='dataset'\n", - " )\n", - "g.map(sns.pointplot, \n", - "# size=10,\n", - " orient=\"h\",\n", - " jitter=False,\n", - " order=order,\n", - " palette=\"flare_r\",\n", - " errwidth=2,\n", - " linewidth=0.01,\n", - " markeredgecolor='w',\n", - " join=False,\n", - " estimator=np.mean,\n", - " n_boot=1000,\n", - " markers=['x','o','s','+'],\n", - " markeralpha=0.5\n", - " )\n", - "plt.legend(title='Target Noise')\n", - "titles = [x.replace('_',' ').title().replace('(S)','(s)') for x in x_vars]\n", + "g = sns.PairGrid(\n", + " df_plot,\n", + " x_vars=x_vars,\n", + " y_vars=[\"algorithm\"],\n", + " height=6.5,\n", + " aspect=0.7,\n", + " hue=\"target_noise\",\n", + " # hue_order=[0.01,0.001,0]\n", + " # hue='dataset'\n", + ")\n", + "g.map(\n", + " sns.pointplot,\n", + " # size=10,\n", + " orient=\"h\",\n", + " jitter=False,\n", + " order=order,\n", + " palette=\"flare_r\",\n", + " errwidth=2,\n", + " linewidth=0.01,\n", + " markeredgecolor=\"w\",\n", + " join=False,\n", + " estimator=np.mean,\n", + " n_boot=1000,\n", + " markers=[\"x\", \"o\", \"s\", \"+\"],\n", + " markeralpha=0.5,\n", + ")\n", + "plt.legend(title=\"Target Noise\")\n", + "titles = [x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\") for x in x_vars]\n", "\n", "for ax, title in zip(g.axes.flat, titles):\n", - "\n", " # remove xlabel\n", - " ax.set_xlabel('')\n", - " ax.set_ylabel('')\n", + " ax.set_xlabel(\"\")\n", + " ax.set_ylabel(\"\")\n", " # Set a different title for each axes\n", " ax.set(title=title)\n", - " \n", - " if any([n in title.lower() for n in ['size','complexity','time']]):\n", - " ax.set_xscale('log')\n", - " if 'R2' in title and 'Rank' not in title:\n", - " ax.set(title=title.replace('R2','$R^2$'))\n", - " ax.set_xlim([0,1])\n", + "\n", + " if any([n in title.lower() for n in [\"size\", \"complexity\", \"time\"]]):\n", + " ax.set_xscale(\"log\")\n", + " if \"R2\" in title and \"Rank\" not in title:\n", + " ax.set(title=title.replace(\"R2\", \"$R^2$\"))\n", + " ax.set_xlim([0, 1])\n", "\n", " # Make the grid horizontal instead of vertical\n", " ax.yaxis.grid(True)\n", - "save(g, 'pairgrid_'+'_'.join(x_vars))" + "save(g, \"pairgrid_\" + \"_\".join(x_vars))" ] } ], diff --git a/postprocessing/paretoCSVs.py b/postprocessing/paretoCSVs.py index 73fb7fbd..3be4d2d9 100644 --- a/postprocessing/paretoCSVs.py +++ b/postprocessing/paretoCSVs.py @@ -1,59 +1,62 @@ -import pandas as pd import numpy as np -from glob import glob +import pandas as pd from pareto_utils import front np.random.seed(42) -def bootstrap(val, n = 1000, fn=np.mean): + +def bootstrap(val, n=1000, fn=np.mean): val_samples = [] for i in range(n): - sample = np.random.randint(0,len(val)-1, size=len(val)) - val_samples.append( fn(val[sample]) ) + sample = np.random.randint(0, len(val) - 1, size=len(val)) + val_samples.append(fn(val[sample])) m = np.mean(val_samples) sd = np.std(val_samples) - ci_upper = np.quantile(val_samples,0.95) - ci_lower = np.quantile(val_samples,0.05) - return m, sd, ci_upper,ci_lower + ci_upper = np.quantile(val_samples, 0.95) + ci_lower = np.quantile(val_samples, 0.05) + return m, sd, ci_upper, ci_lower + df = pd.read_csv("../docs/csv/blackbox_results.csv") -xcol = 'r2_test' -ycol = 'model_size' -zcol = 'training time (s)' +xcol = "r2_test" +ycol = "model_size" +zcol = "training time (s)" # outline pareto front -pareto_data = df.groupby('algorithm').median() +pareto_data = df.groupby("algorithm").median() + def create_front(df, xc, yc): - objs = df[[xc,yc]].values + objs = df[[xc, yc]].values levels = 5 PFs = [] for el in range(levels): - PF = front(-objs[:,0],objs[:,1]) - objs[PF,:] = 1e10 + PF = front(-objs[:, 0], objs[:, 1]) + objs[PF, :] = 1e10 PFs.append(PF) - if np.all(objs[:,0]==1e10): + if np.all(objs[:, 0] == 1e10): break front_alg = {} for i, pf in enumerate(PFs): print(f"FRONT {i}") for ix in pf: - name = df.iloc[ix].name + name = df.iloc[ix].name if name not in front_alg: front_alg[name] = i print(f"{name}") return front_alg -df['g1']=-1 -for k,v in create_front(pareto_data.copy(), xcol, ycol).items(): - df.loc[df.algorithm == k,'g1'] = v -df['g2']=-1 -for k,v in create_front(pareto_data.copy(), xcol, zcol).items(): - df.loc[df.algorithm == k,'g2'] = v -df['g3']=-1 -for k,v in create_front(pareto_data.copy(), ycol, zcol).items(): - df.loc[df.algorithm == k,'g3'] = v + +df["g1"] = -1 +for k, v in create_front(pareto_data.copy(), xcol, ycol).items(): + df.loc[df.algorithm == k, "g1"] = v +df["g2"] = -1 +for k, v in create_front(pareto_data.copy(), xcol, zcol).items(): + df.loc[df.algorithm == k, "g2"] = v +df["g3"] = -1 +for k, v in create_front(pareto_data.copy(), ycol, zcol).items(): + df.loc[df.algorithm == k, "g3"] = v df.groupby(by="algorithm").median().to_csv("../docs/csv/pareto.csv") diff --git a/postprocessing/pareto_utils.py b/postprocessing/pareto_utils.py index 0838db7c..4e251d11 100644 --- a/postprocessing/pareto_utils.py +++ b/postprocessing/pareto_utils.py @@ -1,53 +1,55 @@ import numpy as np + + # PARETO FRONT TOOLS -def check_dominance(p1,p2): +def check_dominance(p1, p2): flag1 = 0 flag2 = 0 - for o1,o2 in zip(p1,p2): + for o1, o2 in zip(p1, p2): if o1 < o2: flag1 = 1 elif o1 > o2: flag2 = 1 - if flag1==1 and flag2 == 0: + if flag1 == 1 and flag2 == 0: return 1 - elif flag1==0 and flag2 == 1: + elif flag1 == 0 and flag2 == 1: return -1 else: return 0 -def front(obj1,obj2): + +def front(obj1, obj2): """return indices from x and y that are on the Pareto front.""" - rank = [] - assert(len(obj1)==len(obj2)) + assert len(obj1) == len(obj2) n_inds = len(obj1) front = [] for i in np.arange(n_inds): - p = (obj1[i],obj2[i]) + p = (obj1[i], obj2[i]) dcount = 0 dom = [] for j in np.arange(n_inds): - q = (obj1[j],obj2[j]) - compare = check_dominance(p,q) + q = (obj1[j], obj2[j]) + compare = check_dominance(p, q) if compare == 1: dom.append(j) -# print(p,'dominates',q) + # print(p,'dominates',q) elif compare == -1: - dcount = dcount +1 -# print(p,'dominated by',q) + dcount = dcount + 1 + # print(p,'dominated by',q) if dcount == 0: -# print(p,'is on the front') + # print(p,'is on the front') front.append(i) -# f_obj1 = [obj1[f] for f in front] + # f_obj1 = [obj1[f] for f in front] f_obj2 = [obj2[f] for f in front] -# s1 = np.argsort(np.array(f_obj1)) + # s1 = np.argsort(np.array(f_obj1)) s2 = np.argsort(np.array(f_obj2)) -# front = [front[s] for s in s1] + # front = [front[s] for s in s1] front = [front[s] for s in s2] return front diff --git a/postprocessing/pick_best_params.py b/postprocessing/pick_best_params.py index e3a98c9a..e6d7c539 100644 --- a/postprocessing/pick_best_params.py +++ b/postprocessing/pick_best_params.py @@ -2,50 +2,48 @@ # coding: utf-8 # # saved tuned models -# -# Go thru PMLB results and pick the most common params in the best estimators. +# +# Go thru PMLB results and pick the most common params in the best estimators. # Use these parameter settings for the Feynman and Strogatz models. + import pandas as pd -import json -import numpy as np -import os symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'AIFeynman', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', + "AFP", + "AFP_FE", + "AIFeynman", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", ] sr_filenames = [ - 'AFPRegressor', - 'FE_AFPRegressor', - 'AIFeynman', - 'BSRRegressor', - 'DSRRegressor', - 'FFXRegressor', - 'FEATRegressor', - 'EPLEXRegressor', - 'GPGOMEARegressor', - 'gplearn', - 'ITEARegressor', - 'MRGPRegressor', - 'OperonRegressor', - 'sembackpropgp' - ] -sr_name_to_filename = {k:v for k,v in zip(symbolic_algs, sr_filenames)} + "AFPRegressor", + "FE_AFPRegressor", + "AIFeynman", + "BSRRegressor", + "DSRRegressor", + "FFXRegressor", + "FEATRegressor", + "EPLEXRegressor", + "GPGOMEARegressor", + "gplearn", + "ITEARegressor", + "MRGPRegressor", + "OperonRegressor", + "sembackpropgp", +] +sr_name_to_filename = {k: v for k, v in zip(symbolic_algs, sr_filenames)} # read data from feather -df_results = pd.read_feather('../results/black-box_results.feather') +df_results = pd.read_feather("../results/black-box_results.feather") # keep only symbolic regressors df_results = df_results.loc[df_results.algorithm.isin(symbolic_algs)] @@ -63,12 +61,12 @@ # df_results.params.apply(str) best_params = [] -for alg, dfg in df_results.groupby('algorithm'): - counts = dfg['params_str'].value_counts() - print(alg, 'mode of params:',counts.index[0]) +for alg, dfg in df_results.groupby("algorithm"): + counts = dfg["params_str"].value_counts() + print(alg, "mode of params:", counts.index[0]) best_params.append([alg, counts.index[0]]) -# best_params +# best_params # # write tuned model scripts for each algorithm @@ -76,9 +74,8 @@ # In[5]: -import os for alg, bp in best_params: -# os.system('cp ../experiment/methods/{ALG}.py ../experiment/methods/tuned/{ALG}.py'.format(ALG=alg)) + # os.system('cp ../experiment/methods/{ALG}.py ../experiment/methods/tuned/{ALG}.py'.format(ALG=alg)) param_file = sr_name_to_filename[alg] - with open(f'../experiment/methods/tuned/params/_{param_file}.py','w') as f: - f.write('params = {}\n'.format(bp)) + with open(f"../experiment/methods/tuned/params/_{param_file}.py", "w") as f: + f.write("params = {}\n".format(bp)) diff --git a/postprocessing/pmlb_plots.ipynb b/postprocessing/pmlb_plots.ipynb index 0f524135..fe006ef2 100644 --- a/postprocessing/pmlb_plots.ipynb +++ b/postprocessing/pmlb_plots.ipynb @@ -6,26 +6,32 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", "from glob import glob\n", - "import seaborn as sns\n", + "\n", "import matplotlib\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", "%matplotlib inline\n", - "matplotlib.rc('pdf', fonttype=42)\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", "\n", - "datadir = '../../../pmlb/datasets/'\n", + "datadir = \"../../../pmlb/datasets/\"\n", "frames = []\n", - "for f in glob(datadir+'/*/*.tsv.gz'):\n", - " df = pd.read_csv(f,sep='\\t') \n", - " group = 'feynman' if 'feynman' in f else 'strogatz' if 'strogatz' in f else 'black-box'\n", - " frames.append(dict(\n", - " name=f.split('/')[-1][:-7],\n", - " nsamples = df.shape[0],\n", - " nfeatures = df.shape[1],\n", - " npoints = df.shape[0]*df.shape[1],\n", - " Group=group\n", - " ))\n", - " \n", + "for f in glob(datadir + \"/*/*.tsv.gz\"):\n", + " df = pd.read_csv(f, sep=\"\\t\")\n", + " group = (\n", + " \"feynman\" if \"feynman\" in f else \"strogatz\" if \"strogatz\" in f else \"black-box\"\n", + " )\n", + " frames.append(\n", + " dict(\n", + " name=f.split(\"/\")[-1][:-7],\n", + " nsamples=df.shape[0],\n", + " nfeatures=df.shape[1],\n", + " npoints=df.shape[0] * df.shape[1],\n", + " Group=group,\n", + " )\n", + " )\n", + "\n", "df = pd.DataFrame.from_records(frames)\n", "df" ] @@ -37,25 +43,25 @@ "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "sns.set_style('whitegrid')\n", + "\n", + "sns.set_style(\"whitegrid\")\n", "sns.despine(left=True, bottom=True)\n", "## PMLB dataset sizes\n", "g = sns.scatterplot(\n", " data=df,\n", - " x='nsamples',\n", - " y='nfeatures',\n", - " hue='Group',\n", + " x=\"nsamples\",\n", + " y=\"nfeatures\",\n", + " hue=\"Group\",\n", " alpha=0.7,\n", - " s=100, \n", + " s=100,\n", ")\n", "ax = plt.gca()\n", - "plt.legend(loc='upper left')\n", - "ax.set_xscale('log')\n", - "ax.set_yscale('log')\n", - "plt.xlabel('No. of Samples')\n", - "plt.ylabel('No. of Features')\n", - "plt.savefig('figs/pmlb_size.png',dpi=400, bbox_inches='tight')" + "plt.legend(loc=\"upper left\")\n", + "ax.set_xscale(\"log\")\n", + "ax.set_yscale(\"log\")\n", + "plt.xlabel(\"No. of Samples\")\n", + "plt.ylabel(\"No. of Features\")\n", + "plt.savefig(\"figs/pmlb_size.png\", dpi=400, bbox_inches=\"tight\")" ] } ], diff --git a/postprocessing/statistical_comparisons.ipynb b/postprocessing/statistical_comparisons.ipynb index 0c244cef..b73da767 100644 --- a/postprocessing/statistical_comparisons.ipynb +++ b/postprocessing/statistical_comparisons.ipynb @@ -15,26 +15,30 @@ "metadata": {}, "outputs": [], "source": [ - "from scipy.stats import wilcoxon \n", - "import pandas as pd\n", "import itertools as it\n", "import os\n", + "\n", + "import pandas as pd\n", "import seaborn as sns\n", - "sns.set(font_scale=1.5, style='white')\n", + "from scipy.stats import wilcoxon\n", + "\n", + "sns.set(font_scale=1.5, style=\"white\")\n", "\n", - "figdir = 'figs/stats/'\n", - "rdir = '../results/'\n", + "figdir = \"figs/stats/\"\n", + "rdir = \"../results/\"\n", "\n", - "print('figdir:',figdir)\n", - "def save(h=None,name='tmp'):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "print(\"figdir:\", figdir)\n", + "\n", + "\n", + "def save(h=None, name=\"tmp\"):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',figdir+'/'+name+'.png')\n", + " print(\"saving\", figdir + \"/\" + name + \".png\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.png', bbox_inches='tight')" + " plt.savefig(figdir + \"/\" + name + \".png\", bbox_inches=\"tight\")" ] }, { @@ -43,79 +47,73 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", "import numpy as np\n", "\n", + "\n", "def pairwise_pval(df, metric, alg1, alg2):\n", - " df = df.loc[df.algorithm.isin([alg1, alg2]),:].copy()\n", - " x = df.loc[df.algorithm==alg1, metric].values \n", - " y = df.loc[df.algorithm==alg2, metric].values \n", - " eff_size = np.abs(np.nanmedian(x)/np.nanmedian(y))\n", - " if metric.endswith('norm'):\n", - " rmetric = metric.replace('norm','rank')\n", + " df = df.loc[df.algorithm.isin([alg1, alg2]), :].copy()\n", + " x = df.loc[df.algorithm == alg1, metric].values\n", + " y = df.loc[df.algorithm == alg2, metric].values\n", + " eff_size = np.abs(np.nanmedian(x) / np.nanmedian(y))\n", + " if metric.endswith(\"norm\"):\n", + " rmetric = metric.replace(\"norm\", \"rank\")\n", " else:\n", - " rmetric = metric+'_rank'\n", + " rmetric = metric + \"_rank\"\n", " if rmetric not in df.columns:\n", " rmetric = metric\n", - " \n", - " x_rank = df.loc[df.algorithm==alg1, rmetric].values \n", - " y_rank = df.loc[df.algorithm==alg2, rmetric].values \n", - "# pdb.set_trace()\n", + "\n", + " df.loc[df.algorithm == alg1, rmetric].values\n", + " df.loc[df.algorithm == alg2, rmetric].values\n", + " # pdb.set_trace()\n", " if len(x) != len(y):\n", " print(alg1, len(x))\n", " print(alg2, len(y))\n", " assert len(x) == len(y)\n", - "# w, p = mannwhitneyu(x, y)\n", - " if all(y==0) and all(x==0):\n", + " # w, p = mannwhitneyu(x, y)\n", + " if all(y == 0) and all(x == 0):\n", " return 1, 1\n", - " \n", + "\n", " w, p = wilcoxon(x, y)\n", " return p, eff_size\n", "\n", + "\n", "# return pstr, eff_size_str\n", "\n", + "\n", "def signif(pval, alpha, eff):\n", - " pstr = '{:1.2g}'.format(pval)\n", - " eff_size_str = '{:1.1f}X'.format(eff)\n", - " if pval == '-': return pval\n", + " pstr = \"{:1.2g}\".format(pval)\n", + " eff_size_str = \"{:1.1f}X\".format(eff)\n", + " if pval == \"-\":\n", + " return pval\n", " if float(pval) < alpha:\n", - " return 'textbf{'+pstr+'}', 'textbf{'+eff_size_str+'}'\n", - "# return pval+'*'\n", + " return \"textbf{\" + pstr + \"}\", \"textbf{\" + eff_size_str + \"}\"\n", + " # return pval+'*'\n", " else:\n", " return pstr, eff_size_str\n", - " \n", + "\n", + "\n", "def get_pval_df(df, metric, all_algs):\n", " df = df.copy()\n", - " n=0\n", + " n = 0\n", " pvals = []\n", " for alg1, alg2 in it.combinations(all_algs, 2):\n", " pval, eff_size = pairwise_pval(df, metric, alg1, alg2)\n", - " pvals.append(dict(\n", - " alg1 = alg1,\n", - " alg2 = alg2,\n", - " eff_size = eff_size,\n", - " pval = pval\n", - " ))\n", + " pvals.append(dict(alg1=alg1, alg2=alg2, eff_size=eff_size, pval=pval))\n", " n += 1\n", - " c_alpha = alpha/n\n", + " c_alpha = alpha / n\n", "\n", - "\n", - " print('n:',n,'c_alpha:',c_alpha) \n", + " print(\"n:\", n, \"c_alpha:\", c_alpha)\n", " df_pvals = pd.DataFrame.from_records(pvals)\n", " # df_pvals['pval_thresh'] = pd.cut(x=df_pvals['pval'], bins = [0, c_alpha, 1])\n", - " df_pvals['pval_thresh'] = df_pvals['pval'].apply(lambda x: bin_pval(x, c_alpha))\n", + " df_pvals[\"pval_thresh\"] = df_pvals[\"pval\"].apply(lambda x: bin_pval(x, c_alpha))\n", "\n", " # significance\n", - " df_pvals.loc[:,'pval_bold'] = df_pvals.apply(lambda x: signif(x['pval'], \n", - " c_alpha,\n", - " x['eff_size'])[0],\n", - " axis=1\n", - " )\n", - " df_pvals.loc[:,'eff_size_bold'] = df_pvals.apply(lambda x: signif(x['pval'], \n", - " c_alpha,\n", - " x['eff_size'])[1],\n", - " axis=1\n", - " )\n", + " df_pvals.loc[:, \"pval_bold\"] = df_pvals.apply(\n", + " lambda x: signif(x[\"pval\"], c_alpha, x[\"eff_size\"])[0], axis=1\n", + " )\n", + " df_pvals.loc[:, \"eff_size_bold\"] = df_pvals.apply(\n", + " lambda x: signif(x[\"pval\"], c_alpha, x[\"eff_size\"])[1], axis=1\n", + " )\n", " return df_pvals, c_alpha" ] }, @@ -125,66 +123,70 @@ "metadata": {}, "outputs": [], "source": [ - "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", "%matplotlib inline\n", - "from matplotlib.colors import LogNorm\n", + "\n", "\n", "def bin_pval(x, c_alpha):\n", "\n", - " for stars,level in zip([4,3,2,1],[1e-3, 1e-2, 1e-1, 1]):\n", - " if x < level*c_alpha: \n", - " return stars #level #*c_alpha\n", + " for stars, level in zip([4, 3, 2, 1], [1e-3, 1e-2, 1e-1, 1]):\n", + " if x < level * c_alpha:\n", + " return stars # level #*c_alpha\n", " return 0\n", "\n", + "\n", "def pval_heatmap(df, metric, problem, algs):\n", " df = df.copy()\n", - " n = 0 \n", - " pvals = []\n", "\n", " df_pvals, c_alpha = get_pval_df(df, metric, algs)\n", - " \n", + "\n", " # pd.cut(x=df_pvals['pval'], bins = [0, c_alpha, 1])\n", - " tbl = df_pvals.set_index(['alg1','alg2'])['pval_thresh'].unstack().transpose() #.fillna('-')\n", + " tbl = (\n", + " df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval_thresh\"].unstack().transpose()\n", + " ) # .fillna('-')\n", "\n", " mask = np.zeros_like(tbl, dtype=np.bool)\n", - " mask[np.triu_indices_from(mask,k=1)] = True\n", - "\n", - " h = plt.figure(figsize=(10,10))\n", - "\n", - " cmap = sns.color_palette('flare',n_colors=5)\n", - " cmap[0] = [.9, .9, .9]\n", - " ax = sns.heatmap(tbl, \n", - " linewidth=0.25,\n", - " mask=mask,\n", - " square=True,\n", - " cbar_kws=dict(\n", - " ticks = [0.4, 1.2, 2.0, 2.8, 3.6],\n", - " shrink=0.6,\n", - " ),\n", - " cmap=cmap,\n", - "\n", - " )\n", + " mask[np.triu_indices_from(mask, k=1)] = True\n", + "\n", + " h = plt.figure(figsize=(10, 10))\n", + "\n", + " cmap = sns.color_palette(\"flare\", n_colors=5)\n", + " cmap[0] = [0.9, 0.9, 0.9]\n", + " sns.heatmap(\n", + " tbl,\n", + " linewidth=0.25,\n", + " mask=mask,\n", + " square=True,\n", + " cbar_kws=dict(\n", + " ticks=[0.4, 1.2, 2.0, 2.8, 3.6],\n", + " shrink=0.6,\n", + " ),\n", + " cmap=cmap,\n", + " )\n", " cax = h.axes[-1]\n", " cbar_labels = [\n", - " 'no significance',\n", - " '$p<\\\\alpha$',\n", - " '$p<$1e-1$\\cdot \\\\alpha$',\n", - " '$p<$1e-2$\\cdot \\\\alpha$',\n", - " '$p<$1e-3$\\cdot \\\\alpha$',\n", - " ]\n", + " \"no significance\",\n", + " \"$p<\\\\alpha$\",\n", + " \"$p<$1e-1$\\cdot \\\\alpha$\",\n", + " \"$p<$1e-2$\\cdot \\\\alpha$\",\n", + " \"$p<$1e-3$\\cdot \\\\alpha$\",\n", + " ]\n", " cax.set_yticklabels(cbar_labels)\n", - " nice_metric = metric.replace('%','pct').replace('_',' ').replace('R2','$$R^2$$').title()\n", - " plt.title(('Wilcoxon signed-rank test, '\n", - " +nice_metric\n", - " +', $\\\\alpha =$ {:1.1e}').format(c_alpha)\n", - " )\n", - " plt.xlabel('')\n", - " plt.ylabel('')\n", - " savename = ('Pairwise comparison of '\n", - " +nice_metric\n", - " +' on '\n", - " +problem).replace(' ','_')\n", + " nice_metric = (\n", + " metric.replace(\"%\", \"pct\").replace(\"_\", \" \").replace(\"R2\", \"$$R^2$$\").title()\n", + " )\n", + " plt.title(\n", + " (\"Wilcoxon signed-rank test, \" + nice_metric + \", $\\\\alpha =$ {:1.1e}\").format(\n", + " c_alpha\n", + " )\n", + " )\n", + " plt.xlabel(\"\")\n", + " plt.ylabel(\"\")\n", + " savename = (\"Pairwise comparison of \" + nice_metric + \" on \" + problem).replace(\n", + " \" \", \"_\"\n", + " )\n", " save(h, savename)" ] }, @@ -208,17 +210,17 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum = pd.read_csv(rdir+'/symbolic_dataset_results_sum.csv.gz')\n", + "df_sum = pd.read_csv(rdir + \"/symbolic_dataset_results_sum.csv.gz\")\n", "algs = df_sum.algorithm.unique()\n", "datasets = df_sum.dataset.unique()\n", "noises = df_sum.target_noise.unique()\n", "\n", "df_sum\n", "cols = df_sum.columns\n", - "for col in [c for c in cols if 'rank' in c]:\n", - " df_sum.loc[:,col] = df_sum[col].fillna(len(algs)+1)\n", - "for col in [c for c in cols if 'solution_rate' in c]:\n", - " df_sum.loc[:,col] = df_sum[col].fillna(0.0)" + "for col in [c for c in cols if \"rank\" in c]:\n", + " df_sum.loc[:, col] = df_sum[col].fillna(len(algs) + 1)\n", + "for col in [c for c in cols if \"solution_rate\" in c]:\n", + " df_sum.loc[:, col] = df_sum[col].fillna(0.0)" ] }, { @@ -229,25 +231,23 @@ "source": [ "cols = df_sum.columns\n", "filler_frame = {\n", - " 'r2_test' : 0.0,\n", - " 'solution_rate_(%)' : 0.0,\n", + " \"r2_test\": 0.0,\n", + " \"solution_rate_(%)\": 0.0,\n", "}\n", "fill_frames = []\n", "\n", "for dataset in datasets:\n", " for noise in noises:\n", " for alg in algs:\n", - " df = df_sum.loc[df_sum.dataset == dataset].loc[df_sum.target_noise==noise]\n", + " df = df_sum.loc[df_sum.dataset == dataset].loc[df_sum.target_noise == noise]\n", " if alg not in df.algorithm.unique():\n", " new_fill = filler_frame.copy()\n", - " new_fill.update({\n", - " 'algorithm':alg,\n", - " 'dataset':dataset,\n", - " 'target_noise':noise\n", - " })\n", + " new_fill.update(\n", + " {\"algorithm\": alg, \"dataset\": dataset, \"target_noise\": noise}\n", + " )\n", " fill_frames.append(new_fill)\n", - "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys()) \n", - "df_sum_filled = df_sum.append(df_fill)\n" + "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys())\n", + "df_sum_filled = df_sum.append(df_fill)" ] }, { @@ -266,14 +266,16 @@ "alpha = 0.05\n", "SR_pvals = {}\n", "\n", - "for metric in ['r2_test','symbolic_solution_rate','simplified_complexity']:\n", - " for target_noise, dfg in df_sum_filled.groupby('target_noise'):\n", - " print(metric, 'noise=',target_noise)\n", - " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs) \n", + "for metric in [\"r2_test\", \"symbolic_solution_rate\", \"simplified_complexity\"]:\n", + " for target_noise, dfg in df_sum_filled.groupby(\"target_noise\"):\n", + " print(metric, \"noise=\", target_noise)\n", + " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs)\n", "\n", - " pval_tbl = df_pvals.set_index(['alg1','alg2'])['pval_bold'].unstack().fillna('-')\n", - " disp_tbl = df_pvals.set_index(['alg1','alg2'])['pval'].unstack().fillna('-')\n", - " SR_pvals[metric] = {target_noise:disp_tbl}\n", + " pval_tbl = (\n", + " df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval_bold\"].unstack().fillna(\"-\")\n", + " )\n", + " disp_tbl = df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval\"].unstack().fillna(\"-\")\n", + " SR_pvals[metric] = {target_noise: disp_tbl}\n", " display(disp_tbl)\n", "\n", " pval_tbl" @@ -292,11 +294,10 @@ "metadata": {}, "outputs": [], "source": [ - "for target_noise, dfg in df_sum_filled.groupby('target_noise'):\n", - " for metric in ['r2_test','symbolic_solution_rate','simplified_complexity']:\n", - " name = 'symbolic problems target_noise='+str(target_noise)\n", - " pval_heatmap(dfg, metric, name, algs)\n", - " " + "for target_noise, dfg in df_sum_filled.groupby(\"target_noise\"):\n", + " for metric in [\"r2_test\", \"symbolic_solution_rate\", \"simplified_complexity\"]:\n", + " name = \"symbolic problems target_noise=\" + str(target_noise)\n", + " pval_heatmap(dfg, metric, name, algs)" ] }, { @@ -319,7 +320,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum = pd.read_csv(rdir+'/results_black-box_summary.csv.gz')\n", + "df_sum = pd.read_csv(rdir + \"/results_black-box_summary.csv.gz\")\n", "algs = df_sum.algorithm.unique()\n", "datasets = df_sum.dataset.unique()" ] @@ -333,8 +334,8 @@ "# if a result is missing, assign it the worst ranking\n", "# cols = df_sum.columns\n", "filler_frame = {\n", - " 'r2_test_rank' : len(algs)+1,\n", - " 'model_size_rank' : len(algs)+1,\n", + " \"r2_test_rank\": len(algs) + 1,\n", + " \"model_size_rank\": len(algs) + 1,\n", "}\n", "fill_frames = []\n", "j = 0\n", @@ -342,19 +343,21 @@ " for alg in algs:\n", " df = df_sum.loc[df_sum.dataset == dataset]\n", " if alg not in df.algorithm.unique():\n", - " print('filling',alg,'missing for',dataset)\n", - "# for col in cols:\n", + " print(\"filling\", alg, \"missing for\", dataset)\n", + " # for col in cols:\n", " new_fill = filler_frame.copy()\n", - " new_fill.update({\n", - " 'algorithm':alg,\n", - " 'dataset':dataset,\n", - " })\n", + " new_fill.update(\n", + " {\n", + " \"algorithm\": alg,\n", + " \"dataset\": dataset,\n", + " }\n", + " )\n", " print(new_fill)\n", " fill_frames.append(new_fill)\n", " j += 1\n", - " \n", - "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys()) \n", - "df_sum_filled = df_sum.append(df_fill)\n" + "\n", + "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys())\n", + "df_sum_filled = df_sum.append(df_fill)" ] }, { @@ -374,16 +377,16 @@ "alpha = 0.05\n", "BB_pvals = {}\n", "\n", - "for metric in ['r2_test','model_size']:\n", + "for metric in [\"r2_test\", \"model_size\"]:\n", " pvals = []\n", "\n", - " n = 0 \n", - " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs) \n", - " pval_tbl = df_pvals.set_index(['alg1','alg2'])['pval'].unstack() \n", - " BB_pvals[metric] = pval_tbl.copy() \n", - " pval_tbl = pval_tbl.fillna('-')\n", + " n = 0\n", + " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs)\n", + " pval_tbl = df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval\"].unstack()\n", + " BB_pvals[metric] = pval_tbl.copy()\n", + " pval_tbl = pval_tbl.fillna(\"-\")\n", " display(pval_tbl)\n", - " \n", + "\n", " pval_tbl" ] }, @@ -393,8 +396,8 @@ "metadata": {}, "outputs": [], "source": [ - "for metric in ['r2_test','model_size']:\n", - " name = 'black-box problems'\n", + "for metric in [\"r2_test\", \"model_size\"]:\n", + " name = \"black-box problems\"\n", " pval_heatmap(df_sum_filled, metric, name, algs)" ] }, @@ -412,13 +415,13 @@ "outputs": [], "source": [ "# first fill the matrices so we can look at either pair\n", - "for k,v in BB_pvals.items():\n", + "for k, v in BB_pvals.items():\n", " algs = v.columns\n", - " for alg1, alg2 in it.combinations(algs,2):\n", - " if np.isnan(BB_pvals[k].loc[alg1,alg2]): \n", - " BB_pvals[k].loc[alg1,alg2] = BB_pvals[k].loc[alg2,alg1] \n", + " for alg1, alg2 in it.combinations(algs, 2):\n", + " if np.isnan(BB_pvals[k].loc[alg1, alg2]):\n", + " BB_pvals[k].loc[alg1, alg2] = BB_pvals[k].loc[alg2, alg1]\n", " else:\n", - " BB_pvals[k].loc[alg2,alg1] = BB_pvals[k].loc[alg1,alg2] " + " BB_pvals[k].loc[alg2, alg1] = BB_pvals[k].loc[alg1, alg2]" ] }, { @@ -434,7 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test']['Operon'].max() " + "BB_pvals[\"r2_test\"][\"Operon\"].max()" ] }, { @@ -450,7 +453,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['model_size'].loc['FEAT','SBP-GP']" + "BB_pvals[\"model_size\"].loc[\"FEAT\", \"SBP-GP\"]" ] }, { @@ -466,7 +469,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc['XGB',['Operon','SBP-GP']].max() #.replace('-','0').astype(float).max() #.round(2)" + "BB_pvals[\"r2_test\"].loc[\n", + " \"XGB\", [\"Operon\", \"SBP-GP\"]\n", + "].max() # .replace('-','0').astype(float).max() #.round(2)" ] }, { @@ -482,7 +487,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc['LGBM',['Operon','FEAT','SBP-GP']].max() #.replace('-','0').astype(float).max() #.round(2)" + "BB_pvals[\"r2_test\"].loc[\n", + " \"LGBM\", [\"Operon\", \"FEAT\", \"SBP-GP\"]\n", + "].max() # .replace('-','0').astype(float).max() #.round(2)" ] }, { @@ -499,11 +506,12 @@ "outputs": [], "source": [ "import itertools as it\n", + "\n", "import numpy as np\n", "\n", - "gps = ['Operon','FEAT','EPLEX','ITEA','GP-GOMEA']\n", - "trees = ['LGBM','XGB','RandomForest','AdaBoost']\n", - "BB_pvals['model_size'].loc[gps,trees].max().max()\n" + "gps = [\"Operon\", \"FEAT\", \"EPLEX\", \"ITEA\", \"GP-GOMEA\"]\n", + "trees = [\"LGBM\", \"XGB\", \"RandomForest\", \"AdaBoost\"]\n", + "BB_pvals[\"model_size\"].loc[gps, trees].max().max()" ] }, { @@ -512,7 +520,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc[['DSR','BSR','FFX','AIFeynman'],['DSR','BSR','FFX','AIFeynman']]" + "BB_pvals[\"r2_test\"].loc[\n", + " [\"DSR\", \"BSR\", \"FFX\", \"AIFeynman\"], [\"DSR\", \"BSR\", \"FFX\", \"AIFeynman\"]\n", + "]" ] }, { @@ -521,7 +531,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc[['DSR','FFX'],['BSR','AIFeynman']]" + "BB_pvals[\"r2_test\"].loc[[\"DSR\", \"FFX\"], [\"BSR\", \"AIFeynman\"]]" ] }, { @@ -530,7 +540,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'][['BSR','AIFeynman']]" + "BB_pvals[\"r2_test\"][[\"BSR\", \"AIFeynman\"]]" ] } ], diff --git a/postprocessing/symbolic_utils.py b/postprocessing/symbolic_utils.py index 4a1e59ba..66407dde 100644 --- a/postprocessing/symbolic_utils.py +++ b/postprocessing/symbolic_utils.py @@ -1,28 +1,21 @@ -import pdb -import pandas as pd -from yaml import load, Loader -import sympy # from sympy import * -from sympy import Symbol, simplify, factor, Float, preorder_traversal, Integer -from sympy.parsing.sympy_parser import parse_expr -import re -import ast +from sympy import Float, Integer, preorder_traversal def complexity(expr): - c=0 + c = 0 for arg in preorder_traversal(expr): c += 1 return c - + + def round_floats(ex1): ex2 = ex1 for a in preorder_traversal(ex1): if isinstance(a, Float): if abs(a) < 0.0001: - ex2 = ex2.subs(a,Integer(0)) + ex2 = ex2.subs(a, Integer(0)) else: ex2 = ex2.subs(a, round(a, 3)) return ex2 - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..58450937 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +required-version = "==0.17.0" +target-version = "py37" +line-length = 88 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] + +[tool.mypy] +python_version = "3.10" +check_untyped_defs = true +ignore_missing_imports = true +follow_imports = "skip" +no_site_packages = true +warn_unused_ignores = true diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..907de0e5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +# Ruff and mypy versions are pinned in .pre-commit-config.yaml. +pre-commit==4.6.2 diff --git a/scripts/check_method_layout.py b/scripts/check_method_layout.py index affb5357..9b84307c 100644 --- a/scripts/check_method_layout.py +++ b/scripts/check_method_layout.py @@ -129,7 +129,9 @@ def check_algorithm(name: str) -> None: if parsed is not None or name not in METADATA_GRANDFATHERED: missing = [k for k in REQUIRED_METADATA_KEYS if k not in (parsed or {})] if missing and name not in METADATA_GRANDFATHERED: - error(name, f"{metadata} is missing required key(s): {', '.join(missing)}") + error( + name, f"{metadata} is missing required key(s): {', '.join(missing)}" + ) elif missing: warn(name, f"{metadata} is incomplete (missing {', '.join(missing)})") @@ -157,9 +159,14 @@ def check_algorithm(name: str) -> None: error(name, f"{regressor} does not define `{required}` at module level") if os.path.isfile(os.path.join(alg_path, "regressor.py")): - warn(name, f"{alg_path}/regressor.py is ignored; the harness imports {regressor}") + warn( + name, f"{alg_path}/regressor.py is ignored; the harness imports {regressor}" + ) if os.path.isfile(os.path.join(method_path, "metadata.yml")): - warn(name, f"{method_path}/metadata.yml is ignored; metadata.yml belongs in {alg_path}/") + warn( + name, + f"{method_path}/metadata.yml is ignored; metadata.yml belongs in {alg_path}/", + ) for stray in ("install.sh", "environment.yml", "requirements.txt", "Dockerfile"): if os.path.isfile(os.path.join(method_path, stray)): error( @@ -179,10 +186,14 @@ def check_orphan_method(name: str) -> None: def main() -> int: if not os.path.isdir(ALG_DIR) or not os.path.isdir(METHOD_DIR): - print(f"error: run this from the repository root (missing {ALG_DIR}/ or {METHOD_DIR}/)") + print( + f"error: run this from the repository root (missing {ALG_DIR}/ or {METHOD_DIR}/)" + ) return 1 - algorithms = sorted(d for d in os.listdir(ALG_DIR) if os.path.isdir(os.path.join(ALG_DIR, d))) + algorithms = sorted( + d for d in os.listdir(ALG_DIR) if os.path.isdir(os.path.join(ALG_DIR, d)) + ) methods = sorted( d for d in os.listdir(METHOD_DIR) @@ -199,13 +210,23 @@ def main() -> int: print(f"checked {len(algorithms)} algorithms and {len(methods)} method directories") for line in warnings: - print(f"::warning::{line}" if os.environ.get("GITHUB_ACTIONS") else f"warning: {line}") + print( + f"::warning::{line}" + if os.environ.get("GITHUB_ACTIONS") + else f"warning: {line}" + ) if errors: print() for line in errors: - print(f"::error::{line}" if os.environ.get("GITHUB_ACTIONS") else f"error: {line}") - print(f"\n{len(errors)} problem(s) found. See CONTRIBUTING.md for the expected layout.") + print( + f"::error::{line}" + if os.environ.get("GITHUB_ACTIONS") + else f"error: {line}" + ) + print( + f"\n{len(errors)} problem(s) found. See CONTRIBUTING.md for the expected layout." + ) return 1 print("method layout OK") diff --git a/scripts/typecheck.py b/scripts/typecheck.py new file mode 100644 index 00000000..6458018f --- /dev/null +++ b/scripts/typecheck.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Type-check every Python file, including importlib-loaded hyphenated methods. + +Mypy's CLI rejects package names such as gp-elite and ps-tree. Supply explicit, +unique module names so these adapters are checked alongside the rest of the +repository without renaming the methods or excluding their source code. +""" + +import subprocess +from pathlib import Path + +from mypy.build import BuildSource, build +from mypy.errors import CompileError +from mypy.main import process_options + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + # Include tracked and new source files anywhere in the repo, honoring gitignore. + listing = subprocess.check_output( + [ + "git", + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + "*.py", + ], + cwd=root, + ) + paths = sorted({root / name.decode() for name in listing.split(b"\0") if name}) + paths = [path for path in paths if path.is_file()] + + sources = [] + for path in paths: + parts = list(path.relative_to(root).with_suffix("").parts) + if parts[-1] == "__init__": + parts.pop() + module = ".".join(part.replace("-", "_") for part in parts) or "srbench" + sources.append(BuildSource(str(path), module, None)) + + _, options = process_options( + ["--config-file", str(root / "pyproject.toml")], require_targets=False + ) + try: + result = build(sources=sources, options=options) + messages = result.errors + except CompileError as exc: + messages = exc.messages + for message in messages: + print(message) + if messages: + return 1 + print(f"Success: no issues found in {len(sources)} source files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2b7460987fb9947785f0c5b9abf6fff3360dce9a Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 11:03:54 -0400 Subject: [PATCH 03/10] pin base python for pyarrow compatibility --- base_environment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base_environment.yml b/base_environment.yml index 3eac1ee7..ea12790c 100755 --- a/base_environment.yml +++ b/base_environment.yml @@ -2,6 +2,9 @@ channels: - conda-forge - defaults dependencies: + # pyarrow does not yet publish Conda packages for Python 3.15. + # Keep the shared image on the supported version recorded in the lock file. + - python=3.12 - scikit-learn - pandas - numpy From 691d28e2464ab92cf6b874a40823ec7eda78108b Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 11:09:16 -0400 Subject: [PATCH 04/10] allow method-specific python versions, as long as not 3.15 (no support for pyarrow) --- base_environment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/base_environment.yml b/base_environment.yml index ea12790c..6419924d 100755 --- a/base_environment.yml +++ b/base_environment.yml @@ -2,9 +2,10 @@ channels: - conda-forge - defaults dependencies: - # pyarrow does not yet publish Conda packages for Python 3.15. - # Keep the shared image on the supported version recorded in the lock file. - - python=3.12 + # pyarrow does not yet publish Conda packages for Python 3.15. Keep this + # as a ceiling so method-specific environment files can select their own + # compatible Python version (for example, FFX needs 3.11.10). + - python<3.15 - scikit-learn - pandas - numpy From 96ed5db3a5645391d2c8c5dd62d182a30eac5fab Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 11:22:37 -0400 Subject: [PATCH 05/10] fix: cap Python for pyarrow-dependent methods --- algorithms/gp-elite/environment.yml | 5 +++++ algorithms/keplearn/environment.yml | 2 ++ algorithms/pir/environment.yml | 5 +++++ base_environment.yml | 4 ---- 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 algorithms/gp-elite/environment.yml create mode 100644 algorithms/pir/environment.yml diff --git a/algorithms/gp-elite/environment.yml b/algorithms/gp-elite/environment.yml new file mode 100644 index 00000000..c52bc44b --- /dev/null +++ b/algorithms/gp-elite/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge +dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 diff --git a/algorithms/keplearn/environment.yml b/algorithms/keplearn/environment.yml index 96b694b4..9bedbc7d 100644 --- a/algorithms/keplearn/environment.yml +++ b/algorithms/keplearn/environment.yml @@ -1,6 +1,8 @@ channels: - conda-forge dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 - numpy - sympy - scikit-learn diff --git a/algorithms/pir/environment.yml b/algorithms/pir/environment.yml new file mode 100644 index 00000000..c52bc44b --- /dev/null +++ b/algorithms/pir/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge +dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 diff --git a/base_environment.yml b/base_environment.yml index 6419924d..3eac1ee7 100755 --- a/base_environment.yml +++ b/base_environment.yml @@ -2,10 +2,6 @@ channels: - conda-forge - defaults dependencies: - # pyarrow does not yet publish Conda packages for Python 3.15. Keep this - # as a ceiling so method-specific environment files can select their own - # compatible Python version (for example, FFX needs 3.11.10). - - python<3.15 - scikit-learn - pandas - numpy From b7192a53812d71c90a28e7195367dc4df6e80d93 Mon Sep 17 00:00:00 2001 From: Guilherme Aldeia Date: Wed, 14 Oct 2026 11:44:08 -0400 Subject: [PATCH 06/10] Improve code formatting and readability in regressor.py Refactor code for better readability and maintainability. --- experiment/methods/gp-elite/regressor.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/experiment/methods/gp-elite/regressor.py b/experiment/methods/gp-elite/regressor.py index ed40b4c0..fa9fae28 100644 --- a/experiment/methods/gp-elite/regressor.py +++ b/experiment/methods/gp-elite/regressor.py @@ -107,7 +107,7 @@ def fit(self, X, y): robust=self.robust, parallel=self.parallel, time_limit=max(1.0, remaining / (n - k)), - seed=seed0 + 1000 * k + seed=seed0 + 1000 * k, ) score = r.r2_validation if score is None or not np.isfinite(score): @@ -116,10 +116,11 @@ def fit(self, X, y): score = 1.0 - float(np.sum((y - p) ** 2) / ss) if best is None or score > best_score: best, best_score = r, score - self._publish(best) # published as soon as found + self._publish(best) # published as soon as found if best is not None and not getattr(self, "is_fitted_", False): self._publish(best) + return self @@ -150,8 +151,11 @@ def model(est, X=None): operators are spelled as the engine computes them (sqrt(Abs(u)) where u can be negative). """ - cols = list(X.columns) if X is not None and hasattr(X, "columns") \ + cols = ( + list(X.columns) + if X is not None and hasattr(X, "columns") else ["x%d" % i for i in range(getattr(est, "n_features_in_", 0))] + ) return est.model_.sympy(cols) From 216adc2e6939836487eac9e956e592f68793bfed Mon Sep 17 00:00:00 2001 From: Guilherme Aldeia Date: Wed, 14 Oct 2026 11:47:15 -0400 Subject: [PATCH 07/10] Remove unnecessary whitespace in regressor.py --- experiment/methods/gp-elite/regressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/methods/gp-elite/regressor.py b/experiment/methods/gp-elite/regressor.py index fa9fae28..cbf170f9 100644 --- a/experiment/methods/gp-elite/regressor.py +++ b/experiment/methods/gp-elite/regressor.py @@ -120,7 +120,7 @@ def fit(self, X, y): if best is not None and not getattr(self, "is_fitted_", False): self._publish(best) - + return self From ed3bc103e3ec7e241446718888f064af51f7eb72 Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 15:26:23 -0400 Subject: [PATCH 08/10] Better handling of feature names when they are already x1, x_1, X_1, X1, etc. renames generic feature references into the actual feature names used by a dataset, while trying not to accidentally modify models that already use the correct names. --- experiment/symbolic_utils.py | 53 ++++++++++++++++++++++++++++-------- experiment/test_algorithm.py | 50 +++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 12 deletions(-) diff --git a/experiment/symbolic_utils.py b/experiment/symbolic_utils.py index c46c49ed..229aeca3 100644 --- a/experiment/symbolic_utils.py +++ b/experiment/symbolic_utils.py @@ -172,17 +172,48 @@ def clean_pred_model(model_str, dataset, est_name): local_dict = {k: Symbol(k) for k in features} new_model_str = model_str - # rename features - for i, f in reversed(list(enumerate(features))): - print("replacing feature", i, "with", f) - if any([n in est_name.lower() for n in ["mrgp", "operon", "dsr"]]): - i = i + 1 - new_model_str = new_model_str.replace("x" + str(i), f) - new_model_str = new_model_str.replace("x_" + str(i), f) - new_model_str = new_model_str.replace("X_" + str(i), f) - new_model_str = new_model_str.replace("X" + str(i), f) - new_model_str = new_model_str.replace("x[:,{}]".format(i), f) - new_model_str = new_model_str.replace("x[{}]".format(i), f) + # Rename generic feature placeholders. Model strings are expected to use + # dataset feature names, and some datasets genuinely have names such as + # ``x1`` and ``x2``. In that case, treating the names as placeholders would + # corrupt an already-correct model. + placeholder_pattern = r"\b(?:x_|X_|x|X)(\d+)\b" + placeholder_tokens = { + match.group(0) for match in re.finditer(placeholder_pattern, new_model_str) + } + feature_names = {str(feature) for feature in features} + index_offset = int( # Some methods start counting features at 1 + any(name in est_name.lower() for name in ["mrgp", "operon", "dsr"]) + ) + + def feature_name(index): + feature_index = index - index_offset + if 0 <= feature_index < len(features): + return str(features[feature_index]) + return None + + # If every placeholder-looking identifier is an actual feature name, the + # string is already in the documented dataset-name form. This intentionally + # favors preserving a compliant model in the otherwise ambiguous x1/x2 case. + if not placeholder_tokens.issubset(feature_names): + + def replace_placeholder(match): + replacement = feature_name(int(match.group(1))) + return replacement if replacement is not None else match.group(0) + + new_model_str = re.sub(placeholder_pattern, replace_placeholder, new_model_str) + + # Array-style placeholders cannot be dataset feature names, so they can + # always be normalized. Keep this separate from identifier replacement. + def replace_array_placeholder(match): + replacement = feature_name(int(match.group(1))) + return replacement if replacement is not None else match.group(0) + + new_model_str = re.sub( + r"x\[:,\s*(\d+)\s*\]", replace_array_placeholder, new_model_str + ) + new_model_str = re.sub( + r"x\[\s*(\d+)\s*\]", replace_array_placeholder, new_model_str + ) # operators new_model_str = new_model_str.replace("^", "**") # GP-GOMEA diff --git a/experiment/test_algorithm.py b/experiment/test_algorithm.py index ebae2bf3..ac36eb6a 100644 --- a/experiment/test_algorithm.py +++ b/experiment/test_algorithm.py @@ -6,6 +6,8 @@ from os.path import abspath from os.path import dirname as d +import numpy as np +import pandas as pd import pytest sys.path.append(d(abspath(__file__))) @@ -17,7 +19,8 @@ complexity, round_floats, ) -from sympy import Symbol +import symbolic_utils +from sympy import Symbol, simplify # symbolic model stuff from sympy.parsing.sympy_parser import parse_expr @@ -36,6 +39,51 @@ random_state = 42 +def _use_toy_dataframe(monkeypatch, feature_names): + """Make clean_pred_model see a small dataset without a fixture on disk.""" + dataframe = pd.DataFrame( + np.zeros((2, len(feature_names))), columns=feature_names + ) + dataframe["target"] = 0.0 + + def read_toy_file(_dataset): + return ( + dataframe.drop(columns="target"), + dataframe["target"].to_numpy(), + dataframe.drop(columns="target").columns.to_numpy(), + ) + + monkeypatch.setattr(symbolic_utils, "read_file", read_toy_file) + + +def test_clean_pred_model_preserves_x_named_features(monkeypatch): + feature_names = ["x1", "x2", "x3", "y1", "y2", "y3"] + model = "x1*y1 + x2*y2 + x3*y3" + _use_toy_dataframe(monkeypatch, feature_names) + + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", "example") + + assert simplify(cleaned - parse_expr(model)) == 0 + + +def test_clean_pred_model_preserves_mixed_x_named_features(monkeypatch): + feature_names = ["m1", "m2", "G", "x1", "x2", "y1", "y2", "z1", "z2"] + model = "x1**2 - 2*x1*x2 + x2**2" + _use_toy_dataframe(monkeypatch, feature_names) + + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", "example") + + assert simplify(cleaned - parse_expr(model)) == 0 + + +def test_clean_pred_model_still_normalizes_generic_placeholders(monkeypatch): + _use_toy_dataframe(monkeypatch, ["mass", "velocity"]) + + cleaned = symbolic_utils.clean_pred_model("x0 + x1", "toy.tsv", "example") + + assert simplify(cleaned - parse_expr("mass + velocity")) == 0 + + def get_algorithm(ml): algorithm = importlib.__import__( From e42116b2e2aedeb55a818a04744cba1a347bf132 Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 15:32:03 -0400 Subject: [PATCH 09/10] using methods name on clean_pred_model call --- experiment/test_algorithm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/experiment/test_algorithm.py b/experiment/test_algorithm.py index ac36eb6a..077111f6 100644 --- a/experiment/test_algorithm.py +++ b/experiment/test_algorithm.py @@ -56,30 +56,32 @@ def read_toy_file(_dataset): monkeypatch.setattr(symbolic_utils, "read_file", read_toy_file) -def test_clean_pred_model_preserves_x_named_features(monkeypatch): +def test_clean_pred_model_preserves_x_named_features(monkeypatch, ml): feature_names = ["x1", "x2", "x3", "y1", "y2", "y3"] model = "x1*y1 + x2*y2 + x3*y3" _use_toy_dataframe(monkeypatch, feature_names) - cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", "example") + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) assert simplify(cleaned - parse_expr(model)) == 0 -def test_clean_pred_model_preserves_mixed_x_named_features(monkeypatch): +def test_clean_pred_model_preserves_mixed_x_named_features(monkeypatch, ml): feature_names = ["m1", "m2", "G", "x1", "x2", "y1", "y2", "z1", "z2"] model = "x1**2 - 2*x1*x2 + x2**2" _use_toy_dataframe(monkeypatch, feature_names) - cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", "example") + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) assert simplify(cleaned - parse_expr(model)) == 0 -def test_clean_pred_model_still_normalizes_generic_placeholders(monkeypatch): +def test_clean_pred_model_still_normalizes_generic_placeholders(monkeypatch, ml): _use_toy_dataframe(monkeypatch, ["mass", "velocity"]) - cleaned = symbolic_utils.clean_pred_model("x0 + x1", "toy.tsv", "example") + is_one_based = any(name in ml.lower() for name in ["mrgp", "operon", "dsr"]) + model = "x1 + x2" if is_one_based else "x0 + x1" + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) assert simplify(cleaned - parse_expr("mass + velocity")) == 0 From 2545e18b0b6c89437c8854cccb4a4b9a4f76ab5f Mon Sep 17 00:00:00 2001 From: Guilherme Seidyo Imai Aldeia Date: Wed, 14 Oct 2026 15:36:31 -0400 Subject: [PATCH 10/10] ran linter --- experiment/symbolic_utils.py | 3 ++- experiment/test_algorithm.py | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/experiment/symbolic_utils.py b/experiment/symbolic_utils.py index 229aeca3..3d53670b 100644 --- a/experiment/symbolic_utils.py +++ b/experiment/symbolic_utils.py @@ -181,7 +181,8 @@ def clean_pred_model(model_str, dataset, est_name): match.group(0) for match in re.finditer(placeholder_pattern, new_model_str) } feature_names = {str(feature) for feature in features} - index_offset = int( # Some methods start counting features at 1 + + index_offset = int( # Some methods start counting features at 1 any(name in est_name.lower() for name in ["mrgp", "operon", "dsr"]) ) diff --git a/experiment/test_algorithm.py b/experiment/test_algorithm.py index 077111f6..ac7c5c74 100644 --- a/experiment/test_algorithm.py +++ b/experiment/test_algorithm.py @@ -12,14 +12,13 @@ sys.path.append(d(abspath(__file__))) - +import symbolic_utils from evaluate_model import evaluate_model from read_file import read_file from symbolic_utils import ( complexity, round_floats, ) -import symbolic_utils from sympy import Symbol, simplify # symbolic model stuff @@ -41,9 +40,7 @@ def _use_toy_dataframe(monkeypatch, feature_names): """Make clean_pred_model see a small dataset without a fixture on disk.""" - dataframe = pd.DataFrame( - np.zeros((2, len(feature_names))), columns=feature_names - ) + dataframe = pd.DataFrame(np.zeros((2, len(feature_names))), columns=feature_names) dataframe["target"] = 0.0 def read_toy_file(_dataset):