diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..d863df235 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,23 @@ +name: Code quality + +on: + push: + branches: [master, dev] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint-format-typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install pre-commit + run: python -m pip install -r requirements-dev.txt + - name: Lint, format, and type-check all source files + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.gitignore b/.gitignore index ba8c27ea7..3b7523c0b 100755 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,5 @@ ENV/ # mypy .mypy_cache/ +# Ruff +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..760b2fe76 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +default_language_version: + python: python3 +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.0 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy (all Python sources) + entry: python scripts/typecheck.py + language: python + additional_dependencies: [mypy==2.4.0] + pass_filenames: false + always_run: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 033acbbb6..4584bb847 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,38 @@ Ground Rules 2. Make the PR detailed and reference [specific issues](https://github.com/cavalab/srbench/issues) if the PR is meant to address any. 3. **Please be kind and please be patient**. We will be, too. +Code quality +============ + +Install the development tools and enable the commit hooks from the repository root: + +```bash +python -m pip install -r requirements-dev.txt +pre-commit install +pre-commit run --all-files +``` + +The hooks run Ruff linting (with safe automatic fixes), Ruff formatting, and mypy. +CI runs the same hooks over the entire repository on every pull request and on +pushes to `master` and `dev`. If a hook changes files, review and stage the changes, +then rerun it before committing. Python scripts and Jupyter notebook code cells +are linted and formatted; Markdown Python examples are also formatted. Notebook +outputs are preserved. + +Ruff rules and mypy settings live in `pyproject.toml`. Tool versions are pinned +in `.pre-commit-config.yaml`; update Ruff's hook revision and `required-version` +together when upgrading. The formatter targets Python 3.7 syntax to accommodate +older algorithm environments; development hooks and CI run on Python 3.12. + +For type checking alone, install `mypy==2.4.0` and run +`python scripts/typecheck.py`. The runner checks all Python files, including +hyphenated method directories that mypy's CLI cannot treat as package names. +Mypy checks unannotated function bodies without requiring type annotations +throughout this existing codebase. Algorithm libraries are installed in separate +containers, so third-party imports are treated as `Any` and site packages are +ignored for consistent results across environments. Python source in notebooks +is covered by Ruff, but is not type-checked by mypy. + How to contribute an SR method ============================== @@ -108,10 +140,10 @@ specify a mapping in the `model` function such as: ```python def model(est, X=None): - mapping = {'x_'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x_" + str(i): k for i, k in enumerate(X.columns)} new_model = est.model_ - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) ``` 2. The operators/functions in the model are available in [sympy's function set](https://docs.sympy.org/latest/modules/functions/index.html). diff --git a/algorithms/gp-elite/environment.yml b/algorithms/gp-elite/environment.yml new file mode 100644 index 000000000..c52bc44b2 --- /dev/null +++ b/algorithms/gp-elite/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge +dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 diff --git a/algorithms/gp-elite/requirements.txt b/algorithms/gp-elite/requirements.txt index 34e17da55..c8146f9c6 100644 --- a/algorithms/gp-elite/requirements.txt +++ b/algorithms/gp-elite/requirements.txt @@ -1 +1 @@ -gp-elite>=0.5.0 +gp-elite>=0.8.0 diff --git a/algorithms/keplearn/environment.yml b/algorithms/keplearn/environment.yml index 96b694b47..9bedbc7df 100644 --- a/algorithms/keplearn/environment.yml +++ b/algorithms/keplearn/environment.yml @@ -1,6 +1,8 @@ channels: - conda-forge dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 - numpy - sympy - scikit-learn diff --git a/algorithms/pir/environment.yml b/algorithms/pir/environment.yml new file mode 100644 index 000000000..c52bc44b2 --- /dev/null +++ b/algorithms/pir/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge +dependencies: + # Temporary ceiling: pyarrow has no Python 3.15 Conda package yet. + - python<3.15 diff --git a/algorithms/tir/regressor.py b/algorithms/tir/regressor.py index 0dd8c6473..d06384b3a 100644 --- a/algorithms/tir/regressor.py +++ b/algorithms/tir/regressor.py @@ -1,45 +1,50 @@ -import sys -import os import pyTIR as tir -from itertools import product -#os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + +# os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" hyper_params = [ { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Sqrt,Exp,Log,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Sqrt,Exp,Log,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Sqrt,Exp,Log,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Sqrt,Exp,Log,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), }, ] # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = tir.TIRRegressor(npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5,5), error="R^2", alg="MOO") +eval_kwargs = {"scale_x": False, "scale_y": False} +est = tir.TIRRegressor( + npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5, 5), error="R^2", alg="MOO" +) + def pre_train(est, X, y): """Adjust settings based on data before training""" - if X.shape[0]*X.shape[1] <= 1000: + if X.shape[0] * X.shape[1] <= 1000: est.penalty = 0.01 + def complexity(e): return e.len + def model(e, X): - new_model = e.sympy.replace("^","**") - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) + new_model = e.sympy.replace("^", "**") + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) return new_model + def get_population(est): pop = [] - for i in range(min(100, len(est.front))-1,-1,-1): + for i in range(min(100, len(est.front)) - 1, -1, -1): pop.append(est.create_model_from(i)) - return pop + return pop + def get_best_solution(est): return est diff --git a/docs/CompetitionGuide.md b/docs/CompetitionGuide.md index 0dc7b0ac8..f45920c51 100644 --- a/docs/CompetitionGuide.md +++ b/docs/CompetitionGuide.md @@ -79,10 +79,10 @@ specify a mapping in the `model` function such as: ```python def model(est, X): - mapping = {'x_'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x_" + str(i): k for i, k in enumerate(X.columns)} new_model = est.model_ - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) ``` 2. The operators/functions in the model are available in [sympy's function set](https://docs.sympy.org/latest/modules/functions/index.html). @@ -92,8 +92,8 @@ If your method uses protected operators (e.g., `my_prot_div(a,b)= a/b if b!=0 el ```python def model(est, X): - ... # any previous pre-processing - new_model = new_model.replace("my_prot_div","/").replace("my_prot_log","log") + ... # any previous pre-processing + new_model = new_model.replace("my_prot_div", "/").replace("my_prot_log", "log") ``` ### CLI methods @@ -146,7 +146,8 @@ In python, one can grab this value as follows: ```python import os -n_cpus = os.environ['OMP_NUM_THREADS'] + +n_cpus = os.environ["OMP_NUM_THREADS"] ``` ### Time Budget @@ -172,18 +173,17 @@ function in `regressor.py` as part of `eval_kwargs`. As an example, one could define the following in `regressor.py`: ```python -def pre_train_fn(est, X, y): +def pre_train_fn(est, X, y): """set max_time in seconds based on length of X.""" - if len(X)<=1000: - max_time = 360 - 1 # 1 second of slack + if len(X) <= 1000: + max_time = 360 - 1 # 1 second of slack else: - max_time = 3600 - 1 # 1 second of slack + max_time = 3600 - 1 # 1 second of slack est.set_params(max_time) + # pass the function to eval_kwargs -eval_kwargs = { - 'pre_train': pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} ``` #### Timeout Handling @@ -195,22 +195,25 @@ Nonetheless, timeout handling looks like this: ```python import signal + class TimeOutException(Exception): pass + def alarm_handler(signum, frame): print(f"raising TimeOutException") raise TimeOutException + def evaluate(est): """model evaluation""" # alarm that sends SIGALRM signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: est.fit() except TimeOutException: - print('evaluate timeout') + print("evaluate timeout") ``` Example approaches are the following: @@ -280,39 +283,45 @@ Here is an example of how to do so: ```python from sklearn.model_selection import GridSearchCV -#from mylibrary import MyMethod + +# from mylibrary import MyMethod from sklearn import linear_model -hyper_params = { 'alpha': (1e-04,0.001,0.01,0.1,1,) } +hyper_params = { + "alpha": ( + 1e-04, + 0.001, + 0.01, + 0.1, + 1, + ) +} # define your base estimator -base_est=linear_model.LassoLars() +base_est = linear_model.LassoLars() # set est to be a GridSearchCV estimator -est = GridSearchCV(estimator=base_est, - param_grid=hyper_params, - cv=5, - refit=True) +est = GridSearchCV(estimator=base_est, param_grid=hyper_params, cv=5, refit=True) # set max time w.r.t. hyper-param tuning num_combos = 5 # five values of alpha -num_cv = 5 # 5 cross-validation splits as above -total_runs = num_combos * num_cv + 1 # +1 because refit=True +num_cv = 5 # 5 cross-validation splits as above +total_runs = num_combos * num_cv + 1 # +1 because refit=True + -def pre_train_fn(est, X, y): +def pre_train_fn(est, X, y): """set max_time in seconds based on length of X & hyper-parameter tuning""" - if len(X)<=1000: + if len(X) <= 1000: # account for hyper-parameter tuning, remove 1 extra second of slack to be sure it will terminate on time - max_time = 360 // total_runs - 1 + max_time = 360 // total_runs - 1 else: max_time = 3600 // total_runs - 1 est.set_params(max_time) + # pass the function eval_kwargs -eval_kwargs = { - 'pre_train': pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} # additional definitions # ... diff --git a/docs/CompetitionGuide2023.md b/docs/CompetitionGuide2023.md index 2362204cd..7c39715ff 100644 --- a/docs/CompetitionGuide2023.md +++ b/docs/CompetitionGuide2023.md @@ -72,7 +72,7 @@ The final score of each competitor will be composed of: The rank will be calculated for each data set independently such that, with N participants, the k-th ranked competitor (k=1 being the best) will be assigned a value of *N - k + 1*. The final score will be the harmonic mean of all of the scores and each participant will be ranked accordingly: ```python -score = 2*n / sum([ (1/acc[i]) + (1/simpl[i]) for i in (1..n)]) +score = 2 * n / sum([(1 / acc[i]) + (1 / simpl[i]) for i in ((1.0).n)]) ``` # Track 2 - Interpretability diff --git a/experiment/analyze.py b/experiment/analyze.py index 8eb7d11ca..d936a71ed 100755 --- a/experiment/analyze.py +++ b/experiment/analyze.py @@ -1,242 +1,377 @@ -import pdb -import pandas as pd +import argparse +import os import subprocess -import numpy as np from glob import glob -import argparse -import os, errno, sys + from joblib import Parallel, delayed from seeds import SEEDS -from yaml import load, Loader +from yaml import Loader, load -#TODO make this script smarter about running jobs. +# TODO make this script smarter about running jobs. # have it check to see whether results for that job exist before # resubmitting. That way the same command can be run multiple times. -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="An analyst for quick ML applications.", add_help=False) - parser.add_argument('DATASET_DIR', type=str, - help='Dataset directory like (pmlb/datasets)') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='LEARNERS',default=None, - type=str, help='Comma-separated list of ML methods to use (should ' - 'correspond to a py file name in methods/)') - parser.add_argument('--local', action='store_true', dest='LOCAL', default=False, - help='Run locally as opposed to on LPC') - parser.add_argument('--slurm', action='store_true', dest='SLURM', default=False, - help='Run on a SLURM scheduler as opposed to on LPC') - parser.add_argument('--noskips', action='store_true', dest='NOSKIPS', default=False, - help='Overwite existing results if found') - parser.add_argument('-skip_tuning', action='store_true', dest='SKIP_TUNE', default=False, - help='Skip tuning step') - parser.add_argument('-A', action='store', dest='A', default='plgsrbench', - help='SLURM account') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', default=False, - help='Specify a symbolic dataset') - parser.add_argument('-tuned',action='store_true', dest='TUNED', default=False, - help='Run tuned version of estimators. Only applies when ml=None') - parser.add_argument('-n_jobs',action='store',dest='N_JOBS',default=1,type=int, - help='Number of parallel jobs') - parser.add_argument('-time_limit',action='store',dest='TIME',default='48:00', - type=str, help='Time limit (hr:min) e.g. 24:00') - parser.add_argument('-seed',action='store',dest='SEED',default=None, - type=int, help='A specific random seed') - parser.add_argument('-n_trials',action='store',dest='N_TRIALS',default=1, - type=int, help='Number of parallel jobs') - parser.add_argument('-label',action='store',dest='LABEL',default='class', - type=str,help='Name of class label column') - parser.add_argument('-results',action='store',dest='RDIR',default='results', - type=str,help='Results directory') - parser.add_argument('-q',action='store',dest='QUEUE', - default='epistasis_long', - type=str,help='LSF queue') - parser.add_argument('-script',action='store',dest='SCRIPT', - default='evaluate_model', - type=str,help='Python script to run') - parser.add_argument('-m',action='store',dest='M',default=16384,type=int, - help='LSF memory request and limit (MB)') - parser.add_argument('-starting_seed',action='store',dest='START_SEED', - default=0,type=int, help='seed position to start with') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-job_limit',action='store',dest='JOB_LIMIT', - default=1000, type=int, - help='Limit number of jobs submitted at once') + description="An analyst for quick ML applications.", add_help=False + ) + parser.add_argument( + "DATASET_DIR", type=str, help="Dataset directory like (pmlb/datasets)" + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="LEARNERS", + default=None, + type=str, + help="Comma-separated list of ML methods to use (should " + "correspond to a py file name in methods/)", + ) + parser.add_argument( + "--local", + action="store_true", + dest="LOCAL", + default=False, + help="Run locally as opposed to on LPC", + ) + parser.add_argument( + "--slurm", + action="store_true", + dest="SLURM", + default=False, + help="Run on a SLURM scheduler as opposed to on LPC", + ) + parser.add_argument( + "--noskips", + action="store_true", + dest="NOSKIPS", + default=False, + help="Overwite existing results if found", + ) + parser.add_argument( + "-skip_tuning", + action="store_true", + dest="SKIP_TUNE", + default=False, + help="Skip tuning step", + ) + parser.add_argument( + "-A", action="store", dest="A", default="plgsrbench", help="SLURM account" + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + default=False, + help="Specify a symbolic dataset", + ) + parser.add_argument( + "-tuned", + action="store_true", + dest="TUNED", + default=False, + help="Run tuned version of estimators. Only applies when ml=None", + ) + parser.add_argument( + "-n_jobs", + action="store", + dest="N_JOBS", + default=1, + type=int, + help="Number of parallel jobs", + ) + parser.add_argument( + "-time_limit", + action="store", + dest="TIME", + default="48:00", + type=str, + help="Time limit (hr:min) e.g. 24:00", + ) + parser.add_argument( + "-seed", + action="store", + dest="SEED", + default=None, + type=int, + help="A specific random seed", + ) + parser.add_argument( + "-n_trials", + action="store", + dest="N_TRIALS", + default=1, + type=int, + help="Number of parallel jobs", + ) + parser.add_argument( + "-label", + action="store", + dest="LABEL", + default="class", + type=str, + help="Name of class label column", + ) + parser.add_argument( + "-results", + action="store", + dest="RDIR", + default="results", + type=str, + help="Results directory", + ) + parser.add_argument( + "-q", + action="store", + dest="QUEUE", + default="epistasis_long", + type=str, + help="LSF queue", + ) + parser.add_argument( + "-script", + action="store", + dest="SCRIPT", + default="evaluate_model", + type=str, + help="Python script to run", + ) + parser.add_argument( + "-m", + action="store", + dest="M", + default=16384, + type=int, + help="LSF memory request and limit (MB)", + ) + parser.add_argument( + "-starting_seed", + action="store", + dest="START_SEED", + default=0, + type=int, + help="seed position to start with", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-job_limit", + action="store", + dest="JOB_LIMIT", + default=1000, + type=int, + help="Limit number of jobs submitted at once", + ) args = parser.parse_args() - - if args.SLURM and args.QUEUE == 'epistasis_long': - print('setting queue to plgrid') - args.QUEUE = 'plgrid' - if args.LEARNERS == None: - prefix = 'methods/tuned/' if args.TUNED else 'methods/' - learners = [ml.split('/')[-1][:-3] for ml in glob(prefix+'*.py') - if not ml.split('/')[-1].startswith('_')] + if args.SLURM and args.QUEUE == "epistasis_long": + print("setting queue to plgrid") + args.QUEUE = "plgrid" + + if args.LEARNERS is None: + prefix = "methods/tuned/" if args.TUNED else "methods/" + learners = [ + ml.split("/")[-1][:-3] + for ml in glob(prefix + "*.py") + if not ml.split("/")[-1].startswith("_") + ] if args.TUNED: - learners = ['tuned.'+ml for ml in learners] + learners = ["tuned." + ml for ml in learners] else: - learners = [ml for ml in args.LEARNERS.split(',')] # learners - print('learners:',learners) + learners = [ml for ml in args.LEARNERS.split(",")] # learners + print("learners:", learners) - print('dataset directory:',args.DATASET_DIR) + print("dataset directory:", args.DATASET_DIR) - if args.DATASET_DIR.endswith('.tsv.gz'): + if args.DATASET_DIR.endswith(".tsv.gz"): datasets = [args.DATASET_DIR] - elif args.DATASET_DIR.endswith('*'): - print('capturing glob',args.DATASET_DIR+'/*.tsv.gz') - datasets = glob(args.DATASET_DIR+'*/*.tsv.gz') + elif args.DATASET_DIR.endswith("*"): + print("capturing glob", args.DATASET_DIR + "/*.tsv.gz") + datasets = glob(args.DATASET_DIR + "*/*.tsv.gz") else: - datasets = glob(args.DATASET_DIR+'/*/*.tsv.gz') - print('found',len(datasets),'datasets') + datasets = glob(args.DATASET_DIR + "/*/*.tsv.gz") + print("found", len(datasets), "datasets") ##################################################### ## look for existing jobs ######################## current_jobs = [] if args.SLURM: - res = subprocess.check_output(['squeue -o "%j"'],shell=True) - current_jobs = res.decode().split('\n') + res = subprocess.check_output(['squeue -o "%j"'], shell=True) + current_jobs = res.decode().split("\n") elif not args.LOCAL: - res = subprocess.check_output(['bjobs -o "JOB_NAME" -noheader'],shell=True) - current_jobs = res.decode().split('\n') + res = subprocess.check_output(['bjobs -o "JOB_NAME" -noheader'], shell=True) + current_jobs = res.decode().split("\n") # current_jobs = ['_'.join(cj.split('_')[:-1]) for cj in current_jobs] # write run commands jobs_w_results = [] - jobs_wout_results = [] - suffix = ('.json.updated' if args.SCRIPT=='assess_symbolic_model' else - '.json') + jobs_wout_results = [] + suffix = ".json.updated" if args.SCRIPT == "assess_symbolic_model" else ".json" queued_jobs = [] all_commands = [] - job_info=[] - for t in range(args.START_SEED, args.START_SEED+args.N_TRIALS): + job_info = [] + for t in range(args.START_SEED, args.START_SEED + args.N_TRIALS): # random_state = np.random.randint(2**15-1) - if args.SEED and args.N_TRIALS==1: + if args.SEED and args.N_TRIALS == 1: random_state = args.SEED else: random_state = SEEDS[t] # print('random_seed:',random_state) for dataset in datasets: - if (not args.SYM_DATA - and any([n in dataset for n in ['feynman', 'strogatz']]) - ): + if not args.SYM_DATA and any( + [n in dataset for n in ["feynman", "strogatz"]] + ): continue # grab regression datasets metadata = load( - open('/'.join(dataset.split('/')[:-1])+'/metadata.yaml','r'), - Loader=Loader) - if metadata['task'] != 'regression': + open("/".join(dataset.split("/")[:-1]) + "/metadata.yaml", "r"), + Loader=Loader, + ) + if metadata["task"] != "regression": continue - - dataname = dataset.split('/')[-1].split('.tsv.gz')[0] - results_path = '/'.join([args.RDIR, dataname]) + '/' + + dataname = dataset.split("/")[-1].split(".tsv.gz")[0] + results_path = "/".join([args.RDIR, dataname]) + "/" if not os.path.exists(results_path): os.makedirs(results_path) - + for ml in learners: - save_file = (results_path + '/' + dataname + '_' + ml + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataname + "_" + ml + "_" + str(random_state) + ) # if updated, check if json file exists (required) - if ('updated' in suffix - or args.SCRIPT.startswith('fix_')): - if not os.path.exists(save_file+'.json'): - jobs_wout_results.append([save_file,'json result DNE']) + if "updated" in suffix or args.SCRIPT.startswith("fix_"): + if not os.path.exists(save_file + ".json"): + jobs_wout_results.append([save_file, "json result DNE"]) continue if not args.NOSKIPS: - save_file = (results_path + '/' + dataname + '_' + ml + '_' - + str(random_state)) + save_file = ( + results_path + + "/" + + dataname + + "_" + + ml + + "_" + + str(random_state) + ) if args.Y_NOISE > 0: - save_file += '_target-noise'+str(args.Y_NOISE) + save_file += "_target-noise" + str(args.Y_NOISE) if args.X_NOISE > 0: - save_file += '_feature-noise'+str(args.X_NOISE) + save_file += "_feature-noise" + str(args.X_NOISE) # check if there is already a result for this experiment - if (os.path.exists(save_file+suffix) - and args.SCRIPT != 'fix_aifeynman_model_size'): - jobs_w_results.append([save_file,'exists']) + if ( + os.path.exists(save_file + suffix) + and args.SCRIPT != "fix_aifeynman_model_size" + ): + jobs_w_results.append([save_file, "exists"]) continue # check if there is already a queued job for this experiment - if save_file.split('/')[-1] in current_jobs: - queued_jobs.append([save_file,'queued']) + if save_file.split("/")[-1] in current_jobs: + queued_jobs.append([save_file, "queued"]) continue - - all_commands.append('python {SCRIPT}.py ' - '{DATASET}' - ' -ml {ML}' - ' -results_path {RDIR}' - ' -seed {RS} ' - ' -target_noise {TN} ' - ' -feature_noise {FN} ' - '{TEST} {SYM_DATA} {SKIP_TUNE}'.format( - SCRIPT=args.SCRIPT, - ML=ml, - DATASET=dataset, - RDIR=results_path, - RS=random_state, - TN=args.Y_NOISE, - FN=args.X_NOISE, - TEST=('-test' if args.TEST - else ''), - SYM_DATA=('-sym_data' if args.SYM_DATA - else ''), - SKIP_TUNE=('-skip_tuning' if - args.SKIP_TUNE else '') - ) - ) - job_info.append({'ml':ml, - 'dataset':dataname, - 'seed':str(random_state), - 'results_path':results_path, - 'target_noise':args.Y_NOISE - }) + all_commands.append( + "python {SCRIPT}.py " + "{DATASET}" + " -ml {ML}" + " -results_path {RDIR}" + " -seed {RS} " + " -target_noise {TN} " + " -feature_noise {FN} " + "{TEST} {SYM_DATA} {SKIP_TUNE}".format( + SCRIPT=args.SCRIPT, + ML=ml, + DATASET=dataset, + RDIR=results_path, + RS=random_state, + TN=args.Y_NOISE, + FN=args.X_NOISE, + TEST=("-test" if args.TEST else ""), + SYM_DATA=("-sym_data" if args.SYM_DATA else ""), + SKIP_TUNE=("-skip_tuning" if args.SKIP_TUNE else ""), + ) + ) + job_info.append( + { + "ml": ml, + "dataset": dataname, + "seed": str(random_state), + "results_path": results_path, + "target_noise": args.Y_NOISE, + } + ) if len(all_commands) > args.JOB_LIMIT: - print('shaving jobs down to job limit ({})'.format(args.JOB_LIMIT)) - all_commands = all_commands[:args.JOB_LIMIT] + print("shaving jobs down to job limit ({})".format(args.JOB_LIMIT)) + all_commands = all_commands[: args.JOB_LIMIT] if not args.NOSKIPS: - print('skipped',len(jobs_w_results),'jobs with results. Override with --noskips.') - print('skipped',len(jobs_wout_results),'jobs without results. Override with --noskips.') - print('skipped',len(queued_jobs),'queued jobs. Override with --noskips.') - print('submitting',len(all_commands),'jobs...') + print( + "skipped", + len(jobs_w_results), + "jobs with results. Override with --noskips.", + ) + print( + "skipped", + len(jobs_wout_results), + "jobs without results. Override with --noskips.", + ) + print("skipped", len(queued_jobs), "queued jobs. Override with --noskips.") + print("submitting", len(all_commands), "jobs...") if args.LOCAL: - # run locally - Parallel(n_jobs=args.N_JOBS)(delayed(os.system)(run_cmd) - for run_cmd in all_commands) - #for run_cmd in all_commands: + # run locally + Parallel(n_jobs=args.N_JOBS)( + delayed(os.system)(run_cmd) for run_cmd in all_commands + ) + # for run_cmd in all_commands: # print(run_cmd) # os.system(run_cmd) else: # sbatch - for i,run_cmd in enumerate(all_commands): - job_name = '_'.join([ - job_info[i]['dataset'], - job_info[i]['ml'], - job_info[i]['seed'], - args.SCRIPT - ]) - if args.Y_NOISE>0: - job_name += '_target-noise'+str(args.Y_NOISE) - if args.X_NOISE>0: - job_name += '_feature-noise'+str(args.X_NOISE) - out_file = (job_info[i]['results_path'] - + job_name - + '.%J.out') - error_file = out_file[:-4] + '.err' - + for i, run_cmd in enumerate(all_commands): + job_name = "_".join( + [ + job_info[i]["dataset"], + job_info[i]["ml"], + job_info[i]["seed"], + args.SCRIPT, + ] + ) + if args.Y_NOISE > 0: + job_name += "_target-noise" + str(args.Y_NOISE) + if args.X_NOISE > 0: + job_name += "_feature-noise" + str(args.X_NOISE) + out_file = job_info[i]["results_path"] + job_name + ".%J.out" + error_file = out_file[:-4] + ".err" + if args.SLURM: - batch_script = \ -"""#!/usr/bin/bash + batch_script = """#!/usr/bin/bash #SBATCH -o {OUT_FILE} #SBATCH --error={ERR_FILE} #SBATCH -N 1 @@ -251,49 +386,52 @@ {cmd} """.format( - OUT_FILE=out_file, - ERR_FILE=error_file, - JOB_NAME=job_name, - QUEUE=args.QUEUE, - A=args.A, - N_CORES=args.N_JOBS, - M=args.M, - cmd=run_cmd, - TIME=args.TIME - ) - with open('tmp_script','w') as f: - f.write(batch_script) + OUT_FILE=out_file, + ERR_FILE=error_file, + JOB_NAME=job_name, + QUEUE=args.QUEUE, + A=args.A, + N_CORES=args.N_JOBS, + M=args.M, + cmd=run_cmd, + TIME=args.TIME, + ) + with open("tmp_script", "w") as f: + f.write(batch_script) - # print(batch_script) - print(job_name) - sbatch_response = subprocess.check_output(['sbatch tmp_script'], - shell=True).decode() # submit jobs - print(sbatch_response) + # print(batch_script) + print(job_name) + sbatch_response = subprocess.check_output( + ["sbatch tmp_script"], shell=True + ).decode() # submit jobs + print(sbatch_response) - else: # LPC + else: # LPC # activate srbench env, load modules # pre_run_cmds = ["conda activate srbench", # "source lpc_modules.sh"] # run_cmd = '; '.join(pre_run_cmds + [run_cmd]) - bsub_cmd = ('bsub -o {OUT_FILE} ' - '-e {ERR_FILE} ' - '-n {N_CORES} ' - '-J {JOB_NAME} ' - '-q {QUEUE} ' - '-R "span[hosts=1] rusage[mem={M}]" ' - '-W {TIME} ' - '-M {M} ').format( - OUT_FILE=out_file, - ERR_FILE=error_file, - JOB_NAME=job_name, - QUEUE=args.QUEUE, - N_CORES=args.N_JOBS, - M=args.M, - TIME=args.TIME - ) - - bsub_cmd += '"' + run_cmd + '"' + bsub_cmd = ( + "bsub -o {OUT_FILE} " + "-e {ERR_FILE} " + "-n {N_CORES} " + "-J {JOB_NAME} " + "-q {QUEUE} " + '-R "span[hosts=1] rusage[mem={M}]" ' + "-W {TIME} " + "-M {M} " + ).format( + OUT_FILE=out_file, + ERR_FILE=error_file, + JOB_NAME=job_name, + QUEUE=args.QUEUE, + N_CORES=args.N_JOBS, + M=args.M, + TIME=args.TIME, + ) + + bsub_cmd += '"' + run_cmd + '"' print(bsub_cmd) - os.system(bsub_cmd) # submit jobs + os.system(bsub_cmd) # submit jobs - print('Finished submitting',len(all_commands),'jobs.') + print("Finished submitting", len(all_commands), "jobs.") diff --git a/experiment/assess_symbolic_model.py b/experiment/assess_symbolic_model.py index 7ac429cf2..1f379c517 100644 --- a/experiment/assess_symbolic_model.py +++ b/experiment/assess_symbolic_model.py @@ -1,158 +1,204 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingGridSearchCV -from sklearn.model_selection import GridSearchCV, KFold, train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb +from sklearn.experimental import enable_halving_search_cv # noqa import numpy as np import json import os -import inspect -from utils import jsonify -from symbolic_utils import (clean_pred_model,get_sym_model,round_floats, - complexity, rewrite_AIFeynman_model_size) +from utils import jsonify +from symbolic_utils import ( + clean_pred_model, + get_sym_model, + round_floats, + complexity, + rewrite_AIFeynman_model_size, +) from sympy import simplify +import argparse + -def save(r,save_file): - print('saving...') - with open(save_file + '.updated', 'w') as out: +def save(r, save_file): + print("saving...") + with open(save_file + ".updated", "w") as out: json.dump(jsonify(r), out, indent=4) + def assess_symbolic_model_from_file(json_file, dataset): - - print('looking for:',json_file) + + print("looking for:", json_file) if os.path.exists(json_file): - r = json.load(open(json_file, 'r')) + r = json.load(open(json_file, "r")) else: - raise FileNotFoundError(json_file+' not found') + raise FileNotFoundError(json_file + " not found") - est_name = r['algorithm'] + est_name = r["algorithm"] true_model = get_sym_model(dataset, return_str=False) - r['true_model'] = str(true_model) - raw_model = r['symbolic_model'] + r["true_model"] = str(true_model) + raw_model = r["symbolic_model"] - if 'AIFeynman' in est_name: + if "AIFeynman" in est_name: # correct model size - r['model_size'] = rewrite_AIFeynman_model_size(raw_model) + r["model_size"] = rewrite_AIFeynman_model_size(raw_model) try: cleaned_model = clean_pred_model(raw_model, dataset, est_name) - r['simplified_symbolic_model'] = str(cleaned_model) - r['simplified_complexity'] = complexity(cleaned_model) - + r["simplified_symbolic_model"] = str(cleaned_model) + r["simplified_complexity"] = complexity(cleaned_model) + # save simplified model in case this is as far as we get save(r, json_file) # if the model is somewhat accurate, check and see if it # is an exact symbolic match - if r['r2_test'] > 0.5: + if r["r2_test"] > 0.5: sym_diff = round_floats(true_model - cleaned_model) - sym_frac = round_floats(cleaned_model/true_model) - print('sym_diff:',sym_diff) - print('sym_frac:',sym_frac) + sym_frac = round_floats(cleaned_model / true_model) + print("sym_diff:", sym_diff) + print("sym_frac:", sym_frac) # check if we can skip simplification - + if not sym_diff.is_constant() or sym_frac.is_constant(): sym_diff = round_floats(simplify(sym_diff, ratio=1)) - print('simplified sym_diff:',sym_diff) - r['symbolic_error'] = str(sym_diff) - r['symbolic_fraction'] = str(sym_frac) - r['symbolic_error_is_zero'] = str(sym_diff) == '0' - r['symbolic_error_is_constant'] = sym_diff.is_constant() - r['symbolic_fraction_is_constant'] = sym_frac.is_constant() + print("simplified sym_diff:", sym_diff) + r["symbolic_error"] = str(sym_diff) + r["symbolic_fraction"] = str(sym_frac) + r["symbolic_error_is_zero"] = str(sym_diff) == "0" + r["symbolic_error_is_constant"] = sym_diff.is_constant() + r["symbolic_fraction_is_constant"] = sym_frac.is_constant() else: raise ValueError("Model isnt accurate enough to check") except Exception as e: - r['sympy_exception'] = str(e) - if 'symbolic_error_is_zero' not in r.keys(): - r['symbolic_error_is_zero'] = False - if 'symbolic_error_is_constant' not in r.keys(): - r['symbolic_error_is_constant'] = False - if 'symbolic_fraction_is_constant' not in r.keys(): - r['symbolic_fraction_is_constant'] = False + r["sympy_exception"] = str(e) + if "symbolic_error_is_zero" not in r.keys(): + r["symbolic_error_is_zero"] = False + if "symbolic_error_is_constant" not in r.keys(): + r["symbolic_error_is_constant"] = False + if "symbolic_fraction_is_constant" not in r.keys(): + r["symbolic_fraction_is_constant"] = False print(json.dumps(r, indent=4)) save(r, json_file) - print('done.') + print("done.") + -def assess_symbolic_model(dataset, results_path, random_state, est_name, - target_noise=0.0, feature_noise=0.0): +def assess_symbolic_model( + dataset, results_path, random_state, est_name, target_noise=0.0, feature_noise=0.0 +): - print(40*'=','Assessing '+est_name+' model for ',dataset,40*'=',sep='\n') + print( + 40 * "=", "Assessing " + est_name + " model for ", dataset, 40 * "=", sep="\n" + ) np.random.seed(random_state) ################################################# # load json file ################################################# - dataset_name = dataset.split('/')[-1][:-7] + dataset_name = dataset.split("/")[-1][:-7] - save_file = (results_path + '/' + dataset_name + '_' + est_name + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataset_name + "_" + est_name + "_" + str(random_state) + ) if target_noise > 0: - save_file += '_target-noise'+str(target_noise) + save_file += "_target-noise" + str(target_noise) if feature_noise > 0: - save_file += '_feature-noise'+str(feature_noise) + save_file += "_feature-noise" + str(feature_noise) + + assess_symbolic_model_from_file(save_file + ".json", dataset) - assess_symbolic_model_from_file(save_file+'.json', dataset) ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', - help='Use symbolic dataset settings') - parser.add_argument('-json_file',action='store', dest='JSON_FILE', type=str, - default='',help='JSON results file') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + help="Use symbolic dataset settings", + ) + parser.add_argument( + "-json_file", + action="store", + dest="JSON_FILE", + type=str, + default="", + help="JSON results file", + ) args = parser.parse_args() print(args.__dict__) - if args.JSON_FILE != '': + if args.JSON_FILE != "": assess_symbolic_model_from_file(args.JSON_FILE, args.INPUT_FILE) else: - assess_symbolic_model(args.INPUT_FILE, args.RDIR, args.RANDOM_STATE, - args.ALG, target_noise=args.Y_NOISE, - feature_noise=args.X_NOISE) + assess_symbolic_model( + args.INPUT_FILE, + args.RDIR, + args.RANDOM_STATE, + args.ALG, + target_noise=args.Y_NOISE, + feature_noise=args.X_NOISE, + ) diff --git a/experiment/conftest.py b/experiment/conftest.py index e336f7c25..8d0b6fe77 100644 --- a/experiment/conftest.py +++ b/experiment/conftest.py @@ -1,12 +1,13 @@ -# configures pytest to allow CLI arguments. +# configures pytest to allow CLI arguments. + def pytest_addoption(parser): - parser.addoption("--ml", action="store", default="AFPRegressor") + parser.addoption("--ml", action="store", default="AFPRegressor") def pytest_generate_tests(metafunc): # This is called for every test. Only get/set command line arguments # if the argument is specified in the list of test "fixturenames". option_value = metafunc.config.option.ml - if 'ml' in metafunc.fixturenames and option_value is not None: + if "ml" in metafunc.fixturenames and option_value is not None: metafunc.parametrize("ml", [option_value]) diff --git a/experiment/docs/source/conf.py b/experiment/docs/source/conf.py index f0c2f3fa9..671b2620d 100755 --- a/experiment/docs/source/conf.py +++ b/experiment/docs/source/conf.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. @@ -19,14 +21,14 @@ # -- Project information ----------------------------------------------------- -project = 'Symbolic Regression Benchmark' -copyright = '2019, William La Cava, Patryk Orzechowski, Jason H. Moore' -author = 'William La Cava, Patryk Orzechowski, Jason H. Moore' +project = "Symbolic Regression Benchmark" +copyright = "2019, William La Cava, Patryk Orzechowski, Jason H. Moore" +author = "William La Cava, Patryk Orzechowski, Jason H. Moore" # The short X.Y version -version = '' +version = "" # The full version, including alpha/beta/rc tags -release = '0.0' +release = "0.0" # -- General configuration --------------------------------------------------- @@ -39,26 +41,26 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -70,10 +72,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns: list[str] = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -81,7 +83,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -92,7 +94,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -108,24 +110,21 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'SymbolicRegressionBenchmarkdoc' +htmlhelp_basename = "SymbolicRegressionBenchmarkdoc" # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { +latex_elements: dict[str, str] = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -135,8 +134,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'SymbolicRegressionBenchmark.tex', 'Symbolic Regression Benchmark Documentation', - 'William La Cava, Patryk Orzechowski, Jason H. Moore', 'manual'), + ( + master_doc, + "SymbolicRegressionBenchmark.tex", + "Symbolic Regression Benchmark Documentation", + "William La Cava, Patryk Orzechowski, Jason H. Moore", + "manual", + ), ] @@ -145,8 +149,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'symbolicregressionbenchmark', 'Symbolic Regression Benchmark Documentation', - [author], 1) + ( + master_doc, + "symbolicregressionbenchmark", + "Symbolic Regression Benchmark Documentation", + [author], + 1, + ) ] @@ -156,9 +165,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'SymbolicRegressionBenchmark', 'Symbolic Regression Benchmark Documentation', - author, 'SymbolicRegressionBenchmark', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "SymbolicRegressionBenchmark", + "Symbolic Regression Benchmark Documentation", + author, + "SymbolicRegressionBenchmark", + "One line description of project.", + "Miscellaneous", + ), ] @@ -167,4 +182,4 @@ # -- Options for todo extension ---------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True \ No newline at end of file +todo_include_todos = True diff --git a/experiment/evaluate_model.py b/experiment/evaluate_model.py index e82c7be2a..84f8ef65f 100755 --- a/experiment/evaluate_model.py +++ b/experiment/evaluate_model.py @@ -1,38 +1,36 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -from sklearn.model_selection import train_test_split -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb -import numpy as np +import argparse +import importlib +import inspect import json import os -import inspect -from utils import jsonify -from symbolic_utils import complexity, get_sym_model, get_sympy_model +import signal +import time +import numpy as np +import pandas as pd from metrics.evaluation import simplicity +from read_file import read_file +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler +from symbolic_utils import complexity, get_sym_model, get_sympy_model +from utils import jsonify + -import signal class TimeOutException(Exception): pass + def alarm_handler(signum, frame): - print(f"raising TimeOutException") + print("raising TimeOutException") raise TimeOutException + def set_env_vars(n_jobs): - os.environ['OMP_NUM_THREADS'] = n_jobs - os.environ['OPENBLAS_NUM_THREADS'] = n_jobs - os.environ['MKL_NUM_THREADS'] = n_jobs + os.environ["OMP_NUM_THREADS"] = n_jobs + os.environ["OPENBLAS_NUM_THREADS"] = n_jobs + os.environ["MKL_NUM_THREADS"] = n_jobs + def evaluate_model( *, @@ -47,7 +45,6 @@ def evaluate_model( target_noise=0.0, feature_noise=0.0, fit_time_limit=3600, - ########## # valid options for eval_kwargs ########## @@ -60,32 +57,27 @@ def evaluate_model( use_dataframe=True, ): - print(40*'=','Evaluating '+est_name+' on ',dataset,40*'=',sep='\n') + print(40 * "=", "Evaluating " + est_name + " on ", dataset, 40 * "=", sep="\n") np.random.seed(random_state) - if hasattr(est, 'random_state'): + if hasattr(est, "random_state"): est.random_state = random_state - features, labels, feature_names = read_file( - dataset, - use_dataframe=use_dataframe - ) - print('feature_names:',feature_names) + features, labels, feature_names = read_file(dataset, use_dataframe=use_dataframe) + print("feature_names:", feature_names) if sym_data: true_model = get_sym_model(dataset) # generate train/test split - X_train, X_test, y_train, y_test = train_test_split(features, labels, - train_size=0.75, - test_size=0.25, - random_state=random_state) + X_train, X_test, y_train, y_test = train_test_split( + features, labels, train_size=0.75, test_size=0.25, random_state=random_state + ) - # if dataset is large, subsample the training set + # if dataset is large, subsample the training set if max_train_samples > 0 and len(y_train) > max_train_samples: - print('subsampling training data from',len(X_train), - 'to',max_train_samples) - sample_idx = np.random.choice(np.arange(len(X_train)), - size=max_train_samples, - replace=False) + print("subsampling training data from", len(X_train), "to", max_train_samples) + sample_idx = np.random.choice( + np.arange(len(X_train)), size=max_train_samples, replace=False + ) y_train = y_train[sample_idx] if isinstance(X_train, pd.DataFrame): X_train = X_train.iloc[sample_idx] @@ -94,47 +86,50 @@ def evaluate_model( # scale and normalize the data if scale_x: - print('scaling X') - sc_X = StandardScaler() + print("scaling X") + sc_X = StandardScaler() X_train_scaled = sc_X.fit_transform(X_train) X_test_scaled = sc_X.transform(X_test) if use_dataframe: - X_train_scaled = pd.DataFrame(X_train_scaled, - columns=feature_names) - X_test_scaled = pd.DataFrame(X_test_scaled, - columns=feature_names) + X_train_scaled = pd.DataFrame(X_train_scaled, columns=feature_names) + X_test_scaled = pd.DataFrame(X_test_scaled, columns=feature_names) else: X_train_scaled = X_train X_test_scaled = X_test - if scale_y: - print('scaling y') + print("scaling y") sc_y = StandardScaler() - y_train_scaled = sc_y.fit_transform(y_train.reshape(-1,1)).flatten() + y_train_scaled = sc_y.fit_transform(y_train.reshape(-1, 1)).flatten() else: y_train_scaled = y_train - - ################################################## + ################################################## # noise - ################################################## + ################################################## if target_noise > 0: - print('adding',target_noise,'noise to target') - y_train_scaled += np.random.normal(0, - target_noise*np.sqrt(np.mean(np.square(y_train_scaled))), - size=len(y_train_scaled)) + print("adding", target_noise, "noise to target") + y_train_scaled += np.random.normal( + 0, + target_noise * np.sqrt(np.mean(np.square(y_train_scaled))), + size=len(y_train_scaled), + ) # add noise to the features if feature_noise > 0: - print('adding',target_noise,'noise to features') - X_train_scaled = np.array([x - + np.random.normal(0, feature_noise*np.sqrt(np.mean(np.square(x))), - size=len(x)) - for x in X_train_scaled.T]).T + print("adding", target_noise, "noise to features") + X_train_scaled = np.array( + [ + x + + np.random.normal( + 0, feature_noise * np.sqrt(np.mean(np.square(x))), size=len(x) + ) + for x in X_train_scaled.T + ] + ).T - ################################################## + ################################################## # run any method-specific pre_train routines - ################################################## + ################################################## if pre_train: pre_train(est, X_train_scaled, y_train_scaled) @@ -142,42 +137,42 @@ def evaluate_model( if test and len(test_params) != 0: est.set_params(**test_params) - ################################################## + ################################################## # Fit models - ################################################## - if test and fit_time_limit > 360: # lets give them 6 minutes + ################################################## + if test and fit_time_limit > 360: # lets give them 6 minutes fit_time_limit = 360 MAXTIME = fit_time_limit if len(y_train) > 1000 and not test and MAXTIME < 36000: MAXTIME = 36000 - print('max time:', MAXTIME) + print("max time:", MAXTIME) - if not use_dataframe: + if not use_dataframe: assert isinstance(X_train_scaled, np.ndarray) assert isinstance(X_test_scaled, np.ndarray) - print('X_train:',type(X_train_scaled),X_train_scaled.shape) - print('y_train:',y_train_scaled.shape) - print('training',est) + print("X_train:", type(X_train_scaled), X_train_scaled.shape) + print("y_train:", y_train_scaled.shape) + print("training", est) - if hasattr(est, 'max_time'): + if hasattr(est, "max_time"): est.max_time = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'timeout_in_seconds'): + print("max time set:", MAXTIME) + elif hasattr(est, "timeout_in_seconds"): est.timeout_in_seconds = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'timeout'): + print("max time set:", MAXTIME) + elif hasattr(est, "timeout"): est.timeout = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'stop_time'): + print("max time set:", MAXTIME) + elif hasattr(est, "stop_time"): est.stop_time = MAXTIME - print('max time set:', MAXTIME) - elif hasattr(est, 'time_limit'): + print("max time set:", MAXTIME) + elif hasattr(est, "time_limit"): est.time_limit = MAXTIME - print('max time set:', MAXTIME) + print("max time set:", MAXTIME) else: - print('max time not set. Program will be killed if execution takes too long') + print("max time not set. Program will be killed if execution takes too long") signal.signal(signal.SIGALRM, alarm_handler) alarm_grace = 360 if test else 600 @@ -187,79 +182,79 @@ def evaluate_model( try: est.fit(X_train_scaled, y_train_scaled) except TimeOutException: - print('=' * 80) - print('WARNING: fitting timed out. If the program does not handle SIGALRM, it will be killed') - print('=' * 80) + print("=" * 80) + print( + "WARNING: fitting timed out. If the program does not handle SIGALRM, it will be killed" + ) + print("=" * 80) finally: signal.alarm(0) time_time = time.time() - t0t - print('Training time measure:', time_time) + print("Training time measure:", time_time) - if 'geneticengine' in est_name: + if "geneticengine" in est_name: est._is_fitted = True - + ################################################## # store results ################################################## - dataset_name = dataset.split('/')[-1].split('.')[0] + dataset_name = dataset.split("/")[-1].split(".")[0] results = { - 'dataset':dataset_name, - 'algorithm':est_name, - 'params':jsonify(est.get_params()), - 'random_state':random_state, - 'time_time': time_time, + "dataset": dataset_name, + "algorithm": est_name, + "params": jsonify(est.get_params()), + "random_state": random_state, + "time_time": time_time, } if sym_data: - results['true_model'] = true_model + results["true_model"] = true_model # get the final symbolic model as a string - print('fitted est:',est) + print("fitted est:", est) if model is None: - results['symbolic_model'] = 'not implemented' - elif 'X' in inspect.signature(model).parameters.keys(): + results["symbolic_model"] = "not implemented" + elif "X" in inspect.signature(model).parameters.keys(): if not isinstance(X_train_scaled, pd.DataFrame): - X_df = pd.DataFrame(X_train_scaled, - columns=feature_names) + X_df = pd.DataFrame(X_train_scaled, columns=feature_names) else: X_df = X_train_scaled - results['symbolic_model'] = model(est, X_df) + results["symbolic_model"] = model(est, X_df) else: - results['symbolic_model'] = model(est) - print('symbolic model:',results['symbolic_model']) + results["symbolic_model"] = model(est) + print("symbolic model:", results["symbolic_model"]) ################################################## # scores ################################################## - for fold, target, X in [ - ['train', y_train, X_train_scaled], - ['test', y_test, X_test_scaled] - ]: - - y_pred = np.asarray(est.predict(X)).reshape(-1,1) + for fold, target, X in [ + ["train", y_train, X_train_scaled], + ["test", y_test, X_test_scaled], + ]: + y_pred = np.asarray(est.predict(X)).reshape(-1, 1) if scale_y: y_pred = sc_y.inverse_transform(y_pred) - for score, scorer in [('mse',mean_squared_error), - ('mae',mean_absolute_error), - ('r2', r2_score) - ]: - results[score + '_' + fold] = scorer(target, y_pred) - - if results['symbolic_model'] != 'not implemented': - results['simplicity'] = simplicity(results['symbolic_model'], feature_names) + for score, scorer in [ + ("mse", mean_squared_error), + ("mae", mean_absolute_error), + ("r2", r2_score), + ]: + results[score + "_" + fold] = scorer(target, y_pred) + + if results["symbolic_model"] != "not implemented": + results["simplicity"] = simplicity(results["symbolic_model"], feature_names) else: - results['simplicity'] = np.nan + results["simplicity"] = np.nan def sympy_complexity(estimator): sympy_str = None if model is None: return -1 - if 'X' in inspect.signature(model).parameters.keys(): + if "X" in inspect.signature(model).parameters.keys(): if not isinstance(X_train_scaled, pd.DataFrame): - X_df = pd.DataFrame(X_train_scaled, - columns=feature_names) + X_df = pd.DataFrame(X_train_scaled, columns=feature_names) else: X_df = X_train_scaled sympy_str = model(estimator, X_df) @@ -276,101 +271,156 @@ def sympy_complexity(estimator): return -1 cplx = sympy_complexity(est) - results['complexity_function'] = 'sympy' + results["complexity_function"] = "sympy" - if cplx == -1 and ('complexity' in dir(algorithm) and algorithm.complexity is not None): + if cplx == -1 and ( + "complexity" in dir(algorithm) and algorithm.complexity is not None + ): cplx = algorithm.complexity(est) - results['complexity_function'] = 'user_defined' + results["complexity_function"] = "user_defined" - results['model_size'] = cplx - results['target_noise'] = target_noise - results['feature_noise'] = feature_noise + results["model_size"] = cplx + results["target_noise"] = target_noise + results["feature_noise"] = feature_noise ################################################## # write to file ################################################## - print('results:') + print("results:") print(json.dumps(results, indent=4)) - print('---') + print("---") if not os.path.exists(results_path): os.makedirs(results_path) save_file = os.path.join( - results_path, - '_'.join([dataset_name, est_name, str(random_state)]) + results_path, "_".join([dataset_name, est_name, str(random_state)]) ) - print('save_file:',save_file) + print("save_file:", save_file) - with open(save_file + '.json', 'w') as out: + with open(save_file + ".json", "w") as out: json.dump(jsonify(results), out, indent=4) - return save_file + '.json' + return save_file + ".json" + ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-n_jobs',action='store', type=str, default='4', - help='number of cores available') - parser.add_argument('-max_samples',action='store', type=int, default=0, - help='number of training samples') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', - help='Use symbolic dataset settings') - parser.add_argument('-skip_tuning',action='store_true', dest='SKIP_TUNE', - default=False, help='Dont tune the estimator') - parser.add_argument('-fit_time_limit',action='store',dest='FITTIME',default=3600, - type=int, help='Fit time limit (seconds) e.g. 3600 (1 hour). This is ' - 'the maximum time for the fit method, not the job, make sure job time ' - 'limit is greater than this.') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-n_jobs", + action="store", + type=str, + default="4", + help="number of cores available", + ) + parser.add_argument( + "-max_samples", + action="store", + type=int, + default=0, + help="number of training samples", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", action="store_true", help="Use symbolic dataset settings" + ) + parser.add_argument( + "-skip_tuning", + action="store_true", + dest="SKIP_TUNE", + default=False, + help="Dont tune the estimator", + ) + parser.add_argument( + "-fit_time_limit", + action="store", + dest="FITTIME", + default=3600, + type=int, + help="Fit time limit (seconds) e.g. 3600 (1 hour). This is " + "the maximum time for the fit method, not the job, make sure job time " + "limit is greater than this.", + ) args = parser.parse_args() set_env_vars(args.n_jobs) - # import algorithm - print('import from','methods.'+args.ALG+'.regressor') - algorithm = importlib.__import__('methods.'+args.ALG+'.regressor', - globals(), - locals(), - ['*'] - ) + # import algorithm + print("import from", "methods." + args.ALG + ".regressor") + algorithm = importlib.__import__( + "methods." + args.ALG + ".regressor", globals(), locals(), ["*"] + ) - print('algorithm:',algorithm.est) + print("algorithm:", algorithm.est) # optional keyword arguments passed to evaluate - eval_kwargs, test_params = {},{} - if 'eval_kwargs' in dir(algorithm): + eval_kwargs = {} + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs if args.max_samples != 0: - eval_kwargs['max_train_samples'] = args.max_samples + eval_kwargs["max_train_samples"] = args.max_samples evaluate_model( dataset=args.INPUT_FILE, @@ -384,5 +434,5 @@ def sympy_complexity(estimator): fit_time_limit=args.FITTIME, target_noise=args.Y_NOISE, feature_noise=args.X_NOISE, - **eval_kwargs + **eval_kwargs, ) diff --git a/experiment/fix_aifeynman_model_size.py b/experiment/fix_aifeynman_model_size.py index ccc1d8521..592151566 100644 --- a/experiment/fix_aifeynman_model_size.py +++ b/experiment/fix_aifeynman_model_size.py @@ -1,114 +1,144 @@ -import sys -import itertools -import pandas as pd -from sklearn.base import clone -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingGridSearchCV -from sklearn.model_selection import GridSearchCV, KFold, train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score -import warnings -import time -from tempfile import mkdtemp -from shutil import rmtree -from joblib import Memory -from read_file import read_file -import pdb +from sklearn.experimental import enable_halving_search_cv # noqa import numpy as np import json import os -import inspect -from utils import jsonify -from symbolic_utils import (clean_pred_model,get_sym_model,round_floats, - complexity, rewrite_AIFeynman_model_size) -from sympy import simplify +from utils import jsonify +from symbolic_utils import rewrite_AIFeynman_model_size +import argparse + -def save(r,save_file): +def save(r, save_file): print(json.dumps(r, indent=4)) - print('saving...') - with open(save_file + '.json', 'w') as out: + print("saving...") + with open(save_file + ".json", "w") as out: json.dump(jsonify(r), out, indent=4) -def fix_aifeynman_model_size(dataset, results_path, random_state, est_name, - target_noise=0.0, feature_noise=0.0): +def fix_aifeynman_model_size( + dataset, results_path, random_state, est_name, target_noise=0.0, feature_noise=0.0 +): - print(40*'=','Assessing '+est_name+' model for ',dataset,40*'=',sep='\n') + print( + 40 * "=", "Assessing " + est_name + " model for ", dataset, 40 * "=", sep="\n" + ) np.random.seed(random_state) ################################################# # load json file ################################################# - dataset_name = dataset.split('/')[-1][:-7] + dataset_name = dataset.split("/")[-1][:-7] - save_file = (results_path + '/' + dataset_name + '_' + est_name + '_' - + str(random_state)) + save_file = ( + results_path + "/" + dataset_name + "_" + est_name + "_" + str(random_state) + ) if target_noise > 0: - save_file += '_target-noise'+str(target_noise) + save_file += "_target-noise" + str(target_noise) if feature_noise > 0: - save_file += '_feature-noise'+str(feature_noise) + save_file += "_feature-noise" + str(feature_noise) - print('looking for:',save_file+'.json') + print("looking for:", save_file + ".json") - if os.path.exists(save_file+'.json'): - r = json.load(open(save_file+'.json', 'r')) + if os.path.exists(save_file + ".json"): + r = json.load(open(save_file + ".json", "r")) else: - raise FileNotFoundError(save_file+'.json not found') + raise FileNotFoundError(save_file + ".json not found") - raw_model = r['symbolic_model'] + raw_model = r["symbolic_model"] - if 'AIFeynman' in est_name: + if "AIFeynman" in est_name: # correct model size model_size = rewrite_AIFeynman_model_size(raw_model) - print('changing model size from {} to {}'.format( - model_size,r['model_size'])) - r['model_size'] = model_size + print("changing model size from {} to {}".format(model_size, r["model_size"])) + r["model_size"] = model_size else: - raise ValueError('est_name not AIFeynman (it is %s)' % est_name) + raise ValueError("est_name not AIFeynman (it is %s)" % est_name) save(r, save_file) - print('done.') + print("done.") + ################################################################################ # main entry point ################################################################################ -import argparse -import importlib - -if __name__ == '__main__': +if __name__ == "__main__": # parse command line arguments parser = argparse.ArgumentParser( - description="Evaluate a method on a dataset.", add_help=False) - parser.add_argument('INPUT_FILE', type=str, - help='Data file to analyze; ensure that the ' - 'target/label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', - help='Show this help message and exit.') - parser.add_argument('-ml', action='store', dest='ALG',default=None,type=str, - help='Name of estimator (with matching file in methods/)') - parser.add_argument('-results_path', action='store', dest='RDIR', - default='results_test', type=str, - help='Name of save file') - parser.add_argument('-seed', action='store', dest='RANDOM_STATE', - default=42, type=int, help='Seed / trial') - parser.add_argument('-test',action='store_true', dest='TEST', - help='Used for testing a minimal version') - parser.add_argument('-target_noise',action='store',dest='Y_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-feature_noise',action='store',dest='X_NOISE', - default=0.0, type=float, help='Gaussian noise to add' - 'to the target') - parser.add_argument('-sym_data',action='store_true', dest='SYM_DATA', - help='Use symbolic dataset settings') + description="Evaluate a method on a dataset.", add_help=False + ) + parser.add_argument( + "INPUT_FILE", + type=str, + help="Data file to analyze; ensure that the " + 'target/label column is labeled as "class".', + ) + parser.add_argument( + "-h", "--help", action="help", help="Show this help message and exit." + ) + parser.add_argument( + "-ml", + action="store", + dest="ALG", + default=None, + type=str, + help="Name of estimator (with matching file in methods/)", + ) + parser.add_argument( + "-results_path", + action="store", + dest="RDIR", + default="results_test", + type=str, + help="Name of save file", + ) + parser.add_argument( + "-seed", + action="store", + dest="RANDOM_STATE", + default=42, + type=int, + help="Seed / trial", + ) + parser.add_argument( + "-test", + action="store_true", + dest="TEST", + help="Used for testing a minimal version", + ) + parser.add_argument( + "-target_noise", + action="store", + dest="Y_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-feature_noise", + action="store", + dest="X_NOISE", + default=0.0, + type=float, + help="Gaussian noise to addto the target", + ) + parser.add_argument( + "-sym_data", + action="store_true", + dest="SYM_DATA", + help="Use symbolic dataset settings", + ) args = parser.parse_args() print(args.__dict__) - fix_aifeynman_model_size(args.INPUT_FILE, args.RDIR, args.RANDOM_STATE, args.ALG, - target_noise=args.Y_NOISE, feature_noise=args.X_NOISE - ) + fix_aifeynman_model_size( + args.INPUT_FILE, + args.RDIR, + args.RANDOM_STATE, + args.ALG, + target_noise=args.Y_NOISE, + feature_noise=args.X_NOISE, + ) diff --git a/experiment/methods/afp/regressor.py b/experiment/methods/afp/regressor.py index 3e868ef73..f4dd4de44 100644 --- a/experiment/methods/afp/regressor.py +++ b/experiment/methods/afp/regressor.py @@ -3,53 +3,51 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt', - 'sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=1, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g = 250, - time_limit=2*60*60 - ) +est = ellyn( + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=1, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=250, + time_limit=2 * 60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) - diff --git a/experiment/methods/afp_ehc/regressor.py b/experiment/methods/afp_ehc/regressor.py index de6a718ae..576035160 100644 --- a/experiment/methods/afp_ehc/regressor.py +++ b/experiment/methods/afp_ehc/regressor.py @@ -1,14 +1,13 @@ from ellyn import ellyn # 500,000 evaluations = 100,000 with 1 eval, 1 constant hill climbing, and 3 -# EHC iterations +# EHC iterations pop_sizes = [250, 500, 1000] gs = [2500, 1000, 500] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt', - 'sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] @@ -16,44 +15,43 @@ # there is a time limit setting for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model est = ellyn( - eHC_on=True, - eHC_its=3, - selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g=100, - time_limit=60*60 - ) + eHC_on=True, + eHC_its=3, + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=100, + time_limit=60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/afp_fe/regressor.py b/experiment/methods/afp_fe/regressor.py index 29b00c403..525f157ec 100644 --- a/experiment/methods/afp_fe/regressor.py +++ b/experiment/methods/afp_fe/regressor.py @@ -3,10 +3,10 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt','sin','cos'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt", "sin", "cos"], +] hyper_params = [] @@ -14,55 +14,55 @@ # there is a time limit setting for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='afp', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=1000, - g=250, - time_limit=60*60, - - EstimateFitness=True, - FE_pop_size=100, - FE_ind_size=10, - FE_train_size=10, - FE_train_gens=10, - FE_rank=True, - ) +est = ellyn( + selection="afp", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=1000, + g=250, + time_limit=60 * 60, + EstimateFitness=True, + FE_pop_size=100, + FE_ind_size=10, + FE_train_size=10, + FE_train_gens=10, + FE_rank=True, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): model_str = est.stack_2_eqn(est.best_estimator_) # protected sqrt uses |cdot|. removing it - model_str = model_str.replace('|','') + model_str = model_str.replace("|", "") return model_str + def pre_train(est, X, y): """Adjust settings based on data before training""" # adjust generations based on size of X versus FE size g = est.g - est.g = int(g*len(X)/est.FE_ind_size) - print('FE ellyn gens adjusted from',g,'to',est.g) + est.g = int(g * len(X) / est.FE_ind_size) + print("FE ellyn gens adjusted from", g, "to", est.g) + eval_kwargs = dict(pre_train=pre_train, use_dataframe=False) diff --git a/experiment/methods/bingo/regressor.py b/experiment/methods/bingo/regressor.py index 6c80c1ca2..25f8583eb 100644 --- a/experiment/methods/bingo/regressor.py +++ b/experiment/methods/bingo/regressor.py @@ -1,18 +1,26 @@ -from sklearn.base import RegressorMixin +from __future__ import annotations + +from typing import Any -from bingo.symbolic_regression.symbolic_regressor import SymbolicRegressor from bingo.symbolic_regression.srbench_interface import ( - model, - get_population, - get_best_solution, eval_kwargs as base_eval_kwargs, ) +from bingo.symbolic_regression.srbench_interface import ( + get_best_solution as get_best_solution, +) +from bingo.symbolic_regression.srbench_interface import ( + get_population as get_population, +) +from bingo.symbolic_regression.srbench_interface import ( + model as model, +) +from bingo.symbolic_regression.symbolic_regressor import SymbolicRegressor eval_kwargs = dict(base_eval_kwargs) eval_kwargs["use_dataframe"] = False # sklearn's interface should not modify constructor parameters. Bingo modifies -# max_time, so here I'm wrapping it into an sklearn-compatible estimator +# max_time, so here I'm wrapping it into an sklearn-compatible estimator # so gridsearch will work. # class BingoSymbolicRegressor(SymbolicRegressor, RegressorMixin): @@ -88,7 +96,7 @@ # self.best_ind_ = min(self.est_.best_pop_, key=lambda x: x.fitness) # return self - + # def get_params(self, deep=True): # out = dict() # for (key, value) in self.__dict__.items(): @@ -144,10 +152,12 @@ # ) population_size = [100, 500, 1000] -operators=[["+", "-", "*", "/", "sin", "cos", "exp", "log", "sqrt"], - ["+", "-", "*", "/", "log", "sqrt"]] +operators = [ + ["+", "-", "*", "/", "sin", "cos", "exp", "log", "sqrt"], + ["+", "-", "*", "/", "log", "sqrt"], +] -hyper_params = [] +hyper_params: list[dict[str, Any]] = [] # for p in population_size: # for ops in operators: # hyper_params.append({ @@ -169,4 +179,4 @@ max_evals=int(1e19), evolutionary_algorithm="AgeFitnessEA", clo_threshold=1.0e-5, -) \ No newline at end of file +) diff --git a/experiment/methods/brush/regressor.py b/experiment/methods/brush/regressor.py index 837c5228d..57ab82fdd 100644 --- a/experiment/methods/brush/regressor.py +++ b/experiment/methods/brush/regressor.py @@ -1,5 +1,6 @@ -from pybrush import BrushRegressor +from __future__ import annotations +from pybrush import BrushRegressor kwargs = { "verbosity": 1, @@ -10,7 +11,6 @@ "initialization": "uniform", "validation_size": 0.33, "cx_prob": 1 / 7, - "weights_init": False, "mutation_probs": { "point": 1 / 6, @@ -20,16 +20,13 @@ "toggle_weight_on": 1 / 6, "toggle_weight_off": 1 / 6, }, - "sel": "lexicase", "algorithm": "nsga2", "objectives": ["scorer", "complexity"], "scorer": "mse", - "bandit": "dynamic_thompson", "num_islands": 1, "shuffle_split": True, - "functions": [ # Arithmetic "Add", @@ -37,7 +34,6 @@ "Mul", "Div", "Pow", - # Unary functions "Sin", "Cos", @@ -45,11 +41,9 @@ "Exp", "Log", "Sqrt", - # Optional split operators "SplitBest", "SplitOn", - # Terminals "Constant", "Terminal", @@ -73,7 +67,6 @@ "Div": 2, "Add": 2, "Pow": 2, - "Sin": 1, "Cos": 1, "Tanh": 1, @@ -90,7 +83,7 @@ def pretify_expr(string, feature_names): tokens = string.replace(" ", "").replace(")", "").replace("(", ",").split(",") new_string = "" - stack = [] + stack: list[tuple[str, list[str]]] = [] for token in tokens: stack.append((token, [])) @@ -149,4 +142,4 @@ def complexity(estimator): if hasattr(estimator, "fitness"): return estimator.fitness.size - raise AttributeError("Estimator does not expose model complexity.") \ No newline at end of file + raise AttributeError("Estimator does not expose model complexity.") diff --git a/experiment/methods/bsr/regressor.py b/experiment/methods/bsr/regressor.py index e764ab0b9..563ccbd27 100644 --- a/experiment/methods/bsr/regressor.py +++ b/experiment/methods/bsr/regressor.py @@ -1,33 +1,38 @@ from bsr.bsr_class import BSR hyper_params = [] -for val, itrNum in zip([50,100,250],[250,100,50]): - for treeNum in [2,3]: +for val, itrNum in zip([50, 100, 250], [250, 100, 50]): + for treeNum in [2, 3]: hyper_params.append( - {'treeNum': [treeNum], - 'itrNum': [itrNum], - 'val': [val], - }) + { + "treeNum": [treeNum], + "itrNum": [itrNum], + "val": [val], + } + ) # initialize est = BSR( - val=100, - itrNum=500, - treeNum=3, - alpha1= 0.4, - alpha2= 0.4, - beta= -1, - disp=False, - max_time=60*60) + val=100, + itrNum=500, + treeNum=3, + alpha1=0.4, + alpha2=0.4, + beta=-1, + disp=False, + max_time=60 * 60, +) + def complexity(est): return est.complexity() + def model(est): model_str = est.model() # get rid of square brackets - new_model_str = model_str.replace('[','').replace(']','') + new_model_str = model_str.replace("[", "").replace("]", "") # BSR uses caret notation for powers; SymPy parses ^ as XOR here. - new_model_str = new_model_str.replace('^', '**') - + new_model_str = new_model_str.replace("^", "**") + return new_model_str diff --git a/experiment/methods/e2et/regressor.py b/experiment/methods/e2et/regressor.py index 36ccdc62b..8f0a1ed61 100644 --- a/experiment/methods/e2et/regressor.py +++ b/experiment/methods/e2et/regressor.py @@ -1,55 +1,50 @@ -import torch -import os, sys -import symbolicregression +import os + import requests -import numexpr +import symbolicregression +import torch model_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "/pretrained/model.pt" + os.path.dirname(os.path.abspath(__file__)), "/pretrained/model.pt" ) try: - if not os.path.isfile(model_path): + if not os.path.isfile(model_path): url = "https://dl.fbaipublicfiles.com/symbolicregression/model1.pt" r = requests.get(url, allow_redirects=True) - open(model_path, 'wb').write(r.content) + open(model_path, "wb").write(r.content) if not torch.cuda.is_available(): - model = torch.load(model_path, map_location=torch.device('cpu')) + pretrained_model = torch.load(model_path, map_location=torch.device("cpu")) else: - model = torch.load(model_path) - model = model.cuda() - print(model.device) + pretrained_model = torch.load(model_path) + pretrained_model = pretrained_model.cuda() + print(pretrained_model.device) print("Model successfully loaded!") except Exception as e: print("ERROR: model not loaded! path was: {}".format(model_path)) - print(e) + print(e) est = symbolicregression.model.SymbolicTransformerRegressor( - model=model, - max_input_points=200, - n_trees_to_refine=100, - rescale=True - ) + model=pretrained_model, max_input_points=200, n_trees_to_refine=100, rescale=True +) + def model(est, X=None): replace_ops = {"add": "+", "mul": "*", "sub": "-", "pow": "**", "inv": "1/"} model_str = est.retrieve_tree(dataset_idx=0).infix() - for op,replace_op in replace_ops.items(): - model_str = model_str.replace(op,replace_op) + for op, replace_op in replace_ops.items(): + model_str = model_str.replace(op, replace_op) return model_str + def my_pre_train_fn(est, X, y): - """In this example we adjust FEAT generations based on the size of X - versus relative to FEAT's batch size setting. + """In this example we adjust FEAT generations based on the size of X + versus relative to FEAT's batch size setting. """ return + # define eval_kwargs. eval_kwargs = dict( - test_params = { - 'max_input_points':100, - 'n_trees_to_refine':10 - }, - use_dataframe=False - ) + test_params={"max_input_points": 100, "n_trees_to_refine": 10}, use_dataframe=False +) diff --git a/experiment/methods/eplex/regressor.py b/experiment/methods/eplex/regressor.py index 35f91d494..8841edde1 100644 --- a/experiment/methods/eplex/regressor.py +++ b/experiment/methods/eplex/regressor.py @@ -3,45 +3,45 @@ # 500,000 evaluations = 250,000 with 1 constant hill climbing iteration pop_sizes = [100, 500, 1000] gs = [2500, 500, 250] -op_lists=[ - ['n','v','+','-','*','/','sin','cos','exp','log','2','3', 'sqrt'], - ['n','v','+','-','*','/', 'exp','log','2','3', 'sqrt'] - ] +op_lists = [ + ["n", "v", "+", "-", "*", "/", "sin", "cos", "exp", "log", "2", "3", "sqrt"], + ["n", "v", "+", "-", "*", "/", "exp", "log", "2", "3", "sqrt"], +] hyper_params = [] for p, g in zip(pop_sizes, gs): for op_list in op_lists: - hyper_params.append({ - 'popsize':[p], - 'g':[g], - 'op_list':[op_list] - }) + hyper_params.append({"popsize": [p], "g": [g], "op_list": [op_list]}) # Create the pipeline for the model -est = ellyn(selection='epsilon_lexicase', - lex_eps_global=False, - lex_eps_dynamic=False, - islands=False, - num_islands=10, - island_gens=100, - verbosity=0, - print_data=False, - elitism=True, - pHC_on=True, - prto_arch_on=True, - max_len = 64, - max_len_init=20, - popsize=500, - g=500, - time_limit=2*60*60 - ) +est = ellyn( + selection="epsilon_lexicase", + lex_eps_global=False, + lex_eps_dynamic=False, + islands=False, + num_islands=10, + island_gens=100, + verbosity=0, + print_data=False, + elitism=True, + pHC_on=True, + prto_arch_on=True, + max_len=64, + max_len_init=20, + popsize=500, + g=500, + time_limit=2 * 60 * 60, +) + def complexity(est): return len(est.best_estimator_) + def model(est, X=None): return est.stack_2_eqn(est.best_estimator_) -eval_kwargs = dict(use_dataframe=False) \ No newline at end of file + +eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/eql/regressor.py b/experiment/methods/eql/regressor.py index c9a78168b..3b5117929 100644 --- a/experiment/methods/eql/regressor.py +++ b/experiment/methods/eql/regressor.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 -from eql.est import EQL import pandas as pd -from sklearn.model_selection import GridSearchCV - +from eql.est import EQL base = EQL(n_iter=10_000) @@ -10,33 +8,30 @@ n_layers = (1, 2) functions = [ "id;mul;cos;sin;exp;square;sqrt;id;mul;cos;sin;exp;square;sqrt;log", - "id;mul;cos;div;sqrt;cos;sin;div;mul;mul;cos;id;log" + "id;mul;cos;div;sqrt;cos;sin;div;mul;mul;cos;id;log", ] hyper_params = [] -for r, l in zip(reg, n_layers): +for r, layers in zip(reg, n_layers): for f in functions: - hyper_params.append({ - 'reg':[r], - 'n_layers':[l], - 'functions':[f] - }) - + hyper_params.append({"reg": [r], "n_layers": [layers], "functions": [f]}) + # est = GridSearchCV(estimator=base, param_grid=hyper_params, cv=2, refit=True, n_jobs=4, verbose=0) est = base + def model(est, X=None): model_str = str(est.get_eqn()) if isinstance(X, pd.DataFrame): - for i,f in reversed(list(enumerate(X.columns))): - model_str = model_str.replace(f'x{i}',f) + for i, f in reversed(list(enumerate(X.columns))): + model_str = model_str.replace(f"x{i}", f) return model_str eval_kwargs = { # 'test_params': {'n_iter': [10], **hyper_params[-1]}, - 'use_dataframe':False + "use_dataframe": False } diff --git a/experiment/methods/experimental/GSGPRegressor.py b/experiment/methods/experimental/GSGPRegressor.py index 5371e90b0..34fd65ea5 100755 --- a/experiment/methods/experimental/GSGPRegressor.py +++ b/experiment/methods/experimental/GSGPRegressor.py @@ -1,45 +1,58 @@ from .src.gsgp import GSGPRegressor - hyper_params = [ { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.0,),'rt_mut':(1.0,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.0,), + "rt_mut": (1.0,), }, { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.1,),'rt_mut':(0.9,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.1,), + "rt_mut": (0.9,), }, { - 'popsize': (100,), 'g': (1000,), - 'max_len': (6,), - 'rt_cross':(0.2,),'rt_mut':(0.8,), + "popsize": (100,), + "g": (1000,), + "max_len": (6,), + "rt_cross": (0.2,), + "rt_mut": (0.8,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.0,),'rt_mut':(1.0,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.0,), + "rt_mut": (1.0,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.1,),'rt_mut':(0.9,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.1,), + "rt_mut": (0.9,), }, { - 'popsize': (1000,), 'g': (100,), - 'max_len': (6,), - 'rt_cross':(0.2,),'rt_mut':(0.8,), + "popsize": (1000,), + "g": (100,), + "max_len": (6,), + "rt_cross": (0.2,), + "rt_mut": (0.8,), }, ] -est=GSGPRegressor() +est = GSGPRegressor() + -#TODO: define complexity +# TODO: define complexity def complexity(est): return -1 + model = None model = None diff --git a/experiment/methods/feat/regressor.py b/experiment/methods/feat/regressor.py index 5162187b4..32ded7969 100644 --- a/experiment/methods/feat/regressor.py +++ b/experiment/methods/feat/regressor.py @@ -1,45 +1,33 @@ -# This example submission shows the submission of FEAT (cavalab.org/feat). -from feat import FeatRegressor -from sklearn.base import BaseEstimator, RegressorMixin -from feat import Feat, FeatRegressor, FeatClassifier +# This example submission shows the submission of FEAT (cavalab.org/feat). -from sklearn.datasets import load_diabetes, make_blobs -from sklearn.base import clone -from sklearn.pipeline import make_pipeline -from sklearn.metrics.pairwise import rbf_kernel -from sklearn.metrics import r2_score -from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin -import unittest -import argparse -import sys -import pandas as pd import numpy as np -import pickle - +from feat import FeatRegressor +from sklearn.base import RegressorMixin +from sklearn.metrics import r2_score """ est: a sklearn-compatible regressor. if you don't have one they are fairly easy to create. see https://scikit-learn.org/stable/developers/develop.html """ -est:RegressorMixin = FeatRegressor( - pop_size=100, - gens=100, - max_time=8*60*60, # 8 hrs. Your algorithm should have this feature - max_depth=6, - verbosity=2, - batch_size=100, - functions=['+','-','*','/','^2','^3','sqrt','sin','cos','exp','log'], - otype='f' - ) -# want to tune your estimator? wrap it in a sklearn CV class. +est: RegressorMixin = FeatRegressor( + pop_size=100, + gens=100, + max_time=8 * 60 * 60, # 8 hrs. Your algorithm should have this feature + max_depth=6, + verbosity=2, + batch_size=100, + functions=["+", "-", "*", "/", "^2", "^3", "sqrt", "sin", "cos", "exp", "log"], + otype="f", +) +# want to tune your estimator? wrap it in a sklearn CV class. class FeatPopEstimator(RegressorMixin): """ FeatPopEstimator is a custom regressor that wraps a fitted FEAT estimator to call `model` and `predict` from its archive. - + Attributes: est (object): The fitted FEAT estimator. id (int): The identifier for the specific model in the estimator's archive. @@ -57,57 +45,59 @@ class FeatPopEstimator(RegressorMixin): model(): Retrieves the model equation from the estimator's archive. """ + def __init__(self, est, id): self.est = est - self.id = id + self.id = id def fit(self, X, y): self.is_fitted_ = True def predict(self, X): - + X = self.est._prep_X(X) return self.est.cfeat_.predict_archive(self.id, X) def score(self, X, y): yhat = self.predict(X).flatten() - return r2_score(y,yhat) - + return r2_score(y, yhat) + def model(self): archive = self.est.cfeat_.get_archive(False) - ind = [i for i in archive if i['id']==self.id][0] + ind = [i for i in archive if i["id"] == self.id][0] eqn = f"{np.round(ind['ml']['bias'], 5)}" - for eq, w in zip(ind['eqn'].replace('[', '').split(']'), ind['w']): - if str(w)[0]=='-': - eqn = eqn + f'{np.round(float(w), 2)}*{eq}' + for eq, w in zip(ind["eqn"].replace("[", "").split("]"), ind["w"]): + if str(w)[0] == "-": + eqn = eqn + f"{np.round(float(w), 2)}*{eq}" else: - eqn = eqn + f'+{np.round(float(w), 2)}*{eq}' + eqn = eqn + f"+{np.round(float(w), 2)}*{eq}" return eqn + def model(est, X=None) -> str: """ - Return a sympy-compatible string of the final model. + Return a sympy-compatible string of the final model. Parameters ---------- est: sklearn regressor - The fitted model. + The fitted model. X: pd.DataFrame, default=None The training data. This argument can be dropped if desired. Returns ------- - A sympy-compatible string of the final model. + A sympy-compatible string of the final model. Notes ----- - Ensure that the variable names appearing in the model are identical to - those in the training data, `X`, which is a `pd.Dataframe`. - If your method names variables some other way, e.g. `[x_0 ... x_m]`, + Ensure that the variable names appearing in the model are identical to + those in the training data, `X`, which is a `pd.Dataframe`. + If your method names variables some other way, e.g. `[x_0 ... x_m]`, you can specify a mapping in the `model` function such as: ``` @@ -119,9 +109,9 @@ def model(est, X): ``` If you have special operators such as protected division or protected log, - you will need to handle these to assure they conform to sympy format. + you will need to handle these to assure they conform to sympy format. One option is to replace them with the unprotected versions. Post an issue - if you have further questions: + if you have further questions: https://github.com/cavalab/srbench/issues/new/choose """ @@ -132,20 +122,20 @@ def model(est, X): model_str = est.cfeat_.get_eqn() # Here we replace "|" with "" to handle - # protected sqrt (expressed as sqrt(|.|)) in FEAT) - model_str = model_str.replace('|','') + # protected sqrt (expressed as sqrt(|.|)) in FEAT) + model_str = model_str.replace("|", "") # use python syntax for exponents - model_str = model_str.replace('^','**') + model_str = model_str.replace("^", "**") return model_str def complexity(est): if isinstance(est, FeatPopEstimator): - est = est.est # It is wrapped + est = est.est # It is wrapped - return est.get_n_nodes() + 2*est.get_n_params() + 2*est.get_dim() + return est.get_n_nodes() + 2 * est.get_n_params() + 2 * est.get_dim() def get_population(est) -> list[RegressorMixin]: @@ -156,7 +146,7 @@ def get_population(est) -> list[RegressorMixin]: Also, it is expected that the `model()` function can operate with them, so they should have a way of getting a simpy string representation. - + Returns ------- A list of scikit-learn compatible estimators that can be used for prediction. @@ -170,21 +160,19 @@ def get_population(est) -> list[RegressorMixin]: # archive contains individuals serialized in json objects. let's get their ids for ind in archive: # Archive is sorted by complexity - pop.append( - FeatPopEstimator(est, ind['id']) - ) + pop.append(FeatPopEstimator(est, ind["id"])) # Stopping here to avoid too many models if len(pop) >= 100: break - return pop + def get_best_solution(est) -> RegressorMixin: """ - Return the best solution from the final model. - + Return the best solution from the final model. + Returns ------- A scikit-learn compatible estimator @@ -224,21 +212,18 @@ def get_best_solution(est) -> RegressorMixin: y: training labels. """ + def my_pre_train_fn(est, X, y): - """In this example we adjust FEAT generations based on the size of X - versus relative to FEAT's batch size setting. + """In this example we adjust FEAT generations based on the size of X + versus relative to FEAT's batch size setting. """ if est.batch_size < len(X): - est.gens = int(est.gens*len(X)/est.batch_size) - print('FEAT gens adjusted to',est.gens) + est.gens = int(est.gens * len(X) / est.batch_size) + print("FEAT gens adjusted to", est.gens) # adjust max dim - est.max_dim=min(max(est.max_dim, X.shape[1]), 20) - print('FEAT max_dim set to',est.max_dim) + est.max_dim = min(max(est.max_dim, X.shape[1]), 20) + print("FEAT max_dim set to", est.max_dim) + # define eval_kwargs. -eval_kwargs = dict( - pre_train=my_pre_train_fn, - test_params = {'gens': 5, - 'pop_size': 10 - } - ) +eval_kwargs = dict(pre_train=my_pre_train_fn, test_params={"gens": 5, "pop_size": 10}) diff --git a/experiment/methods/ffx/regressor.py b/experiment/methods/ffx/regressor.py index fceb55d62..5e74e96bb 100644 --- a/experiment/methods/ffx/regressor.py +++ b/experiment/methods/ffx/regressor.py @@ -1,15 +1,20 @@ +from __future__ import annotations + +from typing import Any + from ffx import FFXRegressor # FFX has no parameters! -hyper_params = [ - { - }, +hyper_params: list[dict[str, Any]] = [ + {}, ] est = FFXRegressor() + def complexity(est): return est.model_.complexity() + def model(est, X=None): - return est.model_.str2().replace('^', '**') + return est.model_.str2().replace("^", "**") diff --git a/experiment/methods/geneticengine/regressor.py b/experiment/methods/geneticengine/regressor.py index cd5890a2a..5ec0e18da 100644 --- a/experiment/methods/geneticengine/regressor.py +++ b/experiment/methods/geneticengine/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import GeneticProgrammingRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = GeneticProgrammingRegressor( - max_time=60*60, # 8 Your algorithm should have this feature + max_time=60 * 60, # 8 Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_1p1/regressor.py b/experiment/methods/geneticengine_1p1/regressor.py index 8ba80594c..55402a4c1 100644 --- a/experiment/methods/geneticengine_1p1/regressor.py +++ b/experiment/methods/geneticengine_1p1/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import RandomSearchRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = RandomSearchRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_hc/regressor.py b/experiment/methods/geneticengine_hc/regressor.py index 4e58916b7..353598a85 100644 --- a/experiment/methods/geneticengine_hc/regressor.py +++ b/experiment/methods/geneticengine_hc/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import HillClimbingRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = HillClimbingRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/geneticengine_rs/regressor.py b/experiment/methods/geneticengine_rs/regressor.py index 6db0995d6..c14a8df66 100644 --- a/experiment/methods/geneticengine_rs/regressor.py +++ b/experiment/methods/geneticengine_rs/regressor.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from typing import Any + # This example submission shows the submission of FEAT (cavalab.org/feat). from geml.regressors import OnePlusOneRegressor -from geml.regressors import model +from geml.regressors import model as model from sklearn.base import RegressorMixin """ @@ -9,7 +13,7 @@ see https://scikit-learn.org/stable/developers/develop.html """ est: RegressorMixin = OnePlusOneRegressor( - max_time=60*60, # 8 hrs. Your algorithm should have this feature + max_time=60 * 60, # 8 hrs. Your algorithm should have this feature ) @@ -43,4 +47,4 @@ def get_best_solution(est) -> RegressorMixin: # define eval_kwargs. -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/gp-elite/regressor.py b/experiment/methods/gp-elite/regressor.py index e02654221..cbf170f9b 100644 --- a/experiment/methods/gp-elite/regressor.py +++ b/experiment/methods/gp-elite/regressor.py @@ -2,14 +2,23 @@ Exposes the three objects the harness reads: `est`, `hyper_params`, `eval_kwargs`, plus `model(est, X)` and `complexity(est)`. + +Requires gp-elite >= 0.8.0, which brings the two things this wrapper relies on: +a per-call time limit (`time_limit=`), so a restart can no longer overrun the +harness budget, and an exported formula written in the raw input variables and +checked against `predict` at fit time (`SRResult.sympy`). """ + +from __future__ import annotations + import time +from typing import Any import numpy as np try: from sklearn.utils.validation import validate_data -except Exception: # older sklearn +except Exception: # older sklearn validate_data = None from gp_elite import GPEliteRegressor @@ -20,24 +29,48 @@ class GPEliteSRBench(GPEliteRegressor): """GP_ELITE for SRBench. fit() runs sequential restarts (each one a full independent evolution) and - keeps the best by validation R2, stopping once the time budget is nearly - spent or `restarts` is reached. + keeps the best by validation R2. Every restart is time-bounded: it receives + an equal share of the budget still available, so the whole fit ends within + `max_time` (minus a safety margin) and the champion is published as soon as + it is found. """ - def __init__(self, operators="physical", normalize="auto", generations=60, - speed="fast", validation_split=0.20, max_time=3600, - restarts=10, robust=False, parallel=None, random_state=0): - super().__init__(operators=operators, normalize=normalize, - generations=generations, speed=speed, - validation_split=validation_split, restarts=restarts, - robust=robust, parallel=parallel, - random_state=random_state) + def __init__( + self, + operators="physical", + normalize="auto", + generations=60, + speed="fast", + validation_split=0.20, + max_time=3600, + restarts=10, + robust=False, + parallel=None, + random_state=0, + ): + super().__init__( + operators=operators, + normalize=normalize, + generations=generations, + speed=speed, + validation_split=validation_split, + restarts=restarts, + robust=robust, + parallel=parallel, + random_state=random_state, + ) self.max_time = max_time + def _publish(self, r): + self.model_ = r + self.equation_ = r.expression + self.is_fitted_ = True + def fit(self, X, y): if validate_data is not None: - X, y = validate_data(self, X, y, y_numeric=True, - ensure_min_samples=2, dtype="numeric") + X, y = validate_data( + self, X, y, y_numeric=True, ensure_min_samples=2, dtype="numeric" + ) else: X = np.asarray(X, dtype=float) y = np.asarray(y, dtype=float) @@ -49,135 +82,81 @@ def fit(self, X, y): names = ["X%d" % i for i in range(self.n_features_in_)] t0 = time.time() budget = max(30.0, float(self.max_time)) + # 10 % margin: final selection, predict and model export happen after + # the last restart, and the harness alarm must never be the one that + # ends the fit. + deadline = t0 + 0.90 * budget + n = max(1, int(self.restarts)) best, best_score = None, -np.inf seed0 = 0 if self.random_state is None else int(self.random_state) - for k in range(max(1, int(self.restarts))): + for k in range(n): + remaining = deadline - time.time() + if k > 0 and remaining <= 0: + break r = symbolic_regression( - X, y, feature_names=names, - operators=self.operators, normalize=self.normalize, - generations=self.generations, speed=self.speed, - validation_split=self.validation_split, restarts=1, - robust=self.robust, parallel=self.parallel, - seed=seed0 + 1000 * k) + X, + y, + feature_names=names, + operators=self.operators, + normalize=self.normalize, + generations=self.generations, + speed=self.speed, + validation_split=self.validation_split, + restarts=1, + robust=self.robust, + parallel=self.parallel, + time_limit=max(1.0, remaining / (n - k)), + seed=seed0 + 1000 * k, + ) score = r.r2_validation if score is None or not np.isfinite(score): p = r.predict(X) ss = float(np.sum((y - y.mean()) ** 2)) or 1e-30 score = 1.0 - float(np.sum((y - p) ** 2) / ss) - if score > best_score: + if best is None or score > best_score: best, best_score = r, score - # Publish the champion AS SOON AS it is found, so that a - # single overrunning restart (symbolic_regression is not itself - # time-bounded) can never leave the estimator without model_. - # If the harness SIGALRM fires mid-restart, evaluate_model - # still finds a (possibly worse) result instead of raising - # AttributeError and losing the run entirely. - self.model_ = best - self.equation_ = best.expression - self.is_fitted_ = True - - elapsed = time.time() - t0 - if elapsed > 0.85 * budget or elapsed + elapsed / (k + 1) > budget: - break + self._publish(best) # published as soon as found - # Guarantee the attributes exist even if the very first restart failed - # to produce a usable result (defensive; best is normally set above). if best is not None and not getattr(self, "is_fitted_", False): - self.model_ = best - self.equation_ = best.expression - self.is_fitted_ = True + self._publish(best) + return self # ── the three objects SRBench reads ───────────────────────────────────────── -est = GPEliteSRBench(operators="physical", generations=60, speed="fast", - restarts=10, max_time=3600, random_state=0) +est = GPEliteSRBench( + operators="physical", + generations=60, + speed="fast", + restarts=10, + max_time=3600, + random_state=0, +) hyper_params = [ - {"operators": ("physical", "full"), - "generations": (30, 60)}, + {"operators": ("physical", "full"), "generations": (30, 60)}, ] -eval_kwargs = {} - - -# Map every GP_ELITE operator to a sympy-parsable form. Anything produced by the -# "physical" or "full" operator sets must appear here; unknown names would parse -# as opaque undefined functions (e.g. cube(x) is NOT x**3 to sympy) and break -# numeric evaluation downstream. -_UNARY = { - "neg": lambda a: "(-(%s))" % a, - "abs": lambda a: "Abs(%s)" % a, - "inv": lambda a: "(1/(%s))" % a, - "sq": lambda a: "((%s)**2)" % a, - "cube": lambda a: "((%s)**3)" % a, - "sqrt": lambda a: "sqrt(Abs(%s))" % a, # engine's sqrt is domain-guarded - "log": lambda a: "log(Abs(%s))" % a, # engine's log is domain-guarded - "exp": lambda a: "exp(%s)" % a, - "sin": lambda a: "sin(%s)" % a, - "cos": lambda a: "cos(%s)" % a, - "tan": lambda a: "tan(%s)" % a, - "tanh": lambda a: "tanh(%s)" % a, - "step": lambda a: "Heaviside(%s)" % a, - "is_even": lambda a: "(1 - Mod(floor(%s), 2))" % a, -} -_BINARY = { - "+": lambda a, b: "((%s) + (%s))" % (a, b), - "-": lambda a, b: "((%s) - (%s))" % (a, b), - "*": lambda a, b: "((%s) * (%s))" % (a, b), - "/": lambda a, b: "((%s) / (%s))" % (a, b), - "pow": lambda a, b: "((%s)**(%s))" % (a, b), - "max2": lambda a, b: "Max((%s), (%s))" % (a, b), - "min2": lambda a, b: "Min((%s), (%s))" % (a, b), -} - - -def _to_sympy(node, cols): - """Walk the expression tree, emit a sympy-parsable string with the - dataset's real column names.""" - if node.left is None and node.right is None: - v = node.value - # np.float64 subclasses float, but np.int64 does NOT subclass int, so - # test the numpy hierarchy explicitly — otherwise an integer-valued - # numpy constant would fall through to the name branch and be emitted - # as a feature name instead of a number. - if isinstance(v, (int, float, np.integer, np.floating)) \ - and not isinstance(v, bool): - return "%.12g" % float(v) - s = str(v) # like 'X[3]' or 'X3' - digits = "".join(c for c in s if c.isdigit()) - i = int(digits) if digits else 0 - return cols[i] if i < len(cols) else "x%d" % i - - op = node.value - if node.right is None: # unary - a = _to_sympy(node.left, cols) - f = _UNARY.get(op) - if f is None: - raise ValueError("unmapped unary operator %r in _to_sympy" % op) - return f(a) - - a = _to_sympy(node.left, cols) - b = _to_sympy(node.right, cols) - f = _BINARY.get(op) - if f is None: - raise ValueError("unmapped binary operator %r in _to_sympy" % op) - return f(a, b) +eval_kwargs: dict[str, Any] = {} def model(est, X=None): - cols = list(X.columns) if X is not None and hasattr(X, "columns") \ + """The fitted formula as a sympy-parsable string in the dataset's columns. + + gp-elite >= 0.8 writes it in the raw input variables for every internal + normalisation (division by max|x| by default, min-max or z-score on + request) and checks at fit time that it reproduces predict(); protected + operators are spelled as the engine computes them (sqrt(Abs(u)) where u + can be negative). + """ + cols = ( + list(X.columns) + if X is not None and hasattr(X, "columns") else ["x%d" % i for i in range(getattr(est, "n_features_in_", 0))] - # Fold the input normalization into the string so the formula, evaluated - # on RAW columns, reproduces est.predict exactly. The engine's shift-free - # scaler divides each feature by scale_[i]. - scaler = getattr(est.model_, "scaler", None) - if scaler is not None and hasattr(scaler, "scale_"): - cols = ["((%s)/%.12g)" % (c, float(f)) - for c, f in zip(cols, scaler.scale_)] - return _to_sympy(est.model_.node, cols) + ) + return est.model_.sympy(cols) def complexity(est): diff --git a/experiment/methods/gpgomea/regressor.py b/experiment/methods/gpgomea/regressor.py index 0f9e8e2ca..07bf3e607 100644 --- a/experiment/methods/gpgomea/regressor.py +++ b/experiment/methods/gpgomea/regressor.py @@ -1,7 +1,5 @@ -from sklearn.base import RegressorMixin - from pymgpg.sk import MGPGRegressor - +from sklearn.base import RegressorMixin est: RegressorMixin = MGPGRegressor( verbose=False, diff --git a/experiment/methods/gplearn/regressor.py b/experiment/methods/gplearn/regressor.py index 2b7c3564f..0fa2c4a02 100644 --- a/experiment/methods/gplearn/regressor.py +++ b/experiment/methods/gplearn/regressor.py @@ -1,40 +1,40 @@ -from gplearn.genetic import SymbolicRegressor import re +from gplearn.genetic import SymbolicRegressor + hyper_params = [] -for p, g in zip([1000,500,100],[500,1000,5000]): - for fs in [('add', 'sub', 'mul', 'div', 'log','sqrt'), - ('add', 'sub', 'mul', 'div', 'log','sqrt', 'sin','cos') - ]: - hyper_params.append({ - 'population_size' : [p], - 'generations' : [g], - 'function_set': [fs] - }) +for p, g in zip([1000, 500, 100], [500, 1000, 5000]): + for fs in [ + ("add", "sub", "mul", "div", "log", "sqrt"), + ("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos"), + ]: + hyper_params.append( + {"population_size": [p], "generations": [g], "function_set": [fs]} + ) est = SymbolicRegressor( - tournament_size=20, - init_depth=(2, 6), - init_method='half and half', - metric='mean absolute error', - parsimony_coefficient=0.001, - p_crossover=0.9, - p_subtree_mutation=0.01, - p_hoist_mutation=0.01, - p_point_mutation=0.01, - p_point_replace=0.05, - max_samples=1.0, - function_set= ('add', 'sub', 'mul', 'div', 'log', - 'sqrt', 'sin','cos'), - population_size=1000, - generations=500 - ) + tournament_size=20, + init_depth=(2, 6), + init_method="half and half", + metric="mean absolute error", + parsimony_coefficient=0.001, + p_crossover=0.9, + p_subtree_mutation=0.01, + p_hoist_mutation=0.01, + p_point_mutation=0.01, + p_point_replace=0.05, + max_samples=1.0, + function_set=("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos"), + population_size=1000, + generations=500, +) def model(est, X=None): # print(dir(est)) return str(est._program) + def complexity(est): - #TODO: check - return len(re.split(r'\(|,',model(est))) + # TODO: check + return len(re.split(r"\(|,", model(est))) diff --git a/experiment/methods/gpzgd/regressor.py b/experiment/methods/gpzgd/regressor.py index 6cb268f51..7583e94d1 100644 --- a/experiment/methods/gpzgd/regressor.py +++ b/experiment/methods/gpzgd/regressor.py @@ -13,24 +13,38 @@ """ -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_X_y, check_array, check_is_fitted -from sklearn.metrics import mean_squared_error, r2_score - import os -from tempfile import TemporaryDirectory import subprocess +from tempfile import TemporaryDirectory + import numpy as np import pandas as pd +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.metrics import mean_squared_error, r2_score +from sklearn.utils.validation import check_array, check_is_fitted, check_X_y + class GPZGD(BaseEstimator, RegressorMixin): - def __init__(self, pop_size=200, generations=250, - tournament_size=3, validation_prop=0.0, - crossover_rate=0.3, sub_mutation_rate=0.4, point_mutation_rate=0.3, mutation_sigma=0.1, - min_tree_init=2, max_tree_init=4, max_tree_nodes=50, - opset="ADD,SUB,MUL,SIN,ERC,VAR", - learning_rate=0.01, learning_epochs=3, timeout=0, random_state=-1): - """ Builds a Symbolic Regression using the cli interface of your algorithm. + def __init__( + self, + pop_size=200, + generations=250, + tournament_size=3, + validation_prop=0.0, + crossover_rate=0.3, + sub_mutation_rate=0.4, + point_mutation_rate=0.3, + mutation_sigma=0.1, + min_tree_init=2, + max_tree_init=4, + max_tree_nodes=50, + opset="ADD,SUB,MUL,SIN,ERC,VAR", + learning_rate=0.01, + learning_epochs=3, + timeout=0, + random_state=-1, + ): + """Builds a Symbolic Regression using the cli interface of your algorithm. Examples -------- >>> from gpzgd import GPZGD @@ -42,7 +56,7 @@ def __init__(self, pop_size=200, generations=250, """ self.validation_prop = validation_prop - + self.pop_size = pop_size self.generations = generations self.tournament_size = tournament_size @@ -55,7 +69,7 @@ def __init__(self, pop_size=200, generations=250, self.max_tree_init = max_tree_init self.max_tree_nodes = max_tree_nodes self.learning_rate = learning_rate - self.learning_epochs =learning_epochs + self.learning_epochs = learning_epochs self.timeout = timeout self.random_state = random_state @@ -79,12 +93,12 @@ def fit(self, X_train, y_train): # 2. validate the consistency of the data matrices and create a single 2D array with X and y X_train, y_train = check_X_y(X_train, y_train, accept_sparse=False) if len(y_train.shape) == 1: - Z_train = np.hstack((X_train, y_train[:,None])) + Z_train = np.hstack((X_train, y_train[:, None])) else: Z_train = np.hstack((X_train, y_train)) # 3. create a temp config file - cname = temp_dir + "/config" + cname = temp_dir + "/config" with open(cname, "w") as config_file: config_file.write(f"timeout={self.timeout}\n") config_file.write(f"validation_prop={self.validation_prop}\n") @@ -101,41 +115,63 @@ def fit(self, X_train, y_train): config_file.write(f"learning_rate={self.learning_rate}\n") config_file.write(f"learning_epochs={self.learning_epochs}\n") config_file.write(f"opset={self.opset}\n") - config_file.write(f"elitism_rate=1\n") - config_file.write(f"max_tree_depth=-1\n") - config_file.write(f"standardise=Y\n") - config_file.write(f"coef_op=Y\n") - + config_file.write("elitism_rate=1\n") + config_file.write("max_tree_depth=-1\n") + config_file.write("standardise=Y\n") + config_file.write("coef_op=Y\n") + # 4. create a temp file and store the data - fname = temp_dir + "/tmpdata" - np.savetxt(f"{fname}", Z_train, delimiter=" ", header=f"{Z_train.shape[0]} {Z_train.shape[1]}", comments="") + fname = temp_dir + "/tmpdata" + np.savetxt( + f"{fname}", + Z_train, + delimiter=" ", + header=f"{Z_train.shape[0]} {Z_train.shape[1]}", + comments="", + ) # 5. call your cli binary with the parameters cwd = os.path.dirname(os.path.realpath(__file__)) if self.random_state >= 0: - ans = subprocess.check_output([ "gpzgd_regressor", f"{fname}", f"{cname}", "-p", f"rng_seed={self.random_state}" ], cwd=cwd, universal_newlines=True) + ans = subprocess.check_output( + [ + "gpzgd_regressor", + f"{fname}", + f"{cname}", + "-p", + f"rng_seed={self.random_state}", + ], + cwd=cwd, + universal_newlines=True, + ) else: - ans = subprocess.check_output([ "gpzgd_regressor", f"{fname}", f"{cname}" ], cwd=cwd, universal_newlines=True) - - xbar, s, mdl, l, e = ans.split(";") - - self.xbar = eval(xbar) - self.s = eval(s) - self.expr = mdl - self.len = int(l) - self.cli_mse = float(e) - self.train_mse = mean_squared_error(y_train, self.eval_expr(X_train)) ## useful to compare with internal score! - self.score = r2_score(y_train, self.eval_expr(X_train)) - + ans = subprocess.check_output( + ["gpzgd_regressor", f"{fname}", f"{cname}"], + cwd=cwd, + universal_newlines=True, + ) + + xbar, s, mdl, length, e = ans.split(";") + + self.xbar = eval(xbar) + self.s = eval(s) + self.expr = mdl + self.len = int(length) + self.cli_mse = float(e) + self.train_mse = mean_squared_error( + y_train, self.eval_expr(X_train) + ) ## useful to compare with internal score! + self.score = r2_score(y_train, self.eval_expr(X_train)) + self.is_fitted_ = True return self def expr_str(self): return self.expr - + def eval_expr(self, X): - """ Evaluates the expression with data point x. We assume that the expression is compatible with numpy """ + """Evaluates the expression with data point x. We assume that the expression is compatible with numpy""" X = (X - self.xbar) / self.s y = eval(self.expr) @@ -145,7 +181,7 @@ def eval_expr(self, X): return y def predict(self, X_test, ic=None): - """ A reference implementation of a predicting function. + """A reference implementation of a predicting function. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) @@ -161,31 +197,33 @@ def predict(self, X_test, ic=None): ##### TODO: Reverse Transform the predictions return self.eval_expr(X_test) + est = GPZGD() -# want to tune your estimator? wrap it in a sklearn CV class. +# want to tune your estimator? wrap it in a sklearn CV class. + def model(est, X=None): """ - Return a sympy-compatible string of the final model. + Return a sympy-compatible string of the final model. Parameters ---------- est: sklearn regressor - The fitted model. + The fitted model. X: pd.DataFrame, default=None The training data. This argument can be dropped if desired. Returns ------- - A sympy-compatible string of the final model. + A sympy-compatible string of the final model. Notes ----- - Ensure that the variable names appearing in the model are identical to - those in the training data, `X`, which is a `pd.Dataframe`. - If your method names variables some other way, e.g. `[x_0 ... x_m]`, + Ensure that the variable names appearing in the model are identical to + those in the training data, `X`, which is a `pd.Dataframe`. + If your method names variables some other way, e.g. `[x_0 ... x_m]`, you can specify a mapping in the `model` function such as: ``` @@ -197,26 +235,27 @@ def model(est, X): ``` If you have special operators such as protected division or protected log, - you will need to handle these to assure they conform to sympy format. + you will need to handle these to assure they conform to sympy format. One option is to replace them with the unprotected versions. Post an issue - if you have further questions: + if you have further questions: https://github.com/cavalab/srbench/issues/new/choose """ model_str = est.expr_str() - model_str = model_str.replace("np.","").replace("[:,", "").replace("]", "") + model_str = model_str.replace("np.", "").replace("[:,", "").replace("]", "") # use python syntax for exponents - model_str = model_str.replace('^','**') + model_str = model_str.replace("^", "**") names = X.columns.tolist() if isinstance(X, pd.DataFrame) else None if names is not None: - mapping = { 'X'+str(i):k for i, k in enumerate(names) } + mapping = {"X" + str(i): k for i, k in enumerate(names)} for k, v in reversed(mapping.items()): - model_str = model_str.replace(k,v) + model_str = model_str.replace(k, v) return model_str + ################################################################################ # Optional Settings ################################################################################ @@ -248,22 +287,22 @@ def model(est, X): y: training labels. """ + def pre_train_fn(est, X, y): if len(X) <= 1000: pop_size = 500 generations = 5000 - max_time = 3564 ## 99% of one hour + max_time = 3564 ## 99% of one hour else: pop_size = 200 generations = 5000 - max_time = 35640 ## 99% of ten hours + max_time = 35640 ## 99% of ten hours est.tournament_size = 10 est.pop_size = pop_size est.generations = generations est.timeout = max_time - + + # define eval_kwargs. -eval_kwargs = { - "pre_train" : pre_train_fn -} +eval_kwargs = {"pre_train": pre_train_fn} diff --git a/experiment/methods/itea/regressor.py b/experiment/methods/itea/regressor.py index 329f99142..d4cd54fb4 100644 --- a/experiment/methods/itea/regressor.py +++ b/experiment/methods/itea/regressor.py @@ -1,83 +1,89 @@ -import sys import os + os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + import pyITEA as itea -from itertools import product hyper_params = [ { - 'exponents' : ((0,5),), - 'termlimit' : ((2,5),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((0, 5),), + "termlimit": ((2, 5),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((0,5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), - 'nonzeroexps' : (10,), + "exponents": ((0, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), + "nonzeroexps": (10,), }, { - 'exponents' : ((-5,5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Sin]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((-5, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Sin]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((-5, 5),), - 'termlimit' : ((2, 5),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((-5, 5),), + "termlimit": ((2, 5),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((0, 5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), + "exponents": ((0, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), }, { - 'exponents' : ((-5, 5),), - 'termlimit' : ((2, 15),), - 'transfunctions' : ('[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]',), - 'npop' : (1000,), - 'ngens' : (1000,), - 'nonzeroexps' : (10,), + "exponents": ((-5, 5),), + "termlimit": ((2, 15),), + "transfunctions": ("[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]",), + "npop": (1000,), + "ngens": (1000,), + "nonzeroexps": (10,), }, ] def pre_train(est, X, y): """Adjusting ngens to 500 if dataset is too large""" - + # adjust generations based onsize of X versus batch size if len(X) >= 10_000: est.ngens = 250 - print('ITEA ngens adjusted to', est.gens) + print("ITEA ngens adjusted to", est.gens) # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = itea.ITEARegressor(npop=1000, ngens=500, exponents=(-1, 1), termlimit=(2, - 2), nonzeroexps=1, - transfunctions= '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]' - ) +eval_kwargs = {"scale_x": False, "scale_y": False} +est = itea.ITEARegressor( + npop=1000, + ngens=500, + exponents=(-1, 1), + termlimit=(2, 2), + nonzeroexps=1, + transfunctions="[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]", +) + def complexity(e): return e.len + def model(e, X): # get sympy compatible - new_model = e.sympy.replace("^","**") - - new_model = new_model.replace('sqrtAbs','PSQRT') + new_model = e.sympy.replace("^", "**") + + new_model = new_model.replace("sqrtAbs", "PSQRT") + + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) - return new_model diff --git a/experiment/methods/keplearn/regressor.py b/experiment/methods/keplearn/regressor.py index 1b4081ceb..29ca24613 100644 --- a/experiment/methods/keplearn/regressor.py +++ b/experiment/methods/keplearn/regressor.py @@ -1,3 +1,16 @@ +from __future__ import annotations + +import json +import math as _math +import os +import re +import subprocess +from typing import Any + +import numpy as np +import sympy as sp +from sklearn.base import BaseEstimator, RegressorMixin + """SRBench wrapper for the Keplearn engine. Keplearn is a deterministic symbolic-regression engine written in Rust, built @@ -12,31 +25,32 @@ as `--timeout `; the engine stops gracefully at that budget and emits best-so-far. """ -import json -import os -import re -import subprocess - -import numpy as np -import sympy as sp -from sklearn.base import BaseEstimator, RegressorMixin # Path to the engine binary (installed by algorithms/keplearn/install.sh). BIN = os.environ.get("KEPLEARN_BIN", "/opt/conda/bin/keplearn") # numpy evaluators matching the engine's emitted grammar (same set as the harness). _NP = { - "sqrt": np.sqrt, "sin": np.sin, "cos": np.cos, "tan": np.tan, - "exp": np.exp, "log": np.log, "arcsin": np.arcsin, "arccos": np.arccos, - "tanh": np.tanh, "abs": np.abs, + "sqrt": np.sqrt, + "sin": np.sin, + "cos": np.cos, + "tan": np.tan, + "exp": np.exp, + "log": np.log, + "arcsin": np.arcsin, + "arccos": np.arccos, + "tanh": np.tanh, + "abs": np.abs, } class KeplearnRegressor(BaseEstimator, RegressorMixin): def __init__(self, max_time=60, sample=500, random_state=None, max_depth=None): - self.max_time = max_time # SRBench MAXTIME (seconds); overwritten by harness + self.max_time = max_time # SRBench MAXTIME (seconds); overwritten by harness self.sample = sample - self.random_state = random_state # engine is deterministic; seeds the holdout shuffle + self.random_state = ( + random_state # engine is deterministic; seeds the holdout shuffle + ) self.max_depth = max_depth def fit(self, X, y): @@ -55,9 +69,15 @@ def fit(self, X, y): tsv = header + "\n" + "\n".join(rows) + "\n" cmd = [ - BIN, "--target", "target", "--top-k", "1", - "--sample", str(int(self.sample)), - "--timeout", str(int(float(self.max_time) * 1000)), # seconds -> ms + BIN, + "--target", + "target", + "--top-k", + "1", + "--sample", + str(int(self.sample)), + "--timeout", + str(int(float(self.max_time) * 1000)), # seconds -> ms ] if self.random_state is not None: cmd += ["--seed", str(int(self.random_state))] @@ -67,12 +87,15 @@ def fit(self, X, y): # grace over the engine's own --timeout so the subprocess wrapper never # pre-empts the engine's graceful best-so-far emit. proc = subprocess.run( - cmd, input=tsv, capture_output=True, text=True, + cmd, + input=tsv, + capture_output=True, + text=True, timeout=float(self.max_time) + 30, ) out = proc.stdout or "" try: - res = json.loads(out[out.index("{"):]) + res = json.loads(out[out.index("{") :]) except Exception: res = {"results": []} results = res.get("results") or [] @@ -86,13 +109,19 @@ def _build_canonical(self): """Build ONE canonical SymPy expression from the engine string; predict, model() and complexity() all use this same object, so the predicting function and the reported model string cannot diverge.""" - raw = (self.model_str_.replace("arcsin", "asin") - .replace("arccos", "acos") - .replace("^", "**")) + raw = ( + self.model_str_.replace("arcsin", "asin") + .replace("arccos", "acos") + .replace("^", "**") + ) try: loc = {"abs": sp.Abs} - loc.update({f"x{i + 1}": sp.Symbol(f"x{i + 1}") - for i in range(self.n_features_in_)}) + loc.update( + { + f"x{i + 1}": sp.Symbol(f"x{i + 1}") + for i in range(self.n_features_in_) + } + ) expr = _normalize(sp.sympify(raw, locals=loc)) syms = [sp.Symbol(f"x{i + 1}") for i in range(self.n_features_in_)] self.expr_ = expr @@ -111,10 +140,14 @@ def predict(self, X): else: env = {f"x{i + 1}": X[:, i] for i in range(X.shape[1])} env.update(_NP) - v = eval(self.model_str_.replace("^", "**"), {"__builtins__": {}}, env) + v = eval( + self.model_str_.replace("^", "**"), {"__builtins__": {}}, env + ) except Exception: v = self.y_mean_ - v = np.asarray(np.broadcast_to(np.asarray(v, dtype=float), (X.shape[0],)), dtype=float) + v = np.asarray( + np.broadcast_to(np.asarray(v, dtype=float), (X.shape[0],)), dtype=float + ) # Sole deviation from evaluating the model expression verbatim: rows # where it is non-finite (poles, domain edges) fall back to the # training mean so downstream metric computation stays defined. @@ -124,7 +157,7 @@ def predict(self, X): est = KeplearnRegressor() # Deterministic engine: a single (empty) config, i.e. no hyperparameter tuning. -hyper_params = [{}] +hyper_params: list[dict[str, Any]] = [{}] # Raw features (our affine-monomial fit bakes constants; scaling x would perturb # them). fit consumes a plain ndarray. @@ -140,11 +173,10 @@ def predict(self, X): # genuine approximation's messy coefficients are left as-is (its wrong STRUCTURE keeps # it correctly rejected). Any failure falls back to the un-normalized string, so this # can never make the pipeline worse than before. -import math as _math -_SNAP_Q = (2, 3, 4, 5, 6) # simple-rational denominators considered -_SNAP_REL = 1e-7 # coefficient/exponent snap tolerance (relative) -_PI_REL = 1e-6 # transcendental (pi) recognition tolerance +_SNAP_Q = (2, 3, 4, 5, 6) # simple-rational denominators considered +_SNAP_REL = 1e-7 # coefficient/exponent snap tolerance (relative) +_PI_REL = 1e-6 # transcendental (pi) recognition tolerance def _snap_float(f): @@ -161,8 +193,12 @@ def _snap_float(f): p = round(v * q) if p != 0 and abs(v - p / q) <= _SNAP_REL * max(a, 1): return sp.Rational(p, q) - for base, sym in ((_math.pi, sp.pi), (_math.pi ** 2, sp.pi ** 2), - (1.0 / _math.pi, 1 / sp.pi), (1.0 / (_math.pi ** 2), 1 / sp.pi ** 2)): + for base, sym in ( + (_math.pi, sp.pi), + (_math.pi**2, sp.pi**2), + (1.0 / _math.pi, 1 / sp.pi), + (1.0 / (_math.pi**2), 1 / sp.pi**2), + ): k = v / base for q in (1, 2, 3, 4, 5, 6): p = round(k * q) @@ -199,7 +235,7 @@ def _fold_mul(m): # coef*p_d*z*sqrt(..)) that must stay out — folding it regressed those. f = sp.Mul(*nonrad) if f.free_symbols and f.free_symbols <= rad[0].free_symbols: - return sp.Mul(*const) * sp.sqrt(sp.expand(f ** 2 * rad[0])) + return sp.Mul(*const) * sp.sqrt(sp.expand(f**2 * rad[0])) return m return m @@ -244,7 +280,11 @@ def model(est, X=None): else: subs = {sp.Symbol(f"x{i + 1}"): sp.Symbol(names[i]) for i in range(n)} out = str(expr.subs(subs, simultaneous=True)) - return out if (out and out.lower() not in ("nan", "zoo", "oo")) else _string_model(est, names) + return ( + out + if (out and out.lower() not in ("nan", "zoo", "oo")) + else _string_model(est, names) + ) except Exception: return _string_model(est, names) @@ -274,5 +314,8 @@ def complexity(est): except Exception: pass raw = getattr(est, "model_str_", "0") - return len(re.findall( - r"\*\*|[+\-*/]|\b(?:sin|cos|tan|sqrt|exp|log|tanh|arcsin|arccos|abs)\b", raw)) + return len( + re.findall( + r"\*\*|[+\-*/]|\b(?:sin|cos|tan|sqrt|exp|log|tanh|arcsin|arccos|abs)\b", raw + ) + ) diff --git a/experiment/methods/lightgbm/regressor.py b/experiment/methods/lightgbm/regressor.py index 1960eee06..a8b88eb49 100644 --- a/experiment/methods/lightgbm/regressor.py +++ b/experiment/methods/lightgbm/regressor.py @@ -1,7 +1,6 @@ import lightgbm import numpy as np - # original set of hyperparameters (too many for the experimental design): # { # 'n_estimators' : (10, 100, 1000), @@ -13,21 +12,21 @@ hyper_params = [] for n_estimators in (10, 100, 1000): for learning_rate in (0.0001, 0.01): - for boosting_type in ('gbdt', 'dart', 'goss'): - hyper_params.append({ - 'n_estimators' : [n_estimators], - 'learning_rate' : [learning_rate], - 'boosting_type' : [boosting_type] - }) - -est=lightgbm.LGBMRegressor( - max_depth=6, - deterministic = True, - force_row_wise = True - ) + for boosting_type in ("gbdt", "dart", "goss"): + hyper_params.append( + { + "n_estimators": [n_estimators], + "learning_rate": [learning_rate], + "boosting_type": [boosting_type], + } + ) + +est = lightgbm.LGBMRegressor(max_depth=6, deterministic=True, force_row_wise=True) + def complexity(est): - c = np.sum([x['num_leaves'] for x in est._Booster.dump_model()['tree_info']]) + c = np.sum([x["num_leaves"] for x in est._Booster.dump_model()["tree_info"]]) return int(c) + model = None diff --git a/experiment/methods/nesymres/regressor.py b/experiment/methods/nesymres/regressor.py index ee1db048e..98c966a05 100644 --- a/experiment/methods/nesymres/regressor.py +++ b/experiment/methods/nesymres/regressor.py @@ -1,23 +1,20 @@ -import time -import json -import os, sys +from __future__ import annotations -import torch -import numpy as np -from sklearn import feature_selection - -from nesymres.architectures.model import Model -from nesymres.dclasses import FitParams, BFGSParams from functools import partial -import sympy as sp -from sympy import lambdify -import omegaconf - from tempfile import TemporaryDirectory +from typing import Any +import numpy as np +import omegaconf +import sympy as sp +import torch +from nesymres.architectures.model import Model +from nesymres.dclasses import BFGSParams, FitParams +from sklearn import feature_selection from sklearn.base import BaseEstimator, RegressorMixin +from sympy import lambdify -eq_setting = { +eq_setting: dict[str, Any] = { "config": { "max_len": 20, "positive": True, @@ -27,13 +24,9 @@ "int_base": 10, "precision": 10, "rewrite_functions": "", - "variables": [ - "x_1", - "x_2", - "x_3" - ], + "variables": ["x_1", "x_2", "x_3"], "eos_index": 1, - "pad_index": 0 + "pad_index": 0, }, "total_coefficients": [ "cm_0", @@ -115,127 +108,119 @@ "ca_36", "ca_37", "ca_38", - "ca_39" - ], - "total_variables": [ - "x_1", - "x_2", - "x_3" + "ca_39", ], + "total_variables": ["x_1", "x_2", "x_3"], "word2id": { - "0": 28, - "1": 29, - "2": 30, - "3": 31, - "4": 32, - "5": 33, - "x_1": 4, - "x_2": 5, - "x_3": 6, - "abs": 7, - "acos": 8, - "add": 9, - "asin": 10, - "atan": 11, - "cos": 12, - "cosh": 13, - "coth": 14, - "div": 15, - "exp": 16, - "ln": 17, - "mul": 18, - "pow": 19, - "sin": 20, - "sinh": 21, - "sqrt": 22, - "tan": 23, - "tanh": 24, - "-3": 25, - "-2": 26, - "-1": 27, - "P": 0, - "S": 1, - "F": 2, - "c": 3 + "0": 28, + "1": 29, + "2": 30, + "3": 31, + "4": 32, + "5": 33, + "x_1": 4, + "x_2": 5, + "x_3": 6, + "abs": 7, + "acos": 8, + "add": 9, + "asin": 10, + "atan": 11, + "cos": 12, + "cosh": 13, + "coth": 14, + "div": 15, + "exp": 16, + "ln": 17, + "mul": 18, + "pow": 19, + "sin": 20, + "sinh": 21, + "sqrt": 22, + "tan": 23, + "tanh": 24, + "-3": 25, + "-2": 26, + "-1": 27, + "P": 0, + "S": 1, + "F": 2, + "c": 3, }, "id2word": { - "1": "S", - "2": "F", - "3": "c", - "4": "x_1", - "5": "x_2", - "6": "x_3", - "7": "abs", - "8": "acos", - "9": "add", - "10": "asin", - "11": "atan", - "12": "cos", - "13": "cosh", - "14": "coth", - "15": "div", - "16": "exp", - "17": "ln", - "18": "mul", - "19": "pow", - "20": "sin", - "21": "sinh", - "22": "sqrt", - "23": "tan", - "24": "tanh", - "25": "-3", - "26": "-2", - "27": "-1", - "28": "0", - "29": "1", - "30": "2", - "31": "3", - "32": "4", - "33": "5" + "1": "S", + "2": "F", + "3": "c", + "4": "x_1", + "5": "x_2", + "6": "x_3", + "7": "abs", + "8": "acos", + "9": "add", + "10": "asin", + "11": "atan", + "12": "cos", + "13": "cosh", + "14": "coth", + "15": "div", + "16": "exp", + "17": "ln", + "18": "mul", + "19": "pow", + "20": "sin", + "21": "sinh", + "22": "sqrt", + "23": "tan", + "24": "tanh", + "25": "-3", + "26": "-2", + "27": "-1", + "28": "0", + "29": "1", + "30": "2", + "31": "3", + "32": "4", + "33": "5", }, "una_ops": [ - "asin", - "cos", - "exp", - "ln", - "pow2", - "pow3", - "pow4", - "pow5", - "sin", - "sqrt", - "tan" + "asin", + "cos", + "exp", + "ln", + "pow2", + "pow3", + "pow4", + "pow5", + "sin", + "sqrt", + "tan", ], "bin_ops": [ - "asin", - "cos", - "exp", - "ln", - "pow2", - "pow3", - "pow4", - "pow5", - "sin", - "sqrt", - "tan" + "asin", + "cos", + "exp", + "ln", + "pow2", + "pow3", + "pow4", + "pow5", + "sin", + "sqrt", + "tan", ], "rewrite_functions": [], "total_number_of_eqs": 200, "eqs_per_hdf": 200, "generator_details": { - "max_len": 20, - "operators": "add:10,mul:10,sub:5,div:5,sqrt:4,pow2:4,pow3:2,pow4:1,pow5:1,ln:4,exp:4,sin:4,cos:4,tan:4,asin:2", - "max_ops": 5, - "rewrite_functions": "", - "variables": [ - "x_1", - "x_2", - "x_3" - ], - "eos_index": 1, - "pad_index": 0 + "max_len": 20, + "operators": "add:10,mul:10,sub:5,div:5,sqrt:4,pow2:4,pow3:2,pow4:1,pow5:1,ln:4,exp:4,sin:4,cos:4,tan:4,asin:2", + "max_ops": 5, + "rewrite_functions": "", + "variables": ["x_1", "x_2", "x_3"], + "eos_index": 1, + "pad_index": 0, }, - "unique_index": None + "unique_index": None, } cfg = None @@ -355,14 +340,14 @@ sweep: dir: runs/${dataset_train.predict_c}/${now:%Y-%m-%d}/${now:%H-%M-%S} """) - + cfg = omegaconf.OmegaConf.load(cname) def get_top_k_features(X, y, k=10): - if y.ndim==2: - y=y[:,0] - if X.shape[1]<=k: + if y.ndim == 2: + y = y[:, 0] + if X.shape[1] <= k: return [i for i in range(X.shape[1])] else: kbest = feature_selection.SelectKBest(feature_selection.r_regression, k=k) @@ -372,8 +357,9 @@ def get_top_k_features(X, y, k=10): print("keeping only the top-{} features. Order was {}".format(k, top_features)) return list(top_features[:k]) + def get_variables(equation): - """ Parse all free variables in equations and return them in + """Parse all free variables in equations and return them in lexicographic order""" expr = sp.parse_expr(equation) @@ -398,10 +384,10 @@ def get_variables(equation): def evaluate_func(func_str, vars_list, X): assert X.ndim == 2 - assert len(set(vars_list)) == len(vars_list), 'Duplicates in vars_list!' + assert len(set(vars_list)) == len(vars_list), "Duplicates in vars_list!" order_list = vars_list - indeces = [int(x[2:])-1 for x in order_list] + indeces = [int(x[2:]) - 1 for x in order_list] if not order_list: # Empty order list. Constant function predicted @@ -412,8 +398,7 @@ def evaluate_func(func_str, vars_list, X): # that are not in the ground-truth equation X_padded = np.zeros((X.shape[0], len(vars_list))) - - X_padded[:, :X.shape[1]] = X[:,:X_padded.shape[1]] + X_padded[:, : X.shape[1]] = X[:, : X_padded.shape[1]] # Subselect columns of X that corrspond to provided variables X_subsel = X_padded[:, indeces] @@ -423,19 +408,18 @@ def evaluate_func(func_str, vars_list, X): return f(*X_subsel.T) -n_restarts = [1, 10, 20] +n_restarts = [1, 10, 20] total_number_of_eqs = [100, 250, 500] -max_ops = [10, 20, 50] +max_ops = [10, 20, 50] hyper_params = [] for restarts, eqs in zip(n_restarts, total_number_of_eqs): for ops in max_ops: - hyper_params.append({ - 'n_restarts' : [restarts], - 'total_number_of_eqs' : [eqs], - 'max_ops' : [ops] - }) - + hyper_params.append( + {"n_restarts": [restarts], "total_number_of_eqs": [eqs], "max_ops": [ops]} + ) + + class NeSymResRegressor(BaseEstimator, RegressorMixin): def __init__(self, n_restarts=10, total_number_of_eqs=200, max_ops=5): self.n_restarts = n_restarts @@ -443,14 +427,16 @@ def __init__(self, n_restarts=10, total_number_of_eqs=200, max_ops=5): self.max_ops = max_ops def fit(self, X, y): + if cfg is None: + raise RuntimeError("NeSymRes configuration has not been loaded") # Setting parameters cfg.inference.bfgs.n_restarts = self.n_restarts - eq_setting['total_number_of_eqs'] = self.total_number_of_eqs - eq_setting['config']['max_ops'] = self.max_ops + eq_setting["total_number_of_eqs"] = self.total_number_of_eqs + eq_setting["config"]["max_ops"] = self.max_ops ## Set up BFGS load rom the hydra config yaml bfgs = BFGSParams( - activated= cfg.inference.bfgs.activated, + activated=cfg.inference.bfgs.activated, n_restarts=cfg.inference.bfgs.n_restarts, add_coefficients_if_not_existing=cfg.inference.bfgs.add_coefficients_if_not_existing, normalization_o=cfg.inference.bfgs.normalization_o, @@ -463,40 +449,41 @@ def fit(self, X, y): self.top_k_features = get_top_k_features(X, y, k=3) X = X[:, self.top_k_features] - eq_setting["total_variables"] = [f"x_{i+1}" for i in range(X.shape[1])] + eq_setting["total_variables"] = [f"x_{i + 1}" for i in range(X.shape[1])] eq_setting["num_variables"] = int(X.shape[1]) - eq_setting["variables"] = [f"x_{i+1}" for i in range(X.shape[1])] + eq_setting["variables"] = [f"x_{i + 1}" for i in range(X.shape[1])] print(eq_setting["total_variables"]) - params_fit = FitParams(word2id=eq_setting["word2id"], - id2word={int(k): v for k,v in eq_setting["id2word"].items()}, - una_ops=eq_setting["una_ops"], - bin_ops=eq_setting["bin_ops"], - total_variables=eq_setting["total_variables"], - total_coefficients=eq_setting["total_coefficients"], - rewrite_functions=eq_setting["rewrite_functions"], - bfgs=bfgs, - beam_size=cfg.inference.beam_size #This parameter is a tradeoff between accuracy and fitting time - ) + params_fit = FitParams( + word2id=eq_setting["word2id"], + id2word={int(k): v for k, v in eq_setting["id2word"].items()}, + una_ops=eq_setting["una_ops"], + bin_ops=eq_setting["bin_ops"], + total_variables=eq_setting["total_variables"], + total_coefficients=eq_setting["total_coefficients"], + rewrite_functions=eq_setting["rewrite_functions"], + bfgs=bfgs, + beam_size=cfg.inference.beam_size, # This parameter is a tradeoff between accuracy and fitting time + ) weights_path = "/srbench_pretrained/nesymres_100M.ckpt" ## Load architecture, set into eval mode, and pass the config parameters - #cfg.architecture.num_features = eq_setting["num_variables"] - + # cfg.architecture.num_features = eq_setting["num_variables"] + model = Model.load_from_checkpoint(weights_path, cfg=cfg.architecture) model.eval() - if torch.cuda.is_available(): + if torch.cuda.is_available(): model.cuda() fitfunc = partial(model.fitfunc, cfg_params=params_fit) - output_ref = fitfunc(X,y) + output_ref = fitfunc(X, y) self.model_skeleton_ = output_ref - self.model_eq_ = model.get_equation() + self.model_eq_ = model.get_equation() print("NeSymReS output ref: ", output_ref) print("NeSymReS Equation Skeleton: ", self.model_skeleton_) @@ -505,19 +492,19 @@ def fit(self, X, y): self.pred_variables_ = get_variables(self.model_eq_[0]) return self - def predict(self, X): X = X[:, self.top_k_features] - return evaluate_func(self.model_eq_[0], - self.pred_variables_, X) + return evaluate_func(self.model_eq_[0], self.pred_variables_, X) est = NeSymResRegressor() + def model(est, X=None): return str(est.model_eq_[0]) + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/operon/regressor.py b/experiment/methods/operon/regressor.py index bbe40bef7..bab3e1852 100644 --- a/experiment/methods/operon/regressor.py +++ b/experiment/methods/operon/regressor.py @@ -1,17 +1,17 @@ import copy -import numpy as np -import optuna import os -import pandas as pd import time +import numpy as np +import optuna +import pandas as pd import pyoperon as Operon from pyoperon.sklearn import SymbolicRegressor +from sklearn.base import BaseEstimator, RegressorMixin from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error from sklearn.model_selection import KFold -from sklearn.base import BaseEstimator, RegressorMixin -from sympy import lambdify, parse_expr, Symbol +from sympy import Symbol, lambdify, parse_expr def compute_sigma(X, y): @@ -26,9 +26,13 @@ def compute_mdl(tree, X, y): problem.TrainingRange = Operon.Range(0, X.shape[0]) target = max(ds.Variables, key=lambda x: x.Index) problem.Target = target - problem.InputHashes = [ v.Hash for v in sorted(ds.Variables, key=lambda x: x.Index) if v.Hash != target.Hash ] + problem.InputHashes = [ + v.Hash + for v in sorted(ds.Variables, key=lambda x: x.Index) + if v.Hash != target.Hash + ] dtable = Operon.DispatchTable() - mdl_eval = Operon.MinimumDescriptionLengthEvaluator(problem, dtable, 'gauss') + mdl_eval = Operon.MinimumDescriptionLengthEvaluator(problem, dtable, "gauss") mdl_eval.Sigma = [compute_sigma(X, y)] rng = Operon.RomuTrio(1234) ind = Operon.Individual() @@ -53,13 +57,13 @@ class SympyExprModel(RegressorMixin): Constructor which builds a lambda from the expression string using the specified variable names. """ + def __init__(self, expr_str, variable_names): - self.model = expr_str + self.model = expr_str self.variable_names = variable_names - self.symbols = [Symbol(x) for x in variable_names] + self.symbols = [Symbol(x) for x in variable_names] self.best_estimator = lambdify(self.symbols, parse_expr(expr_str)) - self.is_fitted = True - + self.is_fitted = True def predict(self, X): values = X.values if isinstance(X, pd.DataFrame) else X @@ -77,43 +81,52 @@ def __init__(self, X, y, n_folds, seed, fixed_params={}, suggest_callback=None): self.params = fixed_params self.suggest_callback = suggest_callback - def __call__(self, trial): - hyper_params = {} if self.suggest_callback is None else self.suggest_callback(trial) + hyper_params = ( + {} if self.suggest_callback is None else self.suggest_callback(trial) + ) params = copy.deepcopy(self.params) params.update(hyper_params) - params.update({'random_state': self.seed}) + params.update({"random_state": self.seed}) self.reg = SymbolicRegressor(**params) X_, y_ = np.asarray(self.X), np.asarray(self.y) - if self.n_folds > 1: kf = KFold(n_splits=self.n_folds) score = 0 for train, test in kf.split(X_, y_): X_train, y_train = X_[train, :], y_[train] - X_test, y_test = X_[test, :], y_[test] + X_test, y_test = X_[test, :], y_[test] self.reg.fit(X_train, y_train) - best = min(self.reg.pareto_front_, key=lambda x: x['minimum_description_length']) - score += compute_mdl(best['tree'], X_test, y_test) - - return score/self.n_folds + best = min( + self.reg.pareto_front_, + key=lambda x: x["minimum_description_length"], + ) + score += compute_mdl(best["tree"], X_test, y_test) + return score / self.n_folds self.reg.fit(X_, y_) - return min((x['minimum_description_length'] for x in self.reg.pareto_front_)) - + return min((x["minimum_description_length"] for x in self.reg.pareto_front_)) def callback(self, study, trial): if self.best_reg is None or study.best_trial == trial: self.best_reg = self.reg - class OperonOptuna(BaseEstimator, RegressorMixin): - def __init__(self, name, n_folds=3, n_trials=100, max_time=3600, fixed_params={}, suggest_callback=None, random_state=None): + def __init__( + self, + name, + n_folds=3, + n_trials=100, + max_time=3600, + fixed_params={}, + suggest_callback=None, + random_state=None, + ): self.name = name self.n_folds = n_folds self.n_trials = n_trials @@ -122,69 +135,97 @@ def __init__(self, name, n_folds=3, n_trials=100, max_time=3600, fixed_params={} self.fixed_params = fixed_params self.suggest_callback = suggest_callback self.best_estimator = None # to hold the best estimator - self.best_parameters = None # to hold the best parameters - + self.best_parameters = None # to hold the best parameters def fit(self, X, y): t0 = time.time() sErr = compute_sigma(X, y) fixed_params = copy.deepcopy(self.fixed_params) - fixed_params.update({'uncertainty': [sErr]}) + fixed_params.update({"uncertainty": [sErr]}) t1 = time.time() - curr_time = np.ceil(t1-t0) - - objective = Objective(X, y, self.n_folds, fixed_params=fixed_params, suggest_callback=self.suggest_callback, seed=self.random_state) - study = optuna.create_study(study_name=self.name, direction='minimize') - study.optimize(objective, n_trials=self.n_trials, callbacks=[objective.callback], timeout=(self.max_time - curr_time)) - self.best_estimator = objective.best_reg + curr_time = np.ceil(t1 - t0) + + objective = Objective( + X, + y, + self.n_folds, + fixed_params=fixed_params, + suggest_callback=self.suggest_callback, + seed=self.random_state, + ) + study = optuna.create_study(study_name=self.name, direction="minimize") + study.optimize( + objective, + n_trials=self.n_trials, + callbacks=[objective.callback], + timeout=(self.max_time - curr_time), + ) + self.best_estimator = objective.best_reg + if self.best_estimator is None: + raise RuntimeError("Operon optimization produced no fitted estimator") self.best_parameters = self.best_estimator.get_params() - self.best_parameters['completed trials'] = len(study.trials) + self.best_parameters["completed trials"] = len(study.trials) # save the best solution as a SympyExprModel best = self.best_estimator - names = X.columns.tolist() if isinstance(X, pd.DataFrame) else list(best.variables_.values()) - model_string = lambda x: best.get_model_string(x, 6, names).replace('^', '**') + names = ( + X.columns.tolist() + if isinstance(X, pd.DataFrame) + else list(best.variables_.values()) + ) - self.best_solution = SympyExprModel(model_string(best.model_), names) - self.model = self.best_solution.model - self.population = [SympyExprModel(model_string(x['tree']), names) for x in best.pareto_front_[:100]] + def model_string(x): + return best.get_model_string(x, 6, names).replace("^", "**") + self.best_solution = SympyExprModel(model_string(best.model_), names) + self.model = self.best_solution.model + self.population = [ + SympyExprModel(model_string(x["tree"]), names) + for x in best.pareto_front_[:100] + ] def predict(self, X): + if self.best_estimator is None: + raise RuntimeError("Operon must be fitted before prediction") return self.best_estimator.predict(X) -num_threads = int(os.environ['OMP_NUM_THREADS']) if 'OMP_NUM_THREADS' in os.environ else 1 +num_threads = ( + int(os.environ["OMP_NUM_THREADS"]) if "OMP_NUM_THREADS" in os.environ else 1 +) default_params = { - 'offspring_generator': 'basic', - 'initialization_method': 'btc', - 'n_threads': num_threads, - 'objectives': ['r2', 'length'], - 'epsilon': 1e-5, - 'optimizer_iterations': 1, - 'random_state': None, - 'reinserter': 'keep-best', - 'max_evaluations': int(1e6), - 'female_selector' : 'tournament', - 'male_selector': 'tournament', - 'brood_size': 5, - 'population_size': 1000, - 'pool_size': None, - 'max_time': 300, - 'model_selection_criterion': 'minimum_description_length', - } + "offspring_generator": "basic", + "initialization_method": "btc", + "n_threads": num_threads, + "objectives": ["r2", "length"], + "epsilon": 1e-5, + "optimizer_iterations": 1, + "random_state": None, + "reinserter": "keep-best", + "max_evaluations": int(1e6), + "female_selector": "tournament", + "male_selector": "tournament", + "brood_size": 5, + "population_size": 1000, + "pool_size": None, + "max_time": 300, + "model_selection_criterion": "minimum_description_length", +} def suggest_params(trial): return { - 'allowed_symbols' : trial.suggest_categorical('allowed_symbols', [ - 'add,sub,mul,div,constant,variable', - 'add,sub,mul,div,sin,cos,constant,variable', - 'add,sub,mul,div,exp,log,constant,variable', - 'add,sub,mul,div,sin,cos,exp,log,sqrt,pow,constant,variable' - ]), - 'max_length' : trial.suggest_int('max_length', low=10, high=50, step=10), + "allowed_symbols": trial.suggest_categorical( + "allowed_symbols", + [ + "add,sub,mul,div,constant,variable", + "add,sub,mul,div,sin,cos,constant,variable", + "add,sub,mul,div,exp,log,constant,variable", + "add,sub,mul,div,sin,cos,exp,log,sqrt,pow,constant,variable", + ], + ), + "max_length": trial.suggest_int("max_length", low=10, high=50, step=10), } @@ -193,7 +234,7 @@ def model(est, X=None): def complexity(est): - return est.best_estimator.stats_['model_complexity'] + return est.best_estimator.stats_["model_complexity"] def get_best_solution(est): @@ -204,7 +245,13 @@ def get_population(est): return [est] if isinstance(est, SympyExprModel) else est.population -est = OperonOptuna('operon-optuna', n_folds=1, n_trials=100, fixed_params=default_params, suggest_callback=suggest_params) +est = OperonOptuna( + "operon-optuna", + n_folds=1, + n_trials=100, + fixed_params=default_params, + suggest_callback=suggest_params, +) ################################################################################ # Optional Settings @@ -243,7 +290,7 @@ def pre_train_fn(est, X, y): # pass the function to eval_kwargs eval_kwargs = { - 'pre_train': pre_train_fn, - 'test_params': {'max_time': 60 }, - 'use_dataframe' : False, + "pre_train": pre_train_fn, + "test_params": {"max_time": 60}, + "use_dataframe": False, } diff --git a/experiment/methods/pantara/regressor.py b/experiment/methods/pantara/regressor.py index 1bd966425..bd15c37e3 100644 --- a/experiment/methods/pantara/regressor.py +++ b/experiment/methods/pantara/regressor.py @@ -25,18 +25,29 @@ block is silently dropped — predict() returns partial sum. """ -import os -import sys -import numpy as np -import warnings +from __future__ import annotations -import torch +from typing import Any + +import numpy as np import pantara +import torch from pantara import ( - OracleClassifier, N_CLASSES, DEVICE, FAMILIES, N_POINTS, MIN_SCORE, - best_per_family_multispace, make_state_stats, make_oracle_input, - fit_trig, detect_and_fit_power_law, bloc_score, power_law_bloc_score, - transform_space, inverse_transform_y, corr_np, standardize, + DEVICE, + FAMILIES, + MIN_SCORE, + N_CLASSES, + N_POINTS, + OracleClassifier, + best_per_family_multispace, + bloc_score, + detect_and_fit_power_law, + fit_trig, + inverse_transform_y, + make_oracle_input, + make_state_stats, + power_law_bloc_score, + transform_space, ) _MODEL_PATH = pantara.MODEL_PATH @@ -46,10 +57,11 @@ # Each recipe is a dict with a 'type' key and enough state to call predict(). # ───────────────────────────────────────────────────────────────────────────── + def _predict_power_law(block, X): """y = C · x0^n0 · x1^n1 · ...""" eps = 1e-10 - coeffs = block['coeffs'] # (D+1,): [log C, n1, n2, ...] + coeffs = block["coeffs"] # (D+1,): [log C, n1, n2, ...] N = len(X) log_X = np.log(np.abs(X) + eps) A = np.column_stack([np.ones(N), log_X]) @@ -59,36 +71,36 @@ def _predict_power_law(block, X): def _predict_trig(block, X): """y = A · sin/cos(ω·xj + φ) + C (or + B·xj for linear variant)""" - j = block['j_var'] - x = X[:, j] - A = block['A'] - w = block['omega'] - phi = block['phi'] - C = block['C'] - form = block['form'] - if form == 'sin': + j = block["j_var"] + x = X[:, j] + A = block["A"] + w = block["omega"] + phi = block["phi"] + C = block["C"] + form = block["form"] + if form == "sin": return A * np.sin(w * x + phi) + C - elif form == 'cos': + elif form == "cos": return A * np.cos(w * x + phi) + C - elif form == 'sin+lin': - B = block.get('B', 0.0) + elif form == "sin+lin": + B = block.get("B", 0.0) return A * np.sin(w * x + phi) + B * x + C else: # cos+lin - B = block.get('B', 0.0) + B = block.get("B", 0.0) return A * np.cos(w * x + phi) + B * x + C def _predict_1d_linear(block, X): """Replay a 1-D OLS block on new X.""" - space = block['space'] - family = block['family'] - indices = block['indices'] # tuple of feature indices - mn_c = block['mn_c'] - iqr_c = block['iqr_c'] - mn_r = block['mn_r'] - iqr_r = block['iqr_r'] - w = block['w'] - b = block['b'] + space = block["space"] + family = block["family"] + indices = block["indices"] # tuple of feature indices + mn_c = block["mn_c"] + iqr_c = block["iqr_c"] + mn_r = block["mn_r"] + iqr_r = block["iqr_r"] + w = block["w"] + b = block["b"] # Dummy y must be positive: transform_space rejects y<=0 for log_y/log_log # (it would return valid=False and make this block silently predict 0). @@ -101,24 +113,24 @@ def _predict_1d_linear(block, X): if col_new is None: return np.zeros(len(X)) - col_n = (col_new - mn_c) / (iqr_c + 1e-10) - pred_t = (w * col_n + b) * iqr_r + mn_r + col_n = (col_new - mn_c) / (iqr_c + 1e-10) + pred_t = (w * col_n + b) * iqr_r + mn_r return inverse_transform_y(pred_t, space) def _predict_multi_linear(block, X): """Replay a multivariate OLS block (log-space) on new X.""" - space = block['space'] - mu_x = block['mu_x'] - s_x = block['s_x'] - mu_r = block['mu_r'] - s_r = block['s_r'] - W = block['W'] - b = block['b'] - - if space == 'log_log_multi': - eps = 1e-10 - X_t = np.log(np.abs(X) + eps) + space = block["space"] + mu_x = block["mu_x"] + s_x = block["s_x"] + mu_r = block["mu_r"] + s_r = block["s_r"] + W = block["W"] + b = block["b"] + + if space == "log_log_multi": + eps = 1e-10 + X_t = np.log(np.abs(X) + eps) else: # Dummy y must be positive (see _predict_1d_linear): transform_space # rejects y<=0 for log_y/log_log, which would zero out this block. @@ -128,18 +140,18 @@ def _predict_multi_linear(block, X): # Pad or truncate to match stored dimension D_stored = len(mu_x) - D_new = X_t.shape[1] + D_new = X_t.shape[1] if D_new < D_stored: X_t = np.column_stack([X_t, np.zeros((len(X_t), D_stored - D_new))]) elif D_new > D_stored: X_t = X_t[:, :D_stored] - X_std = (X_t - mu_x) / (s_x + 1e-10) + X_std = (X_t - mu_x) / (s_x + 1e-10) pred_s = X_std @ W + b pred_t = pred_s * s_r + mu_r - if space == 'log_log_multi': - return np.exp(pred_t) # Fix B from v7d + if space == "log_log_multi": + return np.exp(pred_t) # Fix B from v7d return inverse_transform_y(pred_t, space) @@ -147,30 +159,35 @@ def _predict_multi_linear(block, X): # FEATURE HELPERS # ───────────────────────────────────────────────────────────────────────────── + def _compute_feature_col(family, indices, X_t): j = indices[0] D = X_t.shape[1] if j >= D: return None - if family == 'lin': + if family == "lin": return X_t[:, j].copy() - elif family == 'sq': + elif family == "sq": return X_t[:, j] ** 2 - elif family == 'pair': + elif family == "pair": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] * X_t[:, k] - elif family == 'qc': + elif family == "qc": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] * X_t[:, k] ** 2 - elif family == 'trip': - k, l = indices[1], indices[2] - if k >= D or l >= D: return None - return X_t[:, j] * X_t[:, k] * X_t[:, l] - elif family == 'qinv': + elif family == "trip": + k, third = indices[1], indices[2] + if k >= D or third >= D: + return None + return X_t[:, j] * X_t[:, k] * X_t[:, third] + elif family == "qinv": k = indices[1] - if k >= D: return None + if k >= D: + return None return X_t[:, j] ** 2 / (X_t[:, k] + 1e-10) return None @@ -181,35 +198,37 @@ def _find_feature_indices(family, col, X_t): def match(c): # Compare in standardized form (shape-invariant) - c_s = c - c.mean(); c_n = c_s / (np.std(c_s) + 1e-10) - co_s = col - col.mean(); co_n = co_s / (np.std(co_s) + 1e-10) + c_s = c - c.mean() + c_n = c_s / (np.std(c_s) + 1e-10) + co_s = col - col.mean() + co_n = co_s / (np.std(co_s) + 1e-10) return float(np.mean((c_n - co_n) ** 2)) < 1e-4 - if family == 'lin': + if family == "lin": for j in range(D): if match(X_t[:, j]): return (j,) - elif family == 'sq': + elif family == "sq": for j in range(D): if match(X_t[:, j] ** 2): return (j,) - elif family == 'pair': + elif family == "pair": for j in range(D): for k in range(j + 1, D): if match(X_t[:, j] * X_t[:, k]): return (j, k) - elif family == 'qc': + elif family == "qc": for j in range(D): for k in range(D): if j != k and match(X_t[:, j] * X_t[:, k] ** 2): return (j, k) - elif family == 'trip': + elif family == "trip": for j in range(D): for k in range(j + 1, D): - for l in range(k + 1, D): - if match(X_t[:, j] * X_t[:, k] * X_t[:, l]): - return (j, k, l) - elif family == 'qinv': + for third in range(k + 1, D): + if match(X_t[:, j] * X_t[:, k] * X_t[:, third]): + return (j, k, third) + elif family == "qinv": for j in range(D): for k in range(D): if j != k and np.all(np.abs(X_t[:, k]) > 1e-10): @@ -220,12 +239,12 @@ def match(c): def _fit_ols_1d(col_t, resid_t): """OLS linear fit in normalized space. Returns (mn_c, iqr_c, mn_r, iqr_r, w, b).""" - mn_c = float(np.median(col_t)) + mn_c = float(np.median(col_t)) iqr_c = float(np.percentile(col_t, 75) - np.percentile(col_t, 25)) + 1e-10 - mn_r = float(np.median(resid_t)) + mn_r = float(np.median(resid_t)) iqr_r = float(np.percentile(resid_t, 75) - np.percentile(resid_t, 25)) + 1e-10 - col_n = (col_t - mn_c) / iqr_c + col_n = (col_t - mn_c) / iqr_c resid_n = (resid_t - mn_r) / iqr_r A = np.column_stack([col_n, np.ones(len(col_n))]) @@ -236,13 +255,13 @@ def _fit_ols_1d(col_t, resid_t): def _fit_ols_multi(X_t, resid_t): """OLS multivariate fit. Returns (mu_x, s_x, mu_r, s_r, W, b).""" - N, D = X_t.shape - mu_x = X_t.mean(axis=0) - s_x = X_t.std(axis=0) + 1e-10 - X_std = (X_t - mu_x) / s_x + N, D = X_t.shape + mu_x = X_t.mean(axis=0) + s_x = X_t.std(axis=0) + 1e-10 + X_std = (X_t - mu_x) / s_x - mu_r = float(resid_t.mean()) - s_r = float(resid_t.std()) + 1e-10 + mu_r = float(resid_t.mean()) + s_r = float(resid_t.std()) + 1e-10 resid_s = (resid_t - mu_r) / s_r A = np.column_stack([X_std, np.ones(N)]) @@ -256,6 +275,7 @@ def _fit_ols_multi(X_t, resid_t): # PANTARA REGRESSOR # ───────────────────────────────────────────────────────────────────────────── + class PantaraRegressor: """ Pantara v7d — Matching-pursuit symbolic regression. @@ -273,7 +293,7 @@ class PantaraRegressor: """ def __init__(self, max_time=3600, random_state=42): - self.max_time = max_time # set by SRBench harness + self.max_time = max_time # set by SRBench harness self.random_state = random_state # ── helpers ────────────────────────────────────────────────────────────── @@ -294,14 +314,15 @@ def _load_oracle(self): def _to_numpy(self, X): """Accept DataFrame or ndarray, return float64 ndarray (N, D).""" import pandas as pd + if isinstance(X, pd.DataFrame): self.feature_names_ = list(X.columns) return X.values.astype(np.float64) arr = np.asarray(X, dtype=np.float64) if arr.ndim == 1: arr = arr.reshape(-1, 1) - if not hasattr(self, 'feature_names_') or self.feature_names_ is None: - self.feature_names_ = [f'x{j}' for j in range(arr.shape[1])] + if not hasattr(self, "feature_names_") or self.feature_names_ is None: + self.feature_names_ = [f"x{j}" for j in range(arr.shape[1])] return arr # ── fit ────────────────────────────────────────────────────────────────── @@ -309,14 +330,14 @@ def _to_numpy(self, X): def fit(self, X, y): np.random.seed(self.random_state) - X_np = self._to_numpy(X) - y_np = np.asarray(y, dtype=np.float64).ravel() + X_np = self._to_numpy(X) + y_np = np.asarray(y, dtype=np.float64).ravel() self.fitted_blocks_ = [] - self.chosen_ = [] + self.chosen_ = [] oracle = self._load_oracle() - resid = y_np.copy() + resid = y_np.copy() for _step in range(5): ratio = resid.std() / (y_np.std() + 1e-10) @@ -324,7 +345,7 @@ def fit(self, X, y): break scores, best_config, _ = best_per_family_multispace(X_np, resid, y_np) - stats = make_state_stats(scores, resid, y_np, X_np) + stats = make_state_stats(scores, resid, y_np, X_np) points = make_oracle_input(X_np, resid, y_np, N_POINTS) ranked, probs = oracle.predict_ranked(points, stats) @@ -334,13 +355,15 @@ def fit(self, X, y): if pl is not None: pred_pl, coeffs_pl, _ = pl new_resid = resid - pred_pl - quality = power_law_bloc_score(resid, new_resid, y_np) + quality = power_law_bloc_score(resid, new_resid, y_np) if quality > 0.30: - self.fitted_blocks_.append({ - 'type': 'power_law', - 'coeffs': coeffs_pl.copy(), - }) - self.chosen_.append('power_law[log_log_lstsq]') + self.fitted_blocks_.append( + { + "type": "power_law", + "coeffs": coeffs_pl.copy(), + } + ) + self.chosen_.append("power_law[log_log_lstsq]") resid = new_resid if quality > 0.80: break @@ -350,15 +373,15 @@ def fit(self, X, y): actions = list(ranked[:4]) # bonus lin[original] - lin_idx = FAMILIES.index('lin') - sc_l, sp_l, _, _, _ = best_config['lin'] - if sp_l == 'original' and sc_l >= 0.85 and lin_idx not in actions: + lin_idx = FAMILIES.index("lin") + sc_l, sp_l, _, _, _ = best_config["lin"] + if sp_l == "original" and sc_l >= 0.85 and lin_idx not in actions: actions.append(lin_idx) # bonus pair[original] - pair_idx = FAMILIES.index('pair') - sc_p, sp_p, _, _, _ = best_config['pair'] - if sp_p == 'original' and sc_p >= 0.85 and pair_idx not in actions: + pair_idx = FAMILIES.index("pair") + sc_p, sp_p, _, _, _ = best_config["pair"] + if sp_p == "original" and sc_p >= 0.85 and pair_idx not in actions: actions.append(pair_idx) candidates = [] @@ -371,38 +394,46 @@ def fit(self, X, y): continue # ── trig: use specialized estimator ───────────────────── - if fam in ('sin', 'cos'): + if fam in ("sin", "cos"): try: - omega_init = float(sp.split('_w')[-1]) + omega_init = float(sp.split("_w")[-1]) except Exception: omega_init = None try: - j_var = int(sp.split('_x')[1].split('_')[0]) + j_var = int(sp.split("_x")[1].split("_")[0]) except Exception: j_var = 0 new_resid, pred_orig, trig_info = fit_trig( - X_np[:, j_var], resid, y_np, + X_np[:, j_var], + resid, + y_np, omega_candidates=[omega_init] if omega_init else None, try_linear=False, steps=600, ) sc_after, _, _ = best_per_family_multispace(X_np, new_resid, y_np) - quality = bloc_score(resid, new_resid, y_np, sc_after) - clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) + quality = bloc_score(resid, new_resid, y_np, sc_after) + clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) combined = quality * (0.6 + 0.4 * clf_w) - candidates.append({ - 'family': fam, 'space': sp, 'action': action, - 'quality': quality, 'combined': combined, - 'new_resid': new_resid, 'pred': pred_orig, - 'trig_info': trig_info, - 'j_var': j_var, - }) + candidates.append( + { + "family": fam, + "space": sp, + "action": action, + "quality": quality, + "combined": combined, + "new_resid": new_resid, + "pred": pred_orig, + "trig_info": trig_info, + "j_var": j_var, + } + ) continue # ── non-trig: run original pipeline step ───────────────── # Determine if multifeature applies use_multi = ( - sp in ('log_log', 'log_x', 'log_y', 'log_log_multi') + sp in ("log_log", "log_x", "log_y", "log_log_multi") and X_t is not None and X_t.ndim == 2 and X_t.shape[1] > 1 @@ -411,20 +442,24 @@ def fit(self, X, y): # Reproduce prediction (same as fit_block in v7d) if use_multi: mu_x, s_x, mu_r, s_r, W, b = _fit_ols_multi(X_t, resid_t) - X_std = (X_t - mu_x) / s_x - pred_s = X_std @ W + b - pred_t = pred_s * s_r + mu_r - if sp == 'log_log_multi': + X_std = (X_t - mu_x) / s_x + pred_s = X_std @ W + b + pred_t = pred_s * s_r + mu_r + if sp == "log_log_multi": pred_orig = np.exp(pred_t) else: pred_orig = inverse_transform_y(pred_t, sp) new_resid = resid - pred_orig block_store = { - 'type': 'multi_linear', - 'family': fam, 'space': sp, - 'mu_x': mu_x, 's_x': s_x, - 'mu_r': mu_r, 's_r': s_r, - 'W': W, 'b': b, + "type": "multi_linear", + "family": fam, + "space": sp, + "mu_x": mu_x, + "s_x": s_x, + "mu_r": mu_r, + "s_r": s_r, + "W": W, + "b": b, } else: # 1-D feature block @@ -432,67 +467,77 @@ def fit(self, X, y): if indices is None: continue mn_c, iqr_c, mn_r, iqr_r, w, b = _fit_ols_1d(col, resid_t) - col_n = (col - mn_c) / iqr_c - pred_t = (w * col_n + b) * iqr_r + mn_r + col_n = (col - mn_c) / iqr_c + pred_t = (w * col_n + b) * iqr_r + mn_r pred_orig = inverse_transform_y(pred_t, sp) new_resid = resid - pred_orig block_store = { - 'type': '1d_linear', - 'family': fam, 'space': sp, - 'indices': indices, - 'mn_c': mn_c, 'iqr_c': iqr_c, - 'mn_r': mn_r, 'iqr_r': iqr_r, - 'w': w, 'b': b, + "type": "1d_linear", + "family": fam, + "space": sp, + "indices": indices, + "mn_c": mn_c, + "iqr_c": iqr_c, + "mn_r": mn_r, + "iqr_r": iqr_r, + "w": w, + "b": b, } sc_after, _, _ = best_per_family_multispace(X_np, new_resid, y_np) - quality = bloc_score(resid, new_resid, y_np, sc_after) - clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) + quality = bloc_score(resid, new_resid, y_np, sc_after) + clf_w = float(probs[action]) / (float(probs[ranked[0]]) + 1e-10) combined = quality * (0.6 + 0.4 * clf_w) # Bonuses (identical to v7d) - if sp == 'original' and sc >= 0.97: + if sp == "original" and sc >= 0.97: combined *= 1.5 - elif fam == 'lin' and sp == 'original' and sc >= 0.85: + elif fam == "lin" and sp == "original" and sc >= 0.85: combined *= 1.3 - elif fam == 'pair' and sp == 'original' and sc >= 0.85: + elif fam == "pair" and sp == "original" and sc >= 0.85: combined *= 1.2 - candidates.append({ - 'family': fam, 'space': sp, 'action': action, - 'quality': quality, 'combined': combined, - 'new_resid': new_resid, 'pred': pred_orig, - 'block_store': block_store, - }) + candidates.append( + { + "family": fam, + "space": sp, + "action": action, + "quality": quality, + "combined": combined, + "new_resid": new_resid, + "pred": pred_orig, + "block_store": block_store, + } + ) if not candidates: break - best = max(candidates, key=lambda c: c['combined']) - if best['combined'] < MIN_SCORE: + best = max(candidates, key=lambda c: c["combined"]) + if best["combined"] < MIN_SCORE: break # Commit this block - if best['family'] in ('sin', 'cos'): - ti = best['trig_info'] + if best["family"] in ("sin", "cos"): + ti = best["trig_info"] blk = { - 'type': 'trig', - 'j_var': best['j_var'], - 'form': ti.get('form', best['family']), - 'A': ti.get('A', 0.0), - 'omega': ti.get('omega', 0.0), - 'phi': ti.get('phi', 0.0), - 'C': ti.get('C', 0.0), - 'B': ti.get('B', 0.0), + "type": "trig", + "j_var": best["j_var"], + "form": ti.get("form", best["family"]), + "A": ti.get("A", 0.0), + "omega": ti.get("omega", 0.0), + "phi": ti.get("phi", 0.0), + "C": ti.get("C", 0.0), + "B": ti.get("B", 0.0), } self.fitted_blocks_.append(blk) else: - self.fitted_blocks_.append(best['block_store']) + self.fitted_blocks_.append(best["block_store"]) self.chosen_.append(f"{best['family']}[{best['space']}]") - resid = best['new_resid'] + resid = best["new_resid"] - if best['quality'] > 0.80: + if best["quality"] > 0.80: break return self @@ -500,24 +545,24 @@ def fit(self, X, y): # ── predict ────────────────────────────────────────────────────────────── def predict(self, X): - if not hasattr(self, 'fitted_blocks_') or not self.fitted_blocks_: + if not hasattr(self, "fitted_blocks_") or not self.fitted_blocks_: # No blocks fitted — return zeros (will produce R²<0 on test) - n = len(X) if hasattr(X, '__len__') else 1 + n = len(X) if hasattr(X, "__len__") else 1 return np.zeros(n) X_np = self._to_numpy(X) - acc = np.zeros(len(X_np)) + acc = np.zeros(len(X_np)) for block in self.fitted_blocks_: - t = block['type'] + t = block["type"] try: - if t == 'power_law': + if t == "power_law": acc += _predict_power_law(block, X_np) - elif t == 'trig': + elif t == "trig": acc += _predict_trig(block, X_np) - elif t == '1d_linear': + elif t == "1d_linear": acc += _predict_1d_linear(block, X_np) - elif t == 'multi_linear': + elif t == "multi_linear": acc += _predict_multi_linear(block, X_np) except Exception: pass # drop silently — partial sum is still meaningful @@ -527,7 +572,7 @@ def predict(self, X): # ── sklearn interface ───────────────────────────────────────────────────── def get_params(self, deep=True): - return {'max_time': self.max_time, 'random_state': self.random_state} + return {"max_time": self.max_time, "random_state": self.random_state} def set_params(self, **params): for k, v in params.items(): @@ -535,7 +580,7 @@ def set_params(self, **params): return self def __repr__(self): - chosen = getattr(self, 'chosen_', []) + chosen = getattr(self, "chosen_", []) return f"PantaraRegressor(blocks={chosen})" @@ -545,12 +590,12 @@ def __repr__(self): est = PantaraRegressor(max_time=3600, random_state=42) -hyper_params = [{}] # no hyperparameter search for now +hyper_params: list[dict[str, Any]] = [{}] # no hyperparameter search for now # Skip StandardScaler: Pantara needs raw (positive) physical values. eval_kwargs = { - 'scale_x': False, - 'scale_y': False, + "scale_x": False, + "scale_y": False, } @@ -568,11 +613,11 @@ def model(est, X=None): """ import sympy as sp - blocks = getattr(est, 'fitted_blocks_', []) + blocks = getattr(est, "fitted_blocks_", []) if not blocks: return None - f_names = getattr(est, 'feature_names_', None) + f_names = getattr(est, "feature_names_", None) if (not f_names) and X is not None: try: f_names = list(X.columns) @@ -582,23 +627,23 @@ def model(est, X=None): def feat(j): if f_names is not None and j < len(f_names): return sp.Symbol(str(f_names[j])) - return sp.Symbol(f'x{j}') + return sp.Symbol(f"x{j}") def xt(j, space): """Feature j after the block's input-space transform (cf. transform_space).""" x = feat(j) - if space in ('log_x', 'log_log', 'log_log_multi'): + if space in ("log_x", "log_log", "log_log_multi"): return sp.log(x) - if space == 'inv_x': + if space == "inv_x": return 1 / x # original, log_y, sq_y leave X untouched return x def inv_y(expr, space): """Invert the y-space transform (cf. inverse_transform_y).""" - if space in ('log_y', 'log_log', 'log_log_multi'): + if space in ("log_y", "log_log", "log_log_multi"): return sp.exp(expr) - if space == 'sq_y': + if space == "sq_y": return sp.sqrt(sp.Abs(expr)) # original, log_x, inv_x return expr @@ -606,69 +651,76 @@ def inv_y(expr, space): def feature_col(family, indices, space): """Symbolic column for a 1-D block family (cf. _compute_feature_col).""" xj = xt(indices[0], space) - if family == 'lin': + if family == "lin": return xj - if family == 'sq': - return xj ** 2 - if family == 'pair': + if family == "sq": + return xj**2 + if family == "pair": return xj * xt(indices[1], space) - if family == 'qc': + if family == "qc": return xj * xt(indices[1], space) ** 2 - if family == 'trip': + if family == "trip": return xj * xt(indices[1], space) * xt(indices[2], space) - if family == 'qinv': - return xj ** 2 / xt(indices[1], space) + if family == "qinv": + return xj**2 / xt(indices[1], space) return None def block_expr(b): - t = b['type'] + t = b["type"] - if t == 'power_law': - coeffs = b['coeffs'] + if t == "power_law": + coeffs = b["coeffs"] expr = sp.Float(float(np.exp(coeffs[0]))) for j, n in enumerate(coeffs[1:]): if abs(float(n)) > 1e-4: expr = expr * feat(j) ** sp.Float(float(n)) return expr - if t == 'trig': - x = feat(b['j_var']) - A = sp.Float(float(b['A'])); w = sp.Float(float(b['omega'])) - phi = sp.Float(float(b['phi'])); C = sp.Float(float(b['C'])) - B = sp.Float(float(b.get('B', 0.0))) + if t == "trig": + x = feat(b["j_var"]) + A = sp.Float(float(b["A"])) + w = sp.Float(float(b["omega"])) + phi = sp.Float(float(b["phi"])) + C = sp.Float(float(b["C"])) + B = sp.Float(float(b.get("B", 0.0))) arg = w * x + phi - form = b['form'] - if form == 'sin': + form = b["form"] + if form == "sin": return A * sp.sin(arg) + C - if form == 'cos': + if form == "cos": return A * sp.cos(arg) + C - if form == 'sin+lin': + if form == "sin+lin": return A * sp.sin(arg) + B * x + C - if form == 'cos+lin': + if form == "cos+lin": return A * sp.cos(arg) + B * x + C return C - if t == '1d_linear': - space = b['space'] - col = feature_col(b['family'], b['indices'], space) + if t == "1d_linear": + space = b["space"] + col = feature_col(b["family"], b["indices"], space) if col is None: return sp.Integer(0) - col_n = (col - sp.Float(float(b['mn_c']))) / sp.Float(float(b['iqr_c']) + 1e-10) - pred_t = (sp.Float(float(b['w'])) * col_n + sp.Float(float(b['b']))) \ - * sp.Float(float(b['iqr_r'])) + sp.Float(float(b['mn_r'])) + col_n = (col - sp.Float(float(b["mn_c"]))) / sp.Float( + float(b["iqr_c"]) + 1e-10 + ) + pred_t = ( + sp.Float(float(b["w"])) * col_n + sp.Float(float(b["b"])) + ) * sp.Float(float(b["iqr_r"])) + sp.Float(float(b["mn_r"])) return inv_y(pred_t, space) - if t == 'multi_linear': - space = b['space'] - mu_x, s_x, W = b['mu_x'], b['s_x'], b['W'] + if t == "multi_linear": + space = b["space"] + mu_x, s_x, W = b["mu_x"], b["s_x"], b["W"] n_feats = len(f_names) if f_names is not None else len(W) - pred_s = sp.Float(float(b['b'])) + pred_s = sp.Float(float(b["b"])) for j in range(len(W)): # predict() zero-pads transformed features beyond the input width x_t = xt(j, space) if j < n_feats else sp.Integer(0) - x_std = (x_t - sp.Float(float(mu_x[j]))) / sp.Float(float(s_x[j]) + 1e-10) + x_std = (x_t - sp.Float(float(mu_x[j]))) / sp.Float( + float(s_x[j]) + 1e-10 + ) pred_s = pred_s + sp.Float(float(W[j])) * x_std - pred_t = pred_s * sp.Float(float(b['s_r'])) + sp.Float(float(b['mu_r'])) + pred_t = pred_s * sp.Float(float(b["s_r"])) + sp.Float(float(b["mu_r"])) return inv_y(pred_t, space) return sp.Integer(0) @@ -687,4 +739,4 @@ def block_expr(b): def complexity(est): """Number of basis blocks chosen.""" - return len(getattr(est, 'fitted_blocks_', [])) + return len(getattr(est, "fitted_blocks_", [])) diff --git a/experiment/methods/pir/regressor.py b/experiment/methods/pir/regressor.py index 3993db917..de48c25fc 100644 --- a/experiment/methods/pir/regressor.py +++ b/experiment/methods/pir/regressor.py @@ -25,17 +25,15 @@ import numpy as np -from sklearn.base import BaseEstimator, RegressorMixin - # The classical engine. algorithms/pir/install.sh pip-installs this from the # public repo at tag v3.5. from physics_engine.sklearn_adapter import PIRRegressor - +from sklearn.base import BaseEstimator, RegressorMixin # --- Vanilla config that produced the verified blind Tier A result ------------ _PIR_VANILLA_CONFIG = dict( - enforce_dimensions=False, # blind sweep ran with dim-filter OFF - allowed_powers=[1, 2], # powers 1,2 only (structural cap) + enforce_dimensions=False, # blind sweep ran with dim-filter OFF + allowed_powers=[1, 2], # powers 1,2 only (structural cap) include_pairwise_products=True, # pairwise on; no 3-var assembly use_ransac=True, use_residual=True, @@ -122,20 +120,18 @@ def model(est, X=None): unchanged. Otherwise generic positional names (x_0, x0, X0, ...) are remapped to the dataset columns. """ - expr = ( - est.model() - if hasattr(est, "model") - else str(getattr(est, "expr_", "0.0")) - ) + expr = est.model() if hasattr(est, "model") else str(getattr(est, "expr_", "0.0")) if X is None or not hasattr(X, "columns"): return expr cols = list(X.columns) + # Word-boundary match, not substring: a column literally named "co" # must not match inside "cos(x_0)" (per gAldeia's review on PR #210). def _col_appears(col_name, s): return re.search(r"\b" + re.escape(str(col_name)) + r"\b", s) is not None + # If any real column name already appears, assume names are correct. if any(_col_appears(c, expr) for c in cols): return expr diff --git a/experiment/methods/ps-tree/regressor.py b/experiment/methods/ps-tree/regressor.py index b47c17d6d..5c9e8d4c6 100644 --- a/experiment/methods/ps-tree/regressor.py +++ b/experiment/methods/ps-tree/regressor.py @@ -1,10 +1,10 @@ -from sklearn.tree import DecisionTreeRegressor -from deap.tools import selRandom -from pstree.cluster_gp_sklearn import selTournamentDCD -from pstree.cluster_gp_sklearn import GPRegressor, PSTreeRegressor -from pstree.complexity_utils import tree_gp_regressor_complexity +from __future__ import annotations + +from typing import Any -from sklearn.model_selection import ParameterSampler +from pstree.cluster_gp_sklearn import GPRegressor, PSTreeRegressor, selTournamentDCD +from pstree.complexity_utils import tree_gp_regressor_complexity +from sklearn.tree import DecisionTreeRegressor # hyper_params = list(ParameterSampler({ # 'basic_primitive': ['optimal', 'log,sqrt,sin,tanh'], @@ -14,41 +14,56 @@ # I ran the code above and got the hyper-parameters below. # I am hardcoding it now so it is explicit here. -# I also had to wrap the parameters in lists, and it seems that there is a +# I also had to wrap the parameters in lists, and it seems that there is a # missconfiguration in initial_height hyper_params = [ - {'max_leaf_nodes': [4], - # 'initial_height': [None], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [8], - # 'initial_height': ['0-6'], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [6], - # 'initial_height': ['0-6'], - 'basic_primitive': ['optimal']}, - - {'max_leaf_nodes': [6], - # 'initial_height': ['0-6'], - 'basic_primitive': ['log,sqrt,sin,tanh']}, - - {'max_leaf_nodes': [8], - # 'initial_height': [None], - 'basic_primitive': ['optimal']}, - - {'max_leaf_nodes': [8], - # 'initial_height': [None], - 'basic_primitive': ['log,sqrt,sin,tanh']} + { + "max_leaf_nodes": [4], + # 'initial_height': [None], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': ['0-6'], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [6], + # 'initial_height': ['0-6'], + "basic_primitive": ["optimal"], + }, + { + "max_leaf_nodes": [6], + # 'initial_height': ['0-6'], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': [None], + "basic_primitive": ["optimal"], + }, + { + "max_leaf_nodes": [8], + # 'initial_height': [None], + "basic_primitive": ["log,sqrt,sin,tanh"], + }, ] # for g in hyper_params: # g['select'] = [selRandom] if g['initial_height'] is None else [selTournamentDCD] -est = PSTreeRegressor(regr_class=GPRegressor, tree_class=DecisionTreeRegressor, - height_limit=6, n_pop=25, n_gen=1_000, - normalize=True, basic_primitive='optimal', - select=selTournamentDCD, size_objective=False, afp=True) +est = PSTreeRegressor( + regr_class=GPRegressor, + tree_class=DecisionTreeRegressor, + height_limit=6, + n_pop=25, + n_gen=1_000, + normalize=True, + basic_primitive="optimal", + select=selTournamentDCD, + size_objective=False, + afp=True, +) def complexity(est): @@ -60,4 +75,4 @@ def model(e, X=None): return e.model() -eval_kwargs = {} +eval_kwargs: dict[str, Any] = {} diff --git a/experiment/methods/pysr/regressor.py b/experiment/methods/pysr/regressor.py index 9067021cd..2cda0e39c 100644 --- a/experiment/methods/pysr/regressor.py +++ b/experiment/methods/pysr/regressor.py @@ -1,30 +1,30 @@ from pysr import PySRRegressor -from multiprocessing import cpu_count hyper_params = [ { - 'niterations' : [1_000_000_000], - 'maxsize' : [20], - 'maxdepth' : [20], + "niterations": [1_000_000_000], + "maxsize": [20], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [30], - 'maxdepth' : [20], + "niterations": [1_000_000], + "maxsize": [30], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [50], - 'maxdepth' : [20], + "niterations": [1_000_000], + "maxsize": [50], + "maxdepth": [20], }, { - 'niterations' : [1_000_000], - 'maxsize' : [30], - 'maxdepth' : [20], - 'population_size' : [200], + "niterations": [1_000_000], + "maxsize": [30], + "maxdepth": [20], + "population_size": [200], }, ] + def complexity(est): return int(est.get_best()["complexity"]) @@ -37,11 +37,11 @@ def model(est, X=None): niterations=1_000_000_000, ncyclesperiteration=2_500, population_size=100, - populations=2, # originally max(15, 2*n_cpu), but srbench runs on a single core + populations=2, # originally max(15, 2*n_cpu), but srbench runs on a single core verbosity=0, # budget 10 minutes for compile time, # ensuring we can finish within 2 hours: - timeout_in_seconds=60*60 - 10*60, + timeout_in_seconds=60 * 60 - 10 * 60, maxsize=30, maxdepth=20, binary_operators=["+", "-", "*", "/"], @@ -53,7 +53,7 @@ def model(est, X=None): log=9, sqrt=9, ), - **{"/": (-1, 9)} + **{"/": (-1, 9)}, }, nested_constraints=dict( sin=dict( @@ -72,10 +72,10 @@ def model(est, X=None): ), sqrt=dict( sqrt=0, - ) + ), ), # prefer multiprocessing: - procs=1, # cpu_count(), + procs=1, # cpu_count(), multithreading=False, batching=True, batch_size=50, diff --git a/experiment/methods/qlattice/regressor.py b/experiment/methods/qlattice/regressor.py index 8d7dd7b88..13ab18f6a 100644 --- a/experiment/methods/qlattice/regressor.py +++ b/experiment/methods/qlattice/regressor.py @@ -4,7 +4,12 @@ import numpy as np import pandas as pd from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_X_y, check_array, check_is_fitted, _check_sample_weight +from sklearn.utils.validation import ( + _check_sample_weight, + check_array, + check_is_fitted, + check_X_y, +) from sympy.printing.printer import Printer @@ -13,26 +18,27 @@ class InternalTimeOutException(Exception): def alarm_handler(signum, frame): - print(f"raising InternalTimeOutException") + print("raising InternalTimeOutException") raise InternalTimeOutException -def auto_run_time(ql, - data, - output_name, - kind="regression", - stypes=None, - n_epochs=200, - threads="auto", - max_complexity=10, - query_string=None, - loss_function=None, - criterion="wide_parsimony", - sample_weights=None, - function_names=None, - starting_models=None, - max_time=None - ): +def auto_run_time( + ql, + data, + output_name, + kind="regression", + stypes=None, + n_epochs=200, + threads="auto", + max_complexity=10, + query_string=None, + loss_function=None, + criterion="wide_parsimony", + sample_weights=None, + function_names=None, + starting_models=None, + max_time=None, +): if max_time: signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(max_time) @@ -84,7 +90,7 @@ def auto_run_time(ql, best = feyn.get_diverse_models(models) except InternalTimeOutException: - print('InternalTimeOutException raised') + print("InternalTimeOutException raised") best = feyn.get_diverse_models(models) return best @@ -92,21 +98,21 @@ def auto_run_time(ql, def numpy_check_to_DataFrame(X, y=None): if isinstance(X, np.ndarray): - columns = [f'x__{i}' for i in range(X.shape[1])] + columns = [f"x__{i}" for i in range(X.shape[1])] X = pd.DataFrame(X, columns=columns) if y is None: return X elif isinstance(y, np.ndarray): if y.ndim == 1: - y = pd.Series(y, name='target') + y = pd.Series(y, name="target") elif y.ndim == 2: - y = pd.DataFrame(y, columns=['target']) + y = pd.DataFrame(y, columns=["target"]) return X, y def feature_names(X, y=None): - if hasattr(X, 'columns'): + if hasattr(X, "columns"): input_names = X.columns else: input_names = None @@ -114,7 +120,7 @@ def feature_names(X, y=None): if y is None: return input_names else: - if hasattr(y, 'name'): + if hasattr(y, "name"): output_name = y.name else: output_name = None @@ -122,21 +128,22 @@ def feature_names(X, y=None): class QLatticeRegressor(BaseEstimator, RegressorMixin): - def __init__(self, - kind='regression', - stypes=None, - n_epochs=200, - threads='auto', - max_complexity=10, - query_string=None, - loss_function=None, - criterion='wide_parsimony', - sample_weights=None, - function_names=None, - starting_models=None, - random_state=None, - max_time=None - ): + def __init__( + self, + kind="regression", + stypes=None, + n_epochs=200, + threads="auto", + max_complexity=10, + query_string=None, + loss_function=None, + criterion="wide_parsimony", + sample_weights=None, + function_names=None, + starting_models=None, + random_state=None, + max_time=None, + ): self.kind = kind self.stypes = stypes self.n_epochs = n_epochs @@ -177,22 +184,23 @@ def fit(self, X, y, sample_weight=None): sample_weight = _check_sample_weight(sample_weight, X) ql = feyn.QLattice(random_seed=rseed) - self.models_ = auto_run_time(ql=ql, - data=data, - output_name=data.columns[-1], - kind=self.kind, - stypes=self.stypes, - n_epochs=self.n_epochs, - threads=self.threads, - max_complexity=self.max_complexity, - query_string=self.query_string, - loss_function=self.loss_function, - criterion=self.criterion, - sample_weights=sample_weight, - function_names=self.function_names, - starting_models=self.starting_models, - max_time=self.max_time - ) + self.models_ = auto_run_time( + ql=ql, + data=data, + output_name=data.columns[-1], + kind=self.kind, + stypes=self.stypes, + n_epochs=self.n_epochs, + threads=self.threads, + max_complexity=self.max_complexity, + query_string=self.query_string, + loss_function=self.loss_function, + criterion=self.criterion, + sample_weights=sample_weight, + function_names=self.function_names, + starting_models=self.starting_models, + max_time=self.max_time, + ) return self def predict(self, X, n=0): @@ -225,10 +233,10 @@ def score(self, X, y, n=0): est = QLatticeRegressor( - kind='regression', + kind="regression", n_epochs=200, max_complexity=10, - criterion='wide_parsimony', + criterion="wide_parsimony", ) @@ -241,7 +249,7 @@ def model(est, X): mapping = {feyn.tools._sympy.get_sanitized_name(col): col for col in X.columns} for k, v in reversed(mapping.items()): string_model = string_model.replace(k, v) - except: + except Exception: pass return string_model @@ -291,6 +299,7 @@ def pre_train_fn(est, X, y): # define eval_kwargs. eval_kwargs = dict( pre_train=pre_train_fn, - test_params={'n_epochs': 2, - } + test_params={ + "n_epochs": 2, + }, ) diff --git a/experiment/methods/rils-rols/regressor.py b/experiment/methods/rils-rols/regressor.py index 87cc92e22..7eff29ad6 100644 --- a/experiment/methods/rils-rols/regressor.py +++ b/experiment/methods/rils-rols/regressor.py @@ -1,43 +1,50 @@ +import sympy as sp from rils_rols.rils_rols import RILSROLSRegressor +from sklearn.base import RegressorMixin from symbolic_utils import complexity as cplx -from sklearn.base import BaseEstimator, RegressorMixin -import sympy as sp -eval_kwargs = {'scale_x': False, 'scale_y': False} +eval_kwargs = {"scale_x": False, "scale_y": False} complexity_penalty = [0.001, 0.01, 0.1] -max_complexity = [25, 50, 100] +max_complexity = [25, 50, 100] hyper_params = [] for c in complexity_penalty: for max_c in max_complexity: - hyper_params.append({ - 'complexity_penalty' : [c], - 'max_complexity' : [max_c], - }) - -est:RegressorMixin = RILSROLSRegressor( - max_time=60*60, - max_fit_calls=1000*1000, + hyper_params.append( + { + "complexity_penalty": [c], + "max_complexity": [max_c], + } + ) + +est: RegressorMixin = RILSROLSRegressor( + max_time=60 * 60, + max_fit_calls=1000 * 1000, max_complexity=50, sample_size=0, - verbose=False) + verbose=False, +) + def model(est, X=None) -> str: if X is None: return str(est.model_string()) else: - mapping = {'x'+str(i):k for i,k in enumerate(X.columns)} + mapping = {"x" + str(i): k for i, k in enumerate(X.columns)} new_model = str(est.model_string()) - for k,v in reversed(mapping.items()): - new_model = new_model.replace(k,v) + for k, v in reversed(mapping.items()): + new_model = new_model.replace(k, v) return new_model + def complexity(est): return cplx(sp.parse_expr(str(est.model_string()))) - + + def get_population(est) -> list[RegressorMixin]: return [est] + def get_best_solution(est) -> RegressorMixin: - return est \ No newline at end of file + return est diff --git a/experiment/methods/sklearn/regressor.py b/experiment/methods/sklearn/regressor.py index 303f1e66d..30ae1faf3 100644 --- a/experiment/methods/sklearn/regressor.py +++ b/experiment/methods/sklearn/regressor.py @@ -2,7 +2,6 @@ from sklearn.linear_model import LinearRegression - est = LinearRegression() hyper_params = [{"fit_intercept": (True, False)}] diff --git a/experiment/methods/sklearn_adaboost/regressor.py b/experiment/methods/sklearn_adaboost/regressor.py index 6e6de6254..667c760bc 100644 --- a/experiment/methods/sklearn_adaboost/regressor.py +++ b/experiment/methods/sklearn_adaboost/regressor.py @@ -1,16 +1,29 @@ from sklearn import ensemble -hyper_params = [{ - 'learning_rate' : (0.01, 0.1, 1.0, 10.0,), - 'n_estimators' : (10, 100, 1000,), -}] +hyper_params = [ + { + "learning_rate": ( + 0.01, + 0.1, + 1.0, + 10.0, + ), + "n_estimators": ( + 10, + 100, + 1000, + ), + } +] + +est = ensemble.AdaBoostRegressor() -est=ensemble.AdaBoostRegressor() def complexity(est): - size = 0 + size = 0 for i in est.estimators_: size += i.tree_.node_count return size + model = None diff --git a/experiment/methods/sklearn_lasso/regressor.py b/experiment/methods/sklearn_lasso/regressor.py index f47db4dfd..477fc0567 100644 --- a/experiment/methods/sklearn_lasso/regressor.py +++ b/experiment/methods/sklearn_lasso/regressor.py @@ -2,11 +2,17 @@ hyper_params = [ { - 'alpha': (1e-04,0.001,0.01,0.1,1,), + "alpha": ( + 1e-04, + 0.001, + 0.01, + 0.1, + 1, + ), }, ] -est=linear_model.LassoLars() +est = linear_model.LassoLars() complexity = None model = None diff --git a/experiment/methods/sklearn_linear/regressor.py b/experiment/methods/sklearn_linear/regressor.py index c35b2f599..354b0106c 100644 --- a/experiment/methods/sklearn_linear/regressor.py +++ b/experiment/methods/sklearn_linear/regressor.py @@ -1,12 +1,12 @@ from sklearn import linear_model hyper_params = [ - { - 'fit_intercept': (True,), - }, + { + "fit_intercept": (True,), + }, ] -est=linear_model.LinearRegression() +est = linear_model.LinearRegression() complexity = None model = None diff --git a/experiment/methods/sklearn_mlp/regressor.py b/experiment/methods/sklearn_mlp/regressor.py index 992685076..7107724b9 100644 --- a/experiment/methods/sklearn_mlp/regressor.py +++ b/experiment/methods/sklearn_mlp/regressor.py @@ -1,17 +1,33 @@ -from sklearn.neural_network import MLPRegressor import numpy as np +from sklearn.neural_network import MLPRegressor hyper_params = [ { - 'activation' : ('logistic', 'tanh', 'relu',), - 'solver' : ('lbfgs','adam','sgd',), - 'learning_rate' : ('constant', 'invscaling', 'adaptive',), + "activation": ( + "logistic", + "tanh", + "relu", + ), + "solver": ( + "lbfgs", + "adam", + "sgd", + ), + "learning_rate": ( + "constant", + "invscaling", + "adaptive", + ), }, ] -est=MLPRegressor() +est = MLPRegressor() + def complexity(est): - return np.sum([c.size*2 for c in est.coefs_] - + [c.size*2 for c in est.intercepts_]) + return np.sum( + [c.size * 2 for c in est.coefs_] + [c.size * 2 for c in est.intercepts_] + ) + + model = None diff --git a/experiment/methods/sklearn_randomforest/regressor.py b/experiment/methods/sklearn_randomforest/regressor.py index 24041dde8..809b53aa8 100644 --- a/experiment/methods/sklearn_randomforest/regressor.py +++ b/experiment/methods/sklearn_randomforest/regressor.py @@ -1,15 +1,20 @@ -from sklearn import ensemble import numpy as np +from sklearn import ensemble + +hyper_params = [ + { + "n_estimators": (10, 100, 1000), + "min_weight_fraction_leaf": (0.0, 0.25, 0.5), + "max_features": ("sqrt", "log2", None), + } +] -hyper_params = [{ - 'n_estimators': (10, 100, 1000), - 'min_weight_fraction_leaf': (0.0, 0.25, 0.5), - 'max_features': ('sqrt','log2',None), -}] +est = ensemble.RandomForestRegressor() -est=ensemble.RandomForestRegressor() def complexity(est): return np.sum([e.tree_.node_count for e in est.estimators_]) + + model = None diff --git a/experiment/methods/sklearn_ridge/regressor.py b/experiment/methods/sklearn_ridge/regressor.py index 5ee9515f5..4467474b1 100644 --- a/experiment/methods/sklearn_ridge/regressor.py +++ b/experiment/methods/sklearn_ridge/regressor.py @@ -1,21 +1,40 @@ from sklearn import kernel_ridge -hyper_params = [{ - 'kernel': ('linear', 'poly','rbf','sigmoid',), - 'alpha': (1e-4,1e-2,0.1,1,), - 'gamma': (0.01,0.1,1,10,), -}] +hyper_params = [ + { + "kernel": ( + "linear", + "poly", + "rbf", + "sigmoid", + ), + "alpha": ( + 1e-4, + 1e-2, + 0.1, + 1, + ), + "gamma": ( + 0.01, + 0.1, + 1, + 10, + ), + } +] + +est = kernel_ridge.KernelRidge() -est=kernel_ridge.KernelRidge() def complexity(est): # this is a loose lower bound on the model complexity, basically jsut - # capturing the number of parameters in the model. Could make this + # capturing the number of parameters in the model. Could make this # kernel-specific and more accurate. - return est.dual_coef_.size + return est.dual_coef_.size + model = None eval_kwargs = { - 'scale_y': False, - 'scale_x': True, - } + "scale_y": False, + "scale_x": True, +} diff --git a/experiment/methods/sklearn_sgd/regressor.py b/experiment/methods/sklearn_sgd/regressor.py index ff273edb2..0e2104b8b 100644 --- a/experiment/methods/sklearn_sgd/regressor.py +++ b/experiment/methods/sklearn_sgd/regressor.py @@ -1,14 +1,22 @@ from sklearn import linear_model - hyper_params = [ { - 'alpha': (1e-06,1e-04,0.01,1,), - 'penalty': ('l2','l1','elasticnet',), + "alpha": ( + 1e-06, + 1e-04, + 0.01, + 1, + ), + "penalty": ( + "l2", + "l1", + "elasticnet", + ), }, ] -est=linear_model.SGDRegressor() +est = linear_model.SGDRegressor() complexity = None model = None diff --git a/experiment/methods/tir/regressor.py b/experiment/methods/tir/regressor.py index 29c5295a4..e6fd2cbb8 100644 --- a/experiment/methods/tir/regressor.py +++ b/experiment/methods/tir/regressor.py @@ -1,58 +1,63 @@ -import sys -import os import pyTIR as tir -from itertools import product -#os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" + +# os.environ["LD_LIBRARY_PATH"] = os.environ["CONDA_PREFIX"] + "/lib" hyper_params = [ { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),), - 'ngens' : (1000,), + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), + "ngens": (1000,), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),), - 'ngens' : (1000,), + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), + "ngens": (1000,), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-5,5),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-5, 5),), }, { - 'transfunctions' : ('Id,Tanh,Sin,Cos,Log,Exp,Sqrt',), - 'ytransfunctions' : ('Id,Exp,ATan,Tan,Tanh',), - 'exponents' : ((-2,2),) + "transfunctions": ("Id,Tanh,Sin,Cos,Log,Exp,Sqrt",), + "ytransfunctions": ("Id,Exp,ATan,Tan,Tanh",), + "exponents": ((-2, 2),), }, ] # Create the pipeline for the model -eval_kwargs = {'scale_x': False, 'scale_y': False} -est = tir.TIRRegressor(npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5,5), error="R^2", alg="MOO") +eval_kwargs = {"scale_x": False, "scale_y": False} +est = tir.TIRRegressor( + npop=1000, ngens=500, pc=0.3, pm=0.7, exponents=(-5, 5), error="R^2", alg="MOO" +) + def pre_train(est, X, y): """Adjust settings based on data before training""" - if X.shape[0]*X.shape[1] <= 1000: + if X.shape[0] * X.shape[1] <= 1000: est.penalty = 0.01 + def complexity(e): return e.len + def model(e, X=None): - new_model = e.sympy.replace("^","**") + new_model = e.sympy.replace("^", "**") if X is not None: - for i,f in reversed(list(enumerate(X.columns))): - new_model = new_model.replace(f'x{i}',f) + for i, f in reversed(list(enumerate(X.columns))): + new_model = new_model.replace(f"x{i}", f) return new_model + def get_population(est): pop = [] - for i in range(min(100, len(est.front))-1,-1,-1): + for i in range(min(100, len(est.front)) - 1, -1, -1): pop.append(est.create_model_from(i)) - return pop + return pop + def get_best_solution(est): return est diff --git a/experiment/methods/tpsr/regressor.py b/experiment/methods/tpsr/regressor.py index bbae170eb..9f29c565d 100644 --- a/experiment/methods/tpsr/regressor.py +++ b/experiment/methods/tpsr/regressor.py @@ -1,28 +1,34 @@ +# handling hardcored paths and no package setup +import sys import time -import json -import torch + import numpy as np import sympy as sp -from sklearn import feature_selection +import torch +from sklearn import feature_selection from sklearn.base import BaseEstimator, RegressorMixin -# handling hardcored paths and no package setup -import sys; sys.path.append("/opt/conda/bin/tpsr") +sys.path.append("/opt/conda/bin/tpsr") # TPSR stuff import symbolicregression -from symbolicregression.envs import build_env -from symbolicregression.model import build_modules -from symbolicregression.e2e_model import Transformer, pred_for_sample_no_refine, refine_for_sample, refine_for_sample_test +from default_pi import E2EHeuristic +from dyna_gym.agents.mcts import update_root from dyna_gym.agents.uct import UCT -from dyna_gym.agents.mcts import update_root, convert_to_json from rl_env import RLEnv -from default_pi import E2EHeuristic +from symbolicregression.e2e_model import ( + Transformer, + refine_for_sample, + refine_for_sample_test, +) +from symbolicregression.envs import build_env +from symbolicregression.model import build_modules + def get_top_k_features(X, y, k=10): - if y.ndim==2: - y=y[:,0] - if X.shape[1]<=k: + if y.ndim == 2: + y = y[:, 0] + if X.shape[1] <= k: return [i for i in range(X.shape[1])] else: kbest = feature_selection.SelectKBest(feature_selection.r_regression, k=k) @@ -32,256 +38,271 @@ def get_top_k_features(X, y, k=10): print("keeping only the top-{} features. Order was {}".format(k, top_features)) return list(top_features[:k]) -class TPRS_params(): + +class TPRS_params: def __init__(self): - self.ablation_to_keep=None - self.accumulate_gradients=1 - self.actor_lr=1e-06 - self.alg='mcts' - self.amp=-1 - self.attention_dropout=0 - self.backbone_model='e2e' - self.batch_load=False - self.batch_size=1 - self.batch_size_eval=64 - self.beam_early_stopping=True - self.beam_eval=True - self.beam_eval_train=0 - self.beam_length_penalty=1 - self.beam_selection_metrics=1 - self.beam_size=1 - self.beam_temperature=0.1 - self.beam_type='sampling' - self.clip_grad_norm=0.5 - self.collate_queue_size=2000 - self.cpu=True - self.critic_lr=1e-05 - self.debug=False - self.debug_slurm=False - self.debug_train_statistics=False - self.dec_emb_dim=512 - self.dec_positional_embeddings='learnable' - self.device='gpu' - self.dropout=0 - self.dump_path='' - self.emb_emb_dim=64 - self.emb_expansion_factor=1 - self.embedder_type='LinearPoint' - self.enc_emb_dim=512 - self.enc_positional_embeddings=None - self.enforce_dim=True - self.entropy_coef=0.01 - self.entropy_weighted_strategy='none' - self.env_base_seed=0 - self.env_name='functions' - self.eval_data='' - self.eval_dump_path=None - self.eval_from_exp='' - self.eval_in_domain=False - self.eval_input_length_modulo=-1 - self.eval_mcts_in_domain=False - self.eval_mcts_on_pmlb=False - self.eval_noise=0 - self.eval_noise_gamma=0.0 - self.eval_noise_type='additive' - self.eval_on_pmlb=False - self.eval_only=True - self.eval_size=10000 - self.eval_verbose=0 - self.eval_verbose_print=False - self.exp_id='' - self.exp_name='debug' - self.export_data=False - self.extra_binary_operators='' - self.extra_constants=None - self.extra_unary_operators='' - self.float_precision=3 - self.fp16=False - self.gpu_to_use='0' - self.horizon=200 - self.kl_coef=0.01 - self.kl_regularizer=0.001 - self.lam=0.1 - self.local_rank=-1 - self.lr=1e-05 - self.lr_patience=100 - self.mantissa_len=1 - self.master_port=-1 - self.max_binary_ops_offset=4 - self.max_binary_ops_per_dim=1 - self.max_centroids=10 - self.max_epoch=100000 - self.max_exponent=100 - self.max_exponent_prefactor=1 - self.max_generated_output_len=200 - self.max_input_dimension=10 - self.max_input_points=200 - self.max_int=10 - self.max_len=200 - self.max_number_bags=10 - self.max_output_dimension=1 - self.max_src_len=200 - self.max_target_len=200 - self.max_token_len=0 - self.max_trials=1 - self.max_unary_depth=6 - self.max_unary_ops=4 - self.min_binary_ops_per_dim=0 - self.min_input_dimension=1 - self.min_len_per_dim=5 - self.min_op_prob=0.01 - self.min_output_dimension=1 - self.min_unary_ops=0 - self.n_dec_heads=16 - self.n_dec_hidden_layers=1 - self.n_dec_layers=16 - self.n_emb_layers=1 - self.n_enc_heads=16 - self.n_enc_hidden_layers=1 - self.n_enc_layers=2 - self.n_prediction_points=200 - self.n_steps_per_epoch=3000 - self.n_trees_to_refine=10 - self.n_words=10292 - self.no_prefix_cache=True # setting these will disable cache - self.no_seq_cache=True - self.norm_attention=False - self.num_beams=1 - self.num_workers=10 - self.nvidia_apex=False - self.operators_to_downsample='div_0,arcsin_0,arccos_0,tan_0.2,arctan_0.2,sqrt_5,pow2_3,inv_3' - self.operators_to_not_repeat='' - self.optimizer='adam_inverse_sqrt,warmup_updates=10000' - self.pad_to_max_dim=True - self.pmlb_data_type='feynman' - self.prediction_sigmas='1,2,4,8,16' - self.print_freq=100 - self.prob_const=0.0 - self.prob_rand=0.0 - self.queue_strategy=None - self.reduce_num_constants=True - self.refinements_types='method=BFGS_batchsize=256_metric=/_mse' - self.reload_checkpoint='' - self.reload_data='' - self.reload_model='' - self.reload_size=-1 - self.required_operators='' - self.rescale=True - self.reward_coef=1 - self.reward_type='nmse' - self.rl_alg='ppo' - self.rollout=3 - self.run_id=1 - self.sample_only=False - self.save_eval_dic='./eval_result' - self.save_model=False - self.save_periodic=25 - self.save_results=False - self.seed=23 - self.share_inout_emb=True - self.simplify=False - self.stopping_criterion='' - self.target_kl=1 - self.target_noise=0.0 - self.tasks='functions' - self.test_env_seed=1 - self.tokens_per_batch=10000 - self.train_noise_gamma=0.0 - self.train_value=False - self.ts_mode='best' - self.ucb_base=10.0 - self.ucb_constant=1.0 - self.uct_alg='uct' - self.update_modules='all' - self.use_abs=False - self.use_controller=True - self.use_skeleton=False - self.use_sympy=False - self.validation_metrics='r2_zero,r2,accuracy_l1_biggio,accuracy_l1_1e-3,accuracy_l1_1e-2,accuracy_l1_1e-1,_complexity' - self.vf_coef=0.0001 - self.warmup_epoch=5 - self.width=3 - self.windows=False - + self.cache = "" + self.ablation_to_keep = None + self.accumulate_gradients = 1 + self.actor_lr = 1e-06 + self.alg = "mcts" + self.amp = -1 + self.attention_dropout = 0 + self.backbone_model = "e2e" + self.batch_load = False + self.batch_size = 1 + self.batch_size_eval = 64 + self.beam_early_stopping = True + self.beam_eval = True + self.beam_eval_train = 0 + self.beam_length_penalty = 1 + self.beam_selection_metrics = 1 + self.beam_size = 1 + self.beam_temperature = 0.1 + self.beam_type = "sampling" + self.clip_grad_norm = 0.5 + self.collate_queue_size = 2000 + self.cpu = True + self.critic_lr = 1e-05 + self.debug = False + self.debug_slurm = False + self.debug_train_statistics = False + self.dec_emb_dim = 512 + self.dec_positional_embeddings = "learnable" + self.device = "gpu" + self.dropout = 0 + self.dump_path = "" + self.emb_emb_dim = 64 + self.emb_expansion_factor = 1 + self.embedder_type = "LinearPoint" + self.enc_emb_dim = 512 + self.enc_positional_embeddings = None + self.enforce_dim = True + self.entropy_coef = 0.01 + self.entropy_weighted_strategy = "none" + self.env_base_seed = 0 + self.env_name = "functions" + self.eval_data = "" + self.eval_dump_path = None + self.eval_from_exp = "" + self.eval_in_domain = False + self.eval_input_length_modulo = -1 + self.eval_mcts_in_domain = False + self.eval_mcts_on_pmlb = False + self.eval_noise = 0 + self.eval_noise_gamma = 0.0 + self.eval_noise_type = "additive" + self.eval_on_pmlb = False + self.eval_only = True + self.eval_size = 10000 + self.eval_verbose = 0 + self.eval_verbose_print = False + self.exp_id = "" + self.exp_name = "debug" + self.export_data = False + self.extra_binary_operators = "" + self.extra_constants = None + self.extra_unary_operators = "" + self.float_precision = 3 + self.fp16 = False + self.gpu_to_use = "0" + self.horizon = 200 + self.kl_coef = 0.01 + self.kl_regularizer = 0.001 + self.lam = 0.1 + self.local_rank = -1 + self.lr = 1e-05 + self.lr_patience = 100 + self.mantissa_len = 1 + self.master_port = -1 + self.max_binary_ops_offset = 4 + self.max_binary_ops_per_dim = 1 + self.max_centroids = 10 + self.max_epoch = 100000 + self.max_exponent = 100 + self.max_exponent_prefactor = 1 + self.max_generated_output_len = 200 + self.max_input_dimension = 10 + self.max_input_points = 200 + self.max_int = 10 + self.max_len = 200 + self.max_number_bags = 10 + self.max_output_dimension = 1 + self.max_src_len = 200 + self.max_target_len = 200 + self.max_token_len = 0 + self.max_trials = 1 + self.max_unary_depth = 6 + self.max_unary_ops = 4 + self.min_binary_ops_per_dim = 0 + self.min_input_dimension = 1 + self.min_len_per_dim = 5 + self.min_op_prob = 0.01 + self.min_output_dimension = 1 + self.min_unary_ops = 0 + self.n_dec_heads = 16 + self.n_dec_hidden_layers = 1 + self.n_dec_layers = 16 + self.n_emb_layers = 1 + self.n_enc_heads = 16 + self.n_enc_hidden_layers = 1 + self.n_enc_layers = 2 + self.n_prediction_points = 200 + self.n_steps_per_epoch = 3000 + self.n_trees_to_refine = 10 + self.n_words = 10292 + self.no_prefix_cache = True # setting these will disable cache + self.no_seq_cache = True + self.norm_attention = False + self.num_beams = 1 + self.num_workers = 10 + self.nvidia_apex = False + self.operators_to_downsample = ( + "div_0,arcsin_0,arccos_0,tan_0.2,arctan_0.2,sqrt_5,pow2_3,inv_3" + ) + self.operators_to_not_repeat = "" + self.optimizer = "adam_inverse_sqrt,warmup_updates=10000" + self.pad_to_max_dim = True + self.pmlb_data_type = "feynman" + self.prediction_sigmas = "1,2,4,8,16" + self.print_freq = 100 + self.prob_const = 0.0 + self.prob_rand = 0.0 + self.queue_strategy = None + self.reduce_num_constants = True + self.refinements_types = "method=BFGS_batchsize=256_metric=/_mse" + self.reload_checkpoint = "" + self.reload_data = "" + self.reload_model = "" + self.reload_size = -1 + self.required_operators = "" + self.rescale = True + self.reward_coef = 1 + self.reward_type = "nmse" + self.rl_alg = "ppo" + self.rollout = 3 + self.run_id = 1 + self.sample_only = False + self.save_eval_dic = "./eval_result" + self.save_model = False + self.save_periodic = 25 + self.save_results = False + self.seed = 23 + self.share_inout_emb = True + self.simplify = False + self.stopping_criterion = "" + self.target_kl = 1 + self.target_noise = 0.0 + self.tasks = "functions" + self.test_env_seed = 1 + self.tokens_per_batch = 10000 + self.train_noise_gamma = 0.0 + self.train_value = False + self.ts_mode = "best" + self.ucb_base = 10.0 + self.ucb_constant = 1.0 + self.uct_alg = "uct" + self.update_modules = "all" + self.use_abs = False + self.use_controller = True + self.use_skeleton = False + self.use_sympy = False + self.validation_metrics = "r2_zero,r2,accuracy_l1_biggio,accuracy_l1_1e-3,accuracy_l1_1e-2,accuracy_l1_1e-1,_complexity" + self.vf_coef = 0.0001 + self.warmup_epoch = 5 + self.width = 3 + self.windows = False + + class TPSRRegressor(BaseEstimator, RegressorMixin): def __init__(self, random_state=42): - + self.tpsr_params = TPRS_params() self.random_state = random_state self.tpsr_params.seed = self.random_state - + def fit(self, X, y): - + np.random.seed(self.tpsr_params.seed) torch.manual_seed(self.tpsr_params.seed) torch.cuda.manual_seed(self.tpsr_params.seed) # manually setting the hyperparameters - self.tpsr_params.cpu = False if torch.cuda.is_available() else True - self.tpsr_params.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.tpsr_params.cpu = False if torch.cuda.is_available() else True + self.tpsr_params.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) symbolicregression.utils.CUDA = self.tpsr_params.device - if (self.tpsr_params.no_prefix_cache==False) and (self.tpsr_params.no_seq_cache == False): - self.tpsr_params.cache = 'b' #both cachings are arctivated - elif (self.tpsr_params.no_prefix_cache==True) and (self.tpsr_params.no_seq_cache ==False): - self.tpsr_params.cache = 's' #only sequence caching - elif (self.tpsr_params.no_prefix_cache==False) and (self.tpsr_params.no_seq_cache==True): - self.tpsr_params.cache = 'k' #only top-k caching + if (not self.tpsr_params.no_prefix_cache) and ( + not self.tpsr_params.no_seq_cache + ): + self.tpsr_params.cache = "b" # both cachings are arctivated + elif (self.tpsr_params.no_prefix_cache) and (not self.tpsr_params.no_seq_cache): + self.tpsr_params.cache = "s" # only sequence caching + elif (not self.tpsr_params.no_prefix_cache) and (self.tpsr_params.no_seq_cache): + self.tpsr_params.cache = "k" # only top-k caching else: - self.tpsr_params.cache = 'n' # no caching + self.tpsr_params.cache = "n" # no caching - self.tpsr_params.backbone_model="e2e" + self.tpsr_params.backbone_model = "e2e" self.equation_env = build_env(self.tpsr_params) - modules = build_modules(self.equation_env, self.tpsr_params) + build_modules(self.equation_env, self.tpsr_params) # print(self.tpsr_params) # pre-processing the data to handle up to 10 features - - top_k_features = get_top_k_features(X, y, k=self.tpsr_params.max_input_dimension) + + top_k_features = get_top_k_features( + X, y, k=self.tpsr_params.max_input_dimension + ) X = X[:, top_k_features] - samples = {'x_to_fit': [X], 'y_to_fit':[y], - 'x_to_pred':0, 'y_to_pred':0} + samples = {"x_to_fit": [X], "y_to_fit": [y], "x_to_pred": 0, "y_to_pred": 0} # --- main --- # Transformer loads the pre trained - model = Transformer(params = self.tpsr_params, env=self.equation_env, samples=samples) - model.to(self.tpsr_params.device) + model = Transformer( + params=self.tpsr_params, env=self.equation_env, samples=samples + ) + model.to(self.tpsr_params.device) rl_env = RLEnv( - params = self.tpsr_params, - samples = samples, - equation_env = self.equation_env, - model = model) + params=self.tpsr_params, + samples=samples, + equation_env=self.equation_env, + model=model, + ) dp = E2EHeuristic( - equation_env=self.equation_env, - rl_env=rl_env, - model=model, - k=self.tpsr_params.width, - num_beams=self.tpsr_params.num_beams, - horizon=self.tpsr_params.horizon, - device=self.tpsr_params.device, - use_seq_cache=not self.tpsr_params.no_seq_cache, - use_prefix_cache=not self.tpsr_params.no_prefix_cache, - length_penalty = self.tpsr_params.beam_length_penalty, - train_value_mode=self.tpsr_params.train_value, - debug=self.tpsr_params.debug) + equation_env=self.equation_env, + rl_env=rl_env, + model=model, + k=self.tpsr_params.width, + num_beams=self.tpsr_params.num_beams, + horizon=self.tpsr_params.horizon, + device=self.tpsr_params.device, + use_seq_cache=not self.tpsr_params.no_seq_cache, + use_prefix_cache=not self.tpsr_params.no_prefix_cache, + length_penalty=self.tpsr_params.beam_length_penalty, + train_value_mode=self.tpsr_params.train_value, + debug=self.tpsr_params.debug, + ) start = time.time() agent = UCT( - action_space=[], # this will not be used as we have a default policy - gamma=1., # no discounting - ucb_constant=1., + action_space=[], # this will not be used as we have a default policy + gamma=1.0, # no discounting + ucb_constant=1.0, horizon=self.tpsr_params.horizon, rollouts=self.tpsr_params.rollout, dp=dp, width=self.tpsr_params.width, reuse_tree=True, alg=self.tpsr_params.uct_alg, - ucb_base=self.tpsr_params.ucb_base) + ucb_base=self.tpsr_params.ucb_base, + ) agent.display() if self.tpsr_params.sample_only: @@ -289,16 +310,17 @@ def fit(self, X, y): # programs are saved in samples json file horizon = 1 else: - horizon = 200 - + horizon = 200 + done = False s = rl_env.state - ret_all = [] for t in range(horizon): if len(s) >= self.tpsr_params.horizon: - print(f'Cannot process programs longer than {self.tpsr_params.horizon}. Stop here.') + print( + f"Cannot process programs longer than {self.tpsr_params.horizon}. Stop here." + ) break if done: @@ -307,42 +329,43 @@ def fit(self, X, y): act = agent.act(rl_env, done) s, r, done, _ = rl_env.step(act) - if t ==0: - real_root = agent.root - update_root(agent, act, s) dp.update_cache(s) - + time_elapsed = time.time() - start - print('TPSR+E2E Time Elapsed:', time_elapsed) + print("TPSR+E2E Time Elapsed:", time_elapsed) # TPSR+E2E NMSE # y_mcts , mcts_str , mcts_tree = pred_for_sample_no_refine( # model, self.equation_env, s ,samples['x_to_fit']) - + # TPSR+E2E NMSE after Refine - y_mcts , mcts_str , mcts_tree = refine_for_sample( - self.tpsr_params, model, self.equation_env, s, - samples['x_to_fit'], samples['y_to_fit']) + y_mcts, mcts_str, mcts_tree = refine_for_sample( + self.tpsr_params, + model, + self.equation_env, + s, + samples["x_to_fit"], + samples["y_to_fit"], + ) self.mcts_str_ = mcts_str - self.tree_ = mcts_tree - self.model_ = model - self.X_ = X - self.y_ = y - self.s_ = s - - return self + self.tree_ = mcts_tree + self.model_ = model + self.X_ = X + self.y_ = y + self.s_ = s + return self def predict(self, X): # y_mcts , _ , _ = pred_for_sample_no_refine( # self.model_, self.equation_env, self.s_ ,[X]) - - _, y_mcts , _ , _ = refine_for_sample_test( - self.model_, self.equation_env, self.s_, - [self.X_], [self.y_], [X]) - + + _, y_mcts, _, _ = refine_for_sample_test( + self.model_, self.equation_env, self.s_, [self.X_], [self.y_], [X] + ) + return y_mcts @@ -351,10 +374,10 @@ def predict(self, X): def model(est, X=None): mcts_str = est.mcts_str_ - + replace_ops = {"add": "+", "mul": "*", "sub": "-", "pow": "**", "inv": "1/"} for op, replace_op in replace_ops.items(): - mcts_str = mcts_str.replace(op,replace_op) + mcts_str = mcts_str.replace(op, replace_op) mcts_eq = sp.parse_expr(mcts_str) @@ -364,4 +387,4 @@ def model(est, X=None): # python regressor.py # define eval_kwargs. -eval_kwargs = dict(use_dataframe=False) \ No newline at end of file +eval_kwargs = dict(use_dataframe=False) diff --git a/experiment/methods/tuned/AFPRegressor.py b/experiment/methods/tuned/AFPRegressor.py index bfecfa19b..c67c4c573 100644 --- a/experiment/methods/tuned/AFPRegressor.py +++ b/experiment/methods/tuned/AFPRegressor.py @@ -1,11 +1,25 @@ -from ..AFPRegressor import complexity,model, est -from ellyn import ellyn +from ..AFPRegressor import complexity as complexity +from ..AFPRegressor import est as est +from ..AFPRegressor import model as model from .params._afpregressor import params - est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) -est.time_limit = 8*60*60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/AIFeynman.py b/experiment/methods/tuned/AIFeynman.py index c1664d960..660c807b4 100644 --- a/experiment/methods/tuned/AIFeynman.py +++ b/experiment/methods/tuned/AIFeynman.py @@ -1,7 +1,8 @@ -from ..AIFeynman import complexity, model, est +from ..AIFeynman import complexity as complexity +from ..AIFeynman import est as est +from ..AIFeynman import model as model from .params._aifeynman import params est.set_params(**params) # 8 hours -est.max_time = 8*60*60 - +est.max_time = 8 * 60 * 60 diff --git a/experiment/methods/tuned/BSRRegressor.py b/experiment/methods/tuned/BSRRegressor.py index d20dae494..2e0c36938 100644 --- a/experiment/methods/tuned/BSRRegressor.py +++ b/experiment/methods/tuned/BSRRegressor.py @@ -1,9 +1,11 @@ -from ..BSRRegressor import complexity,model,est +from ..BSRRegressor import complexity as complexity +from ..BSRRegressor import est as est +from ..BSRRegressor import model as model from .params._bsrregressor import params est.set_params(**params) -est.max_time = 8*60*60 +est.max_time = 8 * 60 * 60 # double the evals -est.itrNum = int(est.itrNum*2**0.5) -est.val = int(est.val*2**0.5) +est.itrNum = int(est.itrNum * 2**0.5) +est.val = int(est.val * 2**0.5) diff --git a/experiment/methods/tuned/DSRRegressor.py b/experiment/methods/tuned/DSRRegressor.py index eb6aaf382..32d2a77a2 100644 --- a/experiment/methods/tuned/DSRRegressor.py +++ b/experiment/methods/tuned/DSRRegressor.py @@ -1,12 +1,11 @@ -from ..DSRRegressor import complexity,model,base_config from dsr import DeepSymbolicRegressor -import numpy as np -import copy -from .params._dsrregressor import params -#double the evals -base_config['training']['n_samples'] = 1000000 +from ..DSRRegressor import base_config +from ..DSRRegressor import complexity as complexity +from ..DSRRegressor import model as model + +# double the evals +base_config["training"]["n_samples"] = 1000000 # Create the model est = DeepSymbolicRegressor(base_config) - diff --git a/experiment/methods/tuned/EHCRegressor.py b/experiment/methods/tuned/EHCRegressor.py index e269dcae7..b01a19534 100644 --- a/experiment/methods/tuned/EHCRegressor.py +++ b/experiment/methods/tuned/EHCRegressor.py @@ -1,10 +1,24 @@ -from ..EHCRegressor import complexity, model, est -from ellyn import ellyn +from ..EHCRegressor import complexity as complexity +from ..EHCRegressor import est as est +from ..EHCRegressor import model as model - -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.eHC_its=5 +est.eHC_its = 5 est.g = 143 est.popsize = 1000 -est.time_limit = 8*60*60 +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/EPLEXRegressor.py b/experiment/methods/tuned/EPLEXRegressor.py index f4927f77f..d7ad4a6da 100644 --- a/experiment/methods/tuned/EPLEXRegressor.py +++ b/experiment/methods/tuned/EPLEXRegressor.py @@ -1,10 +1,26 @@ -from ..EPLEXRegressor import complexity,model, est +from ..EPLEXRegressor import complexity as complexity +from ..EPLEXRegressor import est as est +from ..EPLEXRegressor import model as model from .params._eplexregressor import params est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] # double the evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) -est.time_limit = 8*60*60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) +est.time_limit = 8 * 60 * 60 diff --git a/experiment/methods/tuned/FEATRegressor.py b/experiment/methods/tuned/FEATRegressor.py index 8c6e93a7a..5f1d47f1d 100644 --- a/experiment/methods/tuned/FEATRegressor.py +++ b/experiment/methods/tuned/FEATRegressor.py @@ -1,12 +1,14 @@ -from ..FEATRegressor import complexity,model,pre_train, eval_kwargs, est -from feat import FeatRegressor +from ..FEATRegressor import complexity as complexity +from ..FEATRegressor import est as est +from ..FEATRegressor import eval_kwargs as eval_kwargs +from ..FEATRegressor import model as model from .params._featregressor import params est.set_params(**params) -est.functions = '+,-,*,/,^2,^3,sqrt,sin,cos,exp,log' -est.otype = 'f' +est.functions = "+,-,*,/,^2,^3,sqrt,sin,cos,exp,log" +est.otype = "f" # double the evals -est.max_time=int(8*60*60) # 8 hrs -est.gens = int(est.gens*2**0.5) -est.pop_size = int(est.pop_size*2**0.5) +est.max_time = int(8 * 60 * 60) # 8 hrs +est.gens = int(est.gens * 2**0.5) +est.pop_size = int(est.pop_size * 2**0.5) diff --git a/experiment/methods/tuned/FE_AFPRegressor.py b/experiment/methods/tuned/FE_AFPRegressor.py index c452bbeb9..6dbfa53c9 100644 --- a/experiment/methods/tuned/FE_AFPRegressor.py +++ b/experiment/methods/tuned/FE_AFPRegressor.py @@ -1,9 +1,26 @@ -from ..FE_AFPRegressor import complexity,model, pre_train, eval_kwargs, est +from ..FE_AFPRegressor import complexity as complexity +from ..FE_AFPRegressor import est as est +from ..FE_AFPRegressor import eval_kwargs as eval_kwargs +from ..FE_AFPRegressor import model as model from .params._fe_afpregressor import params est.set_params(**params) -est.op_list = ['n','v','+','-','*','/','exp','log','2','3','sqrt','sin','cos'] -est.time_limit = 8*60*60 -# doubling evals -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) +est.op_list = [ + "n", + "v", + "+", + "-", + "*", + "/", + "exp", + "log", + "2", + "3", + "sqrt", + "sin", + "cos", +] +est.time_limit = 8 * 60 * 60 +# doubling evals +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) diff --git a/experiment/methods/tuned/FFXRegressor.py b/experiment/methods/tuned/FFXRegressor.py index 5823d2919..1d3cb5a42 100644 --- a/experiment/methods/tuned/FFXRegressor.py +++ b/experiment/methods/tuned/FFXRegressor.py @@ -1,2 +1,3 @@ -from ..FFXRegressor import complexity,model,est -from ffx import FFXRegressor +from ..FFXRegressor import complexity as complexity +from ..FFXRegressor import est as est +from ..FFXRegressor import model as model diff --git a/experiment/methods/tuned/GPGOMEARegressor.py b/experiment/methods/tuned/GPGOMEARegressor.py index 2af62b2cf..2b738efa0 100644 --- a/experiment/methods/tuned/GPGOMEARegressor.py +++ b/experiment/methods/tuned/GPGOMEARegressor.py @@ -1,11 +1,12 @@ -from ..GPGOMEARegressor import complexity, model, est -from pyGPGOMEA import GPGOMEARegressor as GPG +from ..GPGOMEARegressor import complexity as complexity +from ..GPGOMEARegressor import est as est +from ..GPGOMEARegressor import model as model from .params._gpgomearegressor import params est.set_params(**params) -est.functions = '+_-_*_p/_plog_sqrt_sin_cos' +est.functions = "+_-_*_p/_plog_sqrt_sin_cos" # double the evals est.evaluations = 1000000 -est.time=8*60*60 +est.time = 8 * 60 * 60 diff --git a/experiment/methods/tuned/ITEARegressor.py b/experiment/methods/tuned/ITEARegressor.py index b34bce4e3..42e5612f0 100644 --- a/experiment/methods/tuned/ITEARegressor.py +++ b/experiment/methods/tuned/ITEARegressor.py @@ -1,9 +1,12 @@ -from ..ITEARegressor import complexity,model,eval_kwargs, est +from ..ITEARegressor import complexity as complexity +from ..ITEARegressor import est as est +from ..ITEARegressor import eval_kwargs as eval_kwargs +from ..ITEARegressor import model as model from .params._itearegressor import params est.set_params(**params) -est.transfunctions = '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]' +est.transfunctions = "[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]" # double the evals -est.npop = int(est.npop*2**0.5) -est.ngens = int(est.ngens*2**0.5) +est.npop = int(est.npop * 2**0.5) +est.ngens = int(est.ngens * 2**0.5) diff --git a/experiment/methods/tuned/MRGPRegressor.py b/experiment/methods/tuned/MRGPRegressor.py index 41d77d3d8..8c228eb2a 100644 --- a/experiment/methods/tuned/MRGPRegressor.py +++ b/experiment/methods/tuned/MRGPRegressor.py @@ -1,10 +1,11 @@ -from ..MRGPRegressor import complexity, model, est -from ..src.mrgp import MRGPRegressor +from ..MRGPRegressor import complexity as complexity +from ..MRGPRegressor import est as est +from ..MRGPRegressor import model as model from .params._mrgpregressor import params est.set_params(**params) # double the evals -est.time_out=8*60*60 -est.g = int(est.g*2**0.5) -est.popsize = int(est.popsize*2**0.5) +est.time_out = 8 * 60 * 60 +est.g = int(est.g * 2**0.5) +est.popsize = int(est.popsize * 2**0.5) diff --git a/experiment/methods/tuned/OperonRegressor.py b/experiment/methods/tuned/OperonRegressor.py index 24d46fe60..3f76844e8 100644 --- a/experiment/methods/tuned/OperonRegressor.py +++ b/experiment/methods/tuned/OperonRegressor.py @@ -1,11 +1,12 @@ -from ..OperonRegressor import complexity,model,est -from operon.sklearn import SymbolicRegressor +from ..OperonRegressor import complexity as complexity +from ..OperonRegressor import est as est +from ..OperonRegressor import model as model from .params._operonregressor import params est.set_params(**params) -est.allowed_symbols = 'add,sub,mul,div,exp,log,sin,cos,sqrt,square,constant,variable' +est.allowed_symbols = "add,sub,mul,div,exp,log,sin,cos,sqrt,square,constant,variable" # double the evals est.max_evaluations = 1000000 -est.generations=100000 # just large enough since we have an evaluation budget -est.time_limit=8*60*60 # 8 hours +est.generations = 100000 # just large enough since we have an evaluation budget +est.time_limit = 8 * 60 * 60 # 8 hours diff --git a/experiment/methods/tuned/gplearn.py b/experiment/methods/tuned/gplearn.py index 39e3fe3b7..869a1d28f 100644 --- a/experiment/methods/tuned/gplearn.py +++ b/experiment/methods/tuned/gplearn.py @@ -1,9 +1,11 @@ -from ..gplearn import complexity,model, est +from ..gplearn.regressor import complexity as complexity +from ..gplearn.regressor import est as est +from ..gplearn.regressor import model as model from .params._gplearn import params est.set_params(**params) -est.function_set = ('add', 'sub', 'mul', 'div', 'log','sqrt', 'sin','cos') +est.function_set = ("add", "sub", "mul", "div", "log", "sqrt", "sin", "cos") # double the evals -est.population_size = int(est.population_size*2**0.5) -est.generations = int(est.generations*2**0.5) +est.population_size = int(est.population_size * 2**0.5) +est.generations = int(est.generations * 2**0.5) diff --git a/experiment/methods/tuned/params/_afpregressor.py b/experiment/methods/tuned/params/_afpregressor.py index 779c59ae2..8d8da237e 100644 --- a/experiment/methods/tuned/params/_afpregressor.py +++ b/experiment/methods/tuned/params/_afpregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': False, 'FE_ind_size': 0, 'FE_pop_size': 0, 'FE_rank': False, 'FE_train_gens': 0, 'FE_train_size': 0, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 22118, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'afp', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 1, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": False, + "FE_ind_size": 0, + "FE_pop_size": 0, + "FE_rank": False, + "FE_train_gens": 0, + "FE_train_size": 0, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 22118, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "afp", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 1, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_aifeynman.py b/experiment/methods/tuned/params/_aifeynman.py index 00c2f9985..dc5add99f 100644 --- a/experiment/methods/tuned/params/_aifeynman.py +++ b/experiment/methods/tuned/params/_aifeynman.py @@ -1 +1,9 @@ -params = {'BF_ops_file_type': '14ops.txt', 'BF_try_time': 60, 'NN_epochs': 4000, 'max_time': 7200, 'polyfit_deg': 4, 'random_state': 11284, 'test_percentage': 20} +params = { + "BF_ops_file_type": "14ops.txt", + "BF_try_time": 60, + "NN_epochs": 4000, + "max_time": 7200, + "polyfit_deg": 4, + "random_state": 11284, + "test_percentage": 20, +} diff --git a/experiment/methods/tuned/params/_bsrregressor.py b/experiment/methods/tuned/params/_bsrregressor.py index 21a3d19c6..ad66e0316 100644 --- a/experiment/methods/tuned/params/_bsrregressor.py +++ b/experiment/methods/tuned/params/_bsrregressor.py @@ -1 +1,11 @@ -params = {'Ops': ['inv', 'ln', 'neg', 'sin', 'cos', 'exp', 'square', 'cubic', '+', '*'], 'alpha1': 0.4, 'alpha2': 0.4, 'beta': -1, 'disp': False, 'itrNum': 5000, 'max_time': 7200, 'treeNum': 3, 'val': 100} +params = { + "Ops": ["inv", "ln", "neg", "sin", "cos", "exp", "square", "cubic", "+", "*"], + "alpha1": 0.4, + "alpha2": 0.4, + "beta": -1, + "disp": False, + "itrNum": 5000, + "max_time": 7200, + "treeNum": 3, + "val": 100, +} diff --git a/experiment/methods/tuned/params/_dsrregressor.py b/experiment/methods/tuned/params/_dsrregressor.py index deefb909a..8ce4ba206 100644 --- a/experiment/methods/tuned/params/_dsrregressor.py +++ b/experiment/methods/tuned/params/_dsrregressor.py @@ -1 +1,5 @@ -params = {} +from __future__ import annotations + +from typing import Any + +params: dict[str, Any] = {} diff --git a/experiment/methods/tuned/params/_eplexregressor.py b/experiment/methods/tuned/params/_eplexregressor.py index 03f79cf8e..42f55d500 100644 --- a/experiment/methods/tuned/params/_eplexregressor.py +++ b/experiment/methods/tuned/params/_eplexregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': False, 'FE_ind_size': 0, 'FE_pop_size': 0, 'FE_rank': False, 'FE_train_gens': 0, 'FE_train_size': 0, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 6265, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'epsilon_lexicase', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 0, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": False, + "FE_ind_size": 0, + "FE_pop_size": 0, + "FE_rank": False, + "FE_train_gens": 0, + "FE_train_size": 0, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 6265, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "epsilon_lexicase", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 0, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_fe_afpregressor.py b/experiment/methods/tuned/params/_fe_afpregressor.py index 15bfb78f3..7bb54e96e 100644 --- a/experiment/methods/tuned/params/_fe_afpregressor.py +++ b/experiment/methods/tuned/params/_fe_afpregressor.py @@ -1 +1,113 @@ -params = {'AR': False, 'AR_lookahead': False, 'AR_na': 1, 'AR_nb': 1, 'AR_nka': 1, 'AR_nkb': 0, 'ERC': True, 'ERCints': False, 'EstimateFitness': True, 'FE_ind_size': 10, 'FE_pop_size': 100, 'FE_rank': True, 'FE_train_gens': 10, 'FE_train_size': 10, 'G_sel': 1, 'G_shuffle': False, 'PS_sel': 1, 'SGD': False, 'align_dev': False, 'class_bool': False, 'class_m4gp': False, 'class_prune': False, 'classification': False, 'complex_measure': 1, 'cross': 3, 'cross_ar': 0.025, 'eHC_init': 0.5, 'eHC_its': 1, 'eHC_mut': False, 'eHC_on': False, 'eHC_prob': 0.1, 'eHC_slim': False, 'elitism': True, 'estimate_generality': False, 'fit_type': 'MSE', 'g': 250, 'init_trees': True, 'init_validate_on': False, 'island_gens': 100, 'islands': False, 'learning_rate': 1.0, 'lex_class': False, 'lex_eps_dynamic': False, 'lex_eps_dynamic_madcap': False, 'lex_eps_dynamic_rand': False, 'lex_eps_error': False, 'lex_eps_error_mad': False, 'lex_eps_global': False, 'lex_eps_std': False, 'lex_eps_target': False, 'lex_eps_target_mad': False, 'lex_epsilon': 0.1, 'lexage': False, 'lexpool': 1, 'limit_evals': False, 'lineHC_its': 0, 'lineHC_on': False, 'maxERC': 1, 'max_evals': 0, 'max_fit': 1e+20, 'max_len': 64, 'max_len_init': 20, 'minERC': -1, 'min_fit': 1e-20, 'min_len': 3, 'mut_ar': 0.025, 'mutate': 1, 'norm_error': False, 'nt': 1, 'numERC': 1, 'num_islands': 10, 'num_log_pts': 0, 'pHC_gauss': 0, 'pHC_its': 1, 'pHC_on': True, 'pop_restart': False, 'pop_restart_path': '', 'popsize': 1000, 'print_archive': False, 'print_best_ind': False, 'print_data': False, 'print_db': False, 'print_epigenome': False, 'print_every_pop': False, 'print_genome': False, 'print_homology': False, 'print_init_pop': False, 'print_last_pop': False, 'print_log': False, 'print_novelty': False, 'print_protected_operators': False, 'prto_arch_on': True, 'prto_arch_size': 1, 'prto_sel_on': False, 'random_state': 29802, 'resultspath': './', 'return_pop': False, 'rt_cross': 0.6, 'rt_mut': 0.4, 'rt_rep': 0, 'savename': '', 'selection': 'afp', 'shuffle_data': False, 'stop_condition': True, 'stop_threshold': 1e-06, 'test_at_end': False, 'tourn_size': 2, 'train': False, 'train_pct': 0.5, 'verbosity': 0, 'weight_error': False} +params = { + "AR": False, + "AR_lookahead": False, + "AR_na": 1, + "AR_nb": 1, + "AR_nka": 1, + "AR_nkb": 0, + "ERC": True, + "ERCints": False, + "EstimateFitness": True, + "FE_ind_size": 10, + "FE_pop_size": 100, + "FE_rank": True, + "FE_train_gens": 10, + "FE_train_size": 10, + "G_sel": 1, + "G_shuffle": False, + "PS_sel": 1, + "SGD": False, + "align_dev": False, + "class_bool": False, + "class_m4gp": False, + "class_prune": False, + "classification": False, + "complex_measure": 1, + "cross": 3, + "cross_ar": 0.025, + "eHC_init": 0.5, + "eHC_its": 1, + "eHC_mut": False, + "eHC_on": False, + "eHC_prob": 0.1, + "eHC_slim": False, + "elitism": True, + "estimate_generality": False, + "fit_type": "MSE", + "g": 250, + "init_trees": True, + "init_validate_on": False, + "island_gens": 100, + "islands": False, + "learning_rate": 1.0, + "lex_class": False, + "lex_eps_dynamic": False, + "lex_eps_dynamic_madcap": False, + "lex_eps_dynamic_rand": False, + "lex_eps_error": False, + "lex_eps_error_mad": False, + "lex_eps_global": False, + "lex_eps_std": False, + "lex_eps_target": False, + "lex_eps_target_mad": False, + "lex_epsilon": 0.1, + "lexage": False, + "lexpool": 1, + "limit_evals": False, + "lineHC_its": 0, + "lineHC_on": False, + "maxERC": 1, + "max_evals": 0, + "max_fit": 1e20, + "max_len": 64, + "max_len_init": 20, + "minERC": -1, + "min_fit": 1e-20, + "min_len": 3, + "mut_ar": 0.025, + "mutate": 1, + "norm_error": False, + "nt": 1, + "numERC": 1, + "num_islands": 10, + "num_log_pts": 0, + "pHC_gauss": 0, + "pHC_its": 1, + "pHC_on": True, + "pop_restart": False, + "pop_restart_path": "", + "popsize": 1000, + "print_archive": False, + "print_best_ind": False, + "print_data": False, + "print_db": False, + "print_epigenome": False, + "print_every_pop": False, + "print_genome": False, + "print_homology": False, + "print_init_pop": False, + "print_last_pop": False, + "print_log": False, + "print_novelty": False, + "print_protected_operators": False, + "prto_arch_on": True, + "prto_arch_size": 1, + "prto_sel_on": False, + "random_state": 29802, + "resultspath": "./", + "return_pop": False, + "rt_cross": 0.6, + "rt_mut": 0.4, + "rt_rep": 0, + "savename": "", + "selection": "afp", + "shuffle_data": False, + "stop_condition": True, + "stop_threshold": 1e-06, + "test_at_end": False, + "tourn_size": 2, + "train": False, + "train_pct": 0.5, + "verbosity": 0, + "weight_error": False, +} diff --git a/experiment/methods/tuned/params/_featregressor.py b/experiment/methods/tuned/params/_featregressor.py index 5bd3cd254..71acca7e3 100644 --- a/experiment/methods/tuned/params/_featregressor.py +++ b/experiment/methods/tuned/params/_featregressor.py @@ -1 +1,45 @@ -params = {'pop_size': 1000, 'gens': 250, 'ml': 'LinearRidgeRegression', 'classification': False, 'verbosity': 0, 'max_stall': 100, 'sel': 'lexicase', 'surv': 'nsga2', 'cross_rate': 0.75, 'root_xo_rate': 0.5, 'otype': 'a', 'functions': '', 'max_depth': 6, 'max_dim': 10, 'random_state': 29802, 'erc': False, 'obj': 'fitness,complexity', 'shuffle': True, 'split': 0.75, 'fb': 0.5, 'scorer': '', 'feature_names': '', 'backprop': True, 'iters': 1, 'lr': 0.1, 'batch_size': 100, 'n_jobs': 1, 'hillclimb': False, 'logfile': '', 'max_time': 7200, 'residual_xo': False, 'stagewise_xo': False, 'stagewise_xo_tol': False, 'softmax_norm': False, 'save_pop': 0, 'normalize': True, 'val_from_arch': True, 'corr_delete_mutate': True, 'simplify': 0.005, 'protected_groups': '', 'tune_initial': False, 'tune_final': True, 'starting_pop': ''} +params = { + "pop_size": 1000, + "gens": 250, + "ml": "LinearRidgeRegression", + "classification": False, + "verbosity": 0, + "max_stall": 100, + "sel": "lexicase", + "surv": "nsga2", + "cross_rate": 0.75, + "root_xo_rate": 0.5, + "otype": "a", + "functions": "", + "max_depth": 6, + "max_dim": 10, + "random_state": 29802, + "erc": False, + "obj": "fitness,complexity", + "shuffle": True, + "split": 0.75, + "fb": 0.5, + "scorer": "", + "feature_names": "", + "backprop": True, + "iters": 1, + "lr": 0.1, + "batch_size": 100, + "n_jobs": 1, + "hillclimb": False, + "logfile": "", + "max_time": 7200, + "residual_xo": False, + "stagewise_xo": False, + "stagewise_xo_tol": False, + "softmax_norm": False, + "save_pop": 0, + "normalize": True, + "val_from_arch": True, + "corr_delete_mutate": True, + "simplify": 0.005, + "protected_groups": "", + "tune_initial": False, + "tune_final": True, + "starting_pop": "", +} diff --git a/experiment/methods/tuned/params/_ffxregressor.py b/experiment/methods/tuned/params/_ffxregressor.py index deefb909a..8ce4ba206 100644 --- a/experiment/methods/tuned/params/_ffxregressor.py +++ b/experiment/methods/tuned/params/_ffxregressor.py @@ -1 +1,5 @@ -params = {} +from __future__ import annotations + +from typing import Any + +params: dict[str, Any] = {} diff --git a/experiment/methods/tuned/params/_gpgomearegressor.py b/experiment/methods/tuned/params/_gpgomearegressor.py index d76749869..fa5812192 100644 --- a/experiment/methods/tuned/params/_gpgomearegressor.py +++ b/experiment/methods/tuned/params/_gpgomearegressor.py @@ -1 +1,31 @@ -params = {'caching': False, 'classweights': False, 'elitism': 1, 'erc': True, 'evaluations': 500000, 'functions': '+_-_*_p/_plog_sqrt_sin_cos', 'generations': -1, 'gomea': True, 'gomfos': 'LT', 'ims': False, 'initmaxtreeheight': 4, 'linearscaling': True, 'maxsize': 1000, 'maxtreeheight': 17, 'parallel': False, 'popsize': 1000, 'prob': 'symbreg', 'reproduction': 0.0, 'sbagx': 0.0, 'sblibtype': False, 'sbrdo': 0.0, 'seed': -1, 'silent': True, 'subcross': 0.5, 'submut': 0.5, 'syntuniqinit': 1000, 'time': 7200, 'tournament': 4, 'unifdepthvar': True} +params = { + "caching": False, + "classweights": False, + "elitism": 1, + "erc": True, + "evaluations": 500000, + "functions": "+_-_*_p/_plog_sqrt_sin_cos", + "generations": -1, + "gomea": True, + "gomfos": "LT", + "ims": False, + "initmaxtreeheight": 4, + "linearscaling": True, + "maxsize": 1000, + "maxtreeheight": 17, + "parallel": False, + "popsize": 1000, + "prob": "symbreg", + "reproduction": 0.0, + "sbagx": 0.0, + "sblibtype": False, + "sbrdo": 0.0, + "seed": -1, + "silent": True, + "subcross": 0.5, + "submut": 0.5, + "syntuniqinit": 1000, + "time": 7200, + "tournament": 4, + "unifdepthvar": True, +} diff --git a/experiment/methods/tuned/params/_gplearn.py b/experiment/methods/tuned/params/_gplearn.py index a527d307f..f2a1f2641 100644 --- a/experiment/methods/tuned/params/_gplearn.py +++ b/experiment/methods/tuned/params/_gplearn.py @@ -1 +1,20 @@ -params = {'generations': 500, 'init_method': 'half and half', 'low_memory': False, 'max_samples': 1.0, 'metric': 'mean absolute error', 'n_jobs': 1, 'p_crossover': 0.9, 'p_hoist_mutation': 0.01, 'p_point_mutation': 0.01, 'p_point_replace': 0.05, 'p_subtree_mutation': 0.01, 'parsimony_coefficient': 0.001, 'population_size': 1000, 'random_state': 11964, 'stopping_criteria': 0.0, 'tournament_size': 20, 'verbose': 0, 'warm_start': False} +params = { + "generations": 500, + "init_method": "half and half", + "low_memory": False, + "max_samples": 1.0, + "metric": "mean absolute error", + "n_jobs": 1, + "p_crossover": 0.9, + "p_hoist_mutation": 0.01, + "p_point_mutation": 0.01, + "p_point_replace": 0.05, + "p_subtree_mutation": 0.01, + "parsimony_coefficient": 0.001, + "population_size": 1000, + "random_state": 11964, + "stopping_criteria": 0.0, + "tournament_size": 20, + "verbose": 0, + "warm_start": False, +} diff --git a/experiment/methods/tuned/params/_itearegressor.py b/experiment/methods/tuned/params/_itearegressor.py index 8f4216477..a7cd84f83 100644 --- a/experiment/methods/tuned/params/_itearegressor.py +++ b/experiment/methods/tuned/params/_itearegressor.py @@ -1 +1,6 @@ -params = {'ngens': 500, 'nonzeroexps': 1, 'npop': 1000, 'transfunctions': '[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]'} +params = { + "ngens": 500, + "nonzeroexps": 1, + "npop": 1000, + "transfunctions": "[Id, Tanh, Sin, Cos, Log, Exp, SqrtAbs]", +} diff --git a/experiment/methods/tuned/params/_mrgpregressor.py b/experiment/methods/tuned/params/_mrgpregressor.py index 81da61361..4eab31c1b 100644 --- a/experiment/methods/tuned/params/_mrgpregressor.py +++ b/experiment/methods/tuned/params/_mrgpregressor.py @@ -1 +1,9 @@ -params = {'g': 250, 'max_len': 6, 'popsize': 1000, 'rt_cross': 0.5, 'rt_mut': 0.5, 'time_out': 7200, 'tmp_dir': None} +params = { + "g": 250, + "max_len": 6, + "popsize": 1000, + "rt_cross": 0.5, + "rt_mut": 0.5, + "time_out": 7200, + "tmp_dir": None, +} diff --git a/experiment/methods/tuned/params/_operonregressor.py b/experiment/methods/tuned/params/_operonregressor.py index bf0c0bcda..542593f30 100644 --- a/experiment/methods/tuned/params/_operonregressor.py +++ b/experiment/methods/tuned/params/_operonregressor.py @@ -1 +1,34 @@ -params = {'allowed_symbols': 'add,mul,aq,constant,variable', 'brood_size': 10, 'comparison_factor': 0, 'crossover_internal_probability': 0.9, 'crossover_probability': 1.0, 'objectives': ['r2'], 'female_selector': 'tournament', 'generations': 10000, 'initialization_method': 'btc', 'local_iterations': 5, 'male_selector': 'tournament', 'max_depth': 10, 'max_evaluations': 500000, 'max_length': 50, 'max_selection_pressure': 100, 'mutation': {'onepoint': 1.0, 'changevar': 1.0, 'changefunc': 1.0, 'insertsubtree': 1.0, 'replacesubtree': 1.0, 'removesubtree': 1.0}, 'mutation_probability': 0.25, 'n_threads': 1, 'offspring_generator': 'basic', 'pool_size': 500, 'population_size': 500, 'random_state': 22118, 'reinserter': 'keep-best', 'time_limit': 7200, 'tournament_size': 5} +params = { + "allowed_symbols": "add,mul,aq,constant,variable", + "brood_size": 10, + "comparison_factor": 0, + "crossover_internal_probability": 0.9, + "crossover_probability": 1.0, + "objectives": ["r2"], + "female_selector": "tournament", + "generations": 10000, + "initialization_method": "btc", + "local_iterations": 5, + "male_selector": "tournament", + "max_depth": 10, + "max_evaluations": 500000, + "max_length": 50, + "max_selection_pressure": 100, + "mutation": { + "onepoint": 1.0, + "changevar": 1.0, + "changefunc": 1.0, + "insertsubtree": 1.0, + "replacesubtree": 1.0, + "removesubtree": 1.0, + }, + "mutation_probability": 0.25, + "n_threads": 1, + "offspring_generator": "basic", + "pool_size": 500, + "population_size": 500, + "random_state": 22118, + "reinserter": "keep-best", + "time_limit": 7200, + "tournament_size": 5, +} diff --git a/experiment/methods/tuned/params/_sembackpropgp.py b/experiment/methods/tuned/params/_sembackpropgp.py index f367548a6..d1dbf3e1b 100644 --- a/experiment/methods/tuned/params/_sembackpropgp.py +++ b/experiment/methods/tuned/params/_sembackpropgp.py @@ -1 +1,31 @@ -params = {'caching': False, 'classweights': False, 'elitism': 1, 'erc': True, 'evaluations': 500000, 'functions': '+_-_*_aq_plog_sin_cos', 'generations': -1, 'gomea': False, 'gomfos': 'LT', 'ims': False, 'initmaxtreeheight': 6, 'linearscaling': True, 'maxsize': 1000, 'maxtreeheight': 20, 'parallel': False, 'popsize': 1000, 'prob': 'symbreg', 'reproduction': 0.0, 'sbagx': False, 'sblibtype': 'p_10_9999_l_n', 'sbrdo': 0.9, 'seed': -1, 'silent': True, 'subcross': 0.0, 'submut': 0.1, 'syntuniqinit': 1000, 'time': 7200, 'tournament': 4, 'unifdepthvar': True} +params = { + "caching": False, + "classweights": False, + "elitism": 1, + "erc": True, + "evaluations": 500000, + "functions": "+_-_*_aq_plog_sin_cos", + "generations": -1, + "gomea": False, + "gomfos": "LT", + "ims": False, + "initmaxtreeheight": 6, + "linearscaling": True, + "maxsize": 1000, + "maxtreeheight": 20, + "parallel": False, + "popsize": 1000, + "prob": "symbreg", + "reproduction": 0.0, + "sbagx": False, + "sblibtype": "p_10_9999_l_n", + "sbrdo": 0.9, + "seed": -1, + "silent": True, + "subcross": 0.0, + "submut": 0.1, + "syntuniqinit": 1000, + "time": 7200, + "tournament": 4, + "unifdepthvar": True, +} diff --git a/experiment/methods/tuned/sembackpropgp.py b/experiment/methods/tuned/sembackpropgp.py index c95cec908..381832070 100644 --- a/experiment/methods/tuned/sembackpropgp.py +++ b/experiment/methods/tuned/sembackpropgp.py @@ -1,10 +1,11 @@ -from ..sembackpropgp import complexity, model, est -from pyGPGOMEA import GPGOMEARegressor as GPG +from ..sembackpropgp import complexity as complexity +from ..sembackpropgp import est as est +from ..sembackpropgp import model as model from .params._sembackpropgp import params est.set_params(**params) -est.functions = '+_-_*_aq_plog_sin_cos' +est.functions = "+_-_*_aq_plog_sin_cos" -#double the evals -est.evaluations=1000000 -est.time = 8*60*60 +# double the evals +est.evaluations = 1000000 +est.time = 8 * 60 * 60 diff --git a/experiment/methods/udsr/regressor.py b/experiment/methods/udsr/regressor.py index f7b0e4a5b..167ffc9fc 100644 --- a/experiment/methods/udsr/regressor.py +++ b/experiment/methods/udsr/regressor.py @@ -1,19 +1,13 @@ -"""Borrows gpzgd's example of how to wrap a CLI learner with a sklearn interface -""" +"""Borrows gpzgd's example of how to wrap a CLI learner with a sklearn interface""" -from dso import DeepSymbolicRegressor -from dso import DeepSymbolicOptimizer - -import sys -import os -from tempfile import TemporaryDirectory import json -import numpy as np - -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_is_fitted, check_X_y, check_array +from tempfile import TemporaryDirectory +import numpy as np import tensorflow as tf +from dso import DeepSymbolicOptimizer +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.utils.validation import check_array, check_is_fitted, check_X_y print("Num GPUs Available: ", tf.test.is_gpu_available()) print("CUDA Available: ", tf.test.is_built_with_cuda()) @@ -29,22 +23,34 @@ hyper_params = [] for f in function_set: - hyper_params.append({ - 'function_set' : [f], - 'degree' : [2], - 'run_gp_meld' : [True] - }) - + hyper_params.append({"function_set": [f], "degree": [2], "run_gp_meld": [True]}) + class uDSRRegressor(BaseEstimator, RegressorMixin): - def __init__(self, random_state=42, - function_set=["add","sub","mul","div","sin","cos","exp","log","sqrt","const","poly"], - degree=3, run_gp_meld=True): - + def __init__( + self, + random_state=42, + function_set=[ + "add", + "sub", + "mul", + "div", + "sin", + "cos", + "exp", + "log", + "sqrt", + "const", + "poly", + ], + degree=3, + run_gp_meld=True, + ): + self.random_state = random_state self.function_set = function_set - self.degree = degree - self.run_gp_meld = run_gp_meld + self.degree = degree + self.run_gp_meld = run_gp_meld def fit(self, X, y): # 1. create a temporary directory to store the training data set @@ -52,58 +58,50 @@ def fit(self, X, y): # 3. create a temp file and store the data X, y = check_X_y(X, y, accept_sparse=False) if len(y.shape) == 1: - Z = np.hstack((X, y[:,None])) + Z = np.hstack((X, y[:, None])) else: Z = np.hstack((X, y)) - fname = temp_dir + "/tmpdata.csv" + fname = temp_dir + "/tmpdata.csv" np.savetxt(f"{fname}", Z, delimiter=",") - + # 4. create a temp config file config = { - 'task' : { - 'task_type' : 'regression', - 'dataset' : fname, - + "task": { + "task_type": "regression", + "dataset": fname, # To customize a function set, edit this! See functions.py for a list of # supported functions. Note "const" will add placeholder constants that # will be optimized within the training loop. This will considerably # increase runtime. "function_set": self.function_set, - # Metric to be used for the reward function. See regression.py for # supported metrics. - "metric" : "inv_nrmse", - "metric_params" : [1.0], - + "metric": "inv_nrmse", + "metric_params": [1.0], # Optional alternate metric to be used at evaluation time. - "extra_metric_test" : None, - "extra_metric_test_params" : [], - + "extra_metric_test": None, + "extra_metric_test_params": [], # NRMSE threshold for early stopping. This is useful for noiseless # benchmark problems when DSO discovers the True solution. - "threshold" : 1e-12, - + "threshold": 1e-12, # With protected=False, floating-point errors (e.g. log of negative # number) will simply returns a minimal reward. With protected=True, # "protected" functions will prevent floating-point errors, but may - # introduce discontinuities in the learned functions. - "protected" : True, - + # introduce discontinuities in the learned functions. + "protected": True, # You can add artificial reward noise directly to the reward function. # Note this does NOT add noise to the dataset. - "reward_noise" : 0.0, - "reward_noise_type" : "r", - "normalize_variance" : False, - + "reward_noise": 0.0, + "reward_noise_type": "r", + "normalize_variance": False, # Set of thresholds (shared by all input variables) for building # decision trees. Note that no StateChecker will be added to Library # if decision_tree_threshold_set is an empty list or null. - "decision_tree_threshold_set" : [], - + "decision_tree_threshold_set": [], # Parameters for optimizing the "poly" token. # Note: poly_optimizer is turned on if and only if "poly" is in function_set. - "poly_optimizer_params" : { + "poly_optimizer_params": { # The (maximal) degree of the polynomials used to fit the data "degree": self.degree, # Cutoff value for the coefficients of polynomials. Coefficients @@ -114,7 +112,7 @@ def fit(self, X, y): # of least squares regressor "dso_least_squares". "regressor": "dso_least_squares", "regressor_params": { - # Cutoff value for p-value of coefficients. Coefficients with + # Cutoff value for p-value of coefficients. Coefficients with # larger p-values are forced to zero. "cutoff_p_value": 1.0, # Maximum number of terms in the polynomial. If more coefficients are nonzero, @@ -122,93 +120,68 @@ def fit(self, X, y): "n_max_terms": None, # Cutoff value for the coefficients of polynomials. Coefficients # with magnitude less than this value will be regarded as 0. - "coef_tol": 1e-6 - } - } + "coef_tol": 1e-6, + }, + }, }, - # Hyperparameters related to genetic programming hybrid methods. - "gp_meld" : { - "run_gp_meld" : self.run_gp_meld, - "population_size" : 25, - "generations" : 25, - "crossover_operator" : "cxOnePoint", - "p_crossover" : 0.5, - "mutation_operator" : "multi_mutate", - "p_mutate" : 0.5, - "tournament_size" : 5, - "train_n" : 50, - "mutate_tree_max" : 3, - "verbose" : True, + "gp_meld": { + "run_gp_meld": self.run_gp_meld, + "population_size": 25, + "generations": 25, + "crossover_operator": "cxOnePoint", + "p_crossover": 0.5, + "mutation_operator": "multi_mutate", + "p_mutate": 0.5, + "tournament_size": 5, + "train_n": 50, + "mutate_tree_max": 3, + "verbose": True, # Speeds up processing when doing expensive evaluations. - "parallel_eval" : False + "parallel_eval": False, }, - # Only the key training hyperparameters are listed here. See # config_common.json for the full list. - "training" : { - "n_samples" : 10000, - "batch_size" : 500, - "epsilon" : 0.02, + "training": { + "n_samples": 10000, + "batch_size": 500, + "epsilon": 0.02, # Recommended to set this to as many cores as you can use! Especially if # using the "const" token. - "n_cores_batch" : -1 + "n_cores_batch": -1, }, - # Only the key RNN controller hyperparameters are listed here. See # config_common.json for the full list. - "controller" : { + "controller": { "learning_rate": 0.0025, - "entropy_weight" : 0.03, - "entropy_gamma" : 0.7, - + "entropy_weight": 0.03, + "entropy_gamma": 0.7, # EXPERIMENTAL: Priority queue training hyperparameters. - "pqt" : True, - "pqt_k" : 10, - "pqt_batch_size" : 1, - "pqt_weight" : 200.0, - "pqt_use_pg" : False + "pqt": True, + "pqt_k": 10, + "pqt_batch_size": 1, + "pqt_weight": 200.0, + "pqt_use_pg": False, }, - # Hyperparameters related to including in situ priors and constraints. Each # prior must explicitly be turned "on" or it will not be used. See # config_common.json for descriptions of each prior. "prior": { - "length" : { - "min_" : 4, - "max_" : 100, - "on" : True - }, - "inverse" : { - "on" : True - }, - "trig" : { - "on" : True - }, - "const" : { - "on" : True - }, - "no_inputs" : { - "on" : True - }, - "uniform_arity" : { - "on" : True - }, - "soft_length" : { - "loc" : 10, - "scale" : 5, - "on" : True - }, - "domain_range" : { - "on" : True - } + "length": {"min_": 4, "max_": 100, "on": True}, + "inverse": {"on": True}, + "trig": {"on": True}, + "const": {"on": True}, + "no_inputs": {"on": True}, + "uniform_arity": {"on": True}, + "soft_length": {"loc": 10, "scale": 5, "on": True}, + "domain_range": {"on": True}, + }, + "experiment": { + "logdir": None, }, - 'experiment' : { - 'logdir' : None, - } } - - cname = temp_dir + "/config.json" + + cname = temp_dir + "/config.json" with open(cname, "w") as config_file: json.dump(config, config_file, indent=4) @@ -231,24 +204,25 @@ def predict(self, X): est = uDSRRegressor() -def model(est, X=None): # Should work for both methods +def model(est, X=None): # Should work for both methods # clean_pred_model from assess_symbolic model has fixes to the string rep return str(est.program_.sympy_expr) + # define eval_kwargs. eval_kwargs = dict(use_dataframe=False) - + if __name__ == "__main__": - import numpy as np - + import numpy as np + # Generate some data np.random.seed(0) X = np.random.random((10, 2)) - y = np.sin(X[:,0]) + X[:,1] ** 2 + y = np.sin(X[:, 0]) + X[:, 1] ** 2 # Fit the model - est.fit(X, y) # Should solve in ~10 seconds + est.fit(X, y) # Should solve in ~10 seconds # View the best expression print(est.program_.pretty()) diff --git a/experiment/methods/xgboost/regressor.py b/experiment/methods/xgboost/regressor.py index caa7d220a..e8a72d5ab 100644 --- a/experiment/methods/xgboost/regressor.py +++ b/experiment/methods/xgboost/regressor.py @@ -1,19 +1,21 @@ -import xgboost import numpy as np +import xgboost hyper_params = [ { - 'n_estimators' : (10, 100, 1000), - 'learning_rate' : (0.0001,0.01, 0.1), - 'gamma' : (0,0.1,0.2,0.3,0.4), - 'subsample' : (0.5, 0.75), + "n_estimators": (10, 100, 1000), + "learning_rate": (0.0001, 0.01, 0.1), + "gamma": (0, 0.1, 0.2, 0.3, 0.4), + "subsample": (0.5, 0.75), }, ] -est=xgboost.XGBRegressor(max_depth=6) +est = xgboost.XGBRegressor(max_depth=6) + def complexity(est): - c = np.sum([m.count(':') for m in est._Booster.get_dump()]) + c = np.sum([m.count(":") for m in est._Booster.get_dump()]) return int(c) + model = None diff --git a/experiment/methods_table.ipynb b/experiment/methods_table.ipynb index 7accd3403..cfed37152 100644 --- a/experiment/methods_table.ipynb +++ b/experiment/methods_table.ipynb @@ -6,61 +6,60 @@ "metadata": {}, "outputs": [], "source": [ - "from glob import glob\n", "import importlib\n", - "from sklearn.model_selection import ParameterGrid\n", + "from glob import glob\n", + "\n", "from symbolic_algs import symbolic_algs\n", "\n", "sr_frames = []\n", "ml_frames = []\n", "\n", - "for m in [n.split('/')[-1][:-3] for n in glob('methods/*.py')]:\n", - " if m.startswith('_'): continue\n", - " if 'EHC' in m: continue\n", - " if 'FFX' in m: continue\n", - " \n", - " m_short = m.replace('Regressor','')\n", - " \n", - " alg = importlib.__import__('methods.'+m,\n", - " globals(),locals(), \n", - " ['*']\n", - " ) \n", - " if m_short == 'DSR':\n", - " hp = {'batch_size':alg.batch_sizes}\n", + "for m in [n.split(\"/\")[-1][:-3] for n in glob(\"methods/*.py\")]:\n", + " if m.startswith(\"_\"):\n", + " continue\n", + " if \"EHC\" in m:\n", + " continue\n", + " if \"FFX\" in m:\n", + " continue\n", + "\n", + " m_short = m.replace(\"Regressor\", \"\")\n", + "\n", + " alg = importlib.__import__(\"methods.\" + m, globals(), locals(), [\"*\"])\n", + " if m_short == \"DSR\":\n", + " hp = {\"batch_size\": alg.batch_sizes}\n", " else:\n", " hp = alg.hyper_params\n", " if isinstance(hp, list):\n", " tuneables = hp[0].keys()\n", " else:\n", " tuneables = hp.keys()\n", - " \n", - "# print(m)\n", - "# print(hp)\n", - " \n", + "\n", + " # print(m)\n", + " # print(hp)\n", + "\n", " # get tuned for symbolic_algs\n", " if m not in symbolic_algs:\n", " if isinstance(hp, list):\n", " for p in hp:\n", " ml_frames.append([m_short, p])\n", " else:\n", - " ml_frames.append([m_short, hp]) # hp_tuned])\n", + " ml_frames.append([m_short, hp]) # hp_tuned])\n", " continue\n", " else:\n", - " alg_tuned = importlib.__import__('methods.tuned.params._'+m.lower(),\n", - " globals(),locals(), \n", - " ['params']\n", - " ) \n", - " hp_tuned = {k:v for k,v in alg_tuned.params.items() if k in tuneables}\n", + " alg_tuned = importlib.__import__(\n", + " \"methods.tuned.params._\" + m.lower(), globals(), locals(), [\"params\"]\n", + " )\n", + " hp_tuned = {k: v for k, v in alg_tuned.params.items() if k in tuneables}\n", " if isinstance(hp, list):\n", " for p in hp:\n", " sr_frames.append([m_short, p])\n", " else:\n", - " sr_frames.append([m_short, hp]) # hp_tuned])\n", - " \n", + " sr_frames.append([m_short, hp]) # hp_tuned])\n", + "\n", "# hp_pg = ParameterGrid(hp)\n", "# print('tuned:',hp_tuned)\n", "# print(list(hp_pg))\n", - "print('ML')\n", + "print(\"ML\")\n", "print(ml_frames)" ] }, @@ -70,21 +69,20 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", - "import json\n", - "from utils import jsonify\n", "def hp_format(p):\n", - "# pdb.set_trace()\n", + " # pdb.set_trace()\n", " if isinstance(p, dict):\n", - " for k,v in p.items():\n", - " if isinstance(v, list) and len(v)==1:\n", + " for k, v in p.items():\n", + " if isinstance(v, list) and len(v) == 1:\n", " p[k] = v[0]\n", " elif isinstance(v, float):\n", - " p[k] = round(v,2)\n", - " elif isinstance(p,list):\n", + " p[k] = round(v, 2)\n", + " elif isinstance(p, list):\n", " return [hp_format(i) for i in p]\n", - " \n", + "\n", " return p\n", + "\n", + "\n", "# args = dict(sort_keys=True, indent=4)\n", "# # if isinstance(p, list):\n", "# # return '\\\\\\\\'.join([json.dumps(jsonify(i), **args)\n", @@ -92,16 +90,17 @@ "# # else:\n", "# return json.dumps(jsonify(p), **args)\n", "\n", + "\n", "def add_midrules(ltx):\n", " ltx2 = ltx.splitlines()\n", " size = len(ltx2) - 1\n", - " for i,line in enumerate(ltx.splitlines()[::-1]):\n", - " method = line.split('&')[0].strip().replace('\\\\','')\n", - " if method in methods: \n", - "# pdb.set_trace()\n", - " ltx2.insert(size-i,'\\midrule')\n", - " \n", - " return '\\n'.join(ltx2)" + " for i, line in enumerate(ltx.splitlines()[::-1]):\n", + " method = line.split(\"&\")[0].strip().replace(\"\\\\\", \"\")\n", + " if method in methods:\n", + " # pdb.set_trace()\n", + " ltx2.insert(size - i, \"\\midrule\")\n", + "\n", + " return \"\\n\".join(ltx2)" ] }, { @@ -110,7 +109,7 @@ "metadata": {}, "outputs": [], "source": [ - "methods" + "# Methods are defined from sr_df in the following cell." ] }, { @@ -119,50 +118,54 @@ "metadata": {}, "outputs": [], "source": [ + "from math import ceil\n", + "\n", "import pandas as pd\n", - "from math import floor, ceil\n", - "column_format = 'l p{37em}'\n", - "pd.set_option('display.max_colwidth', 1000)\n", - "sr_df = pd.DataFrame(sr_frames, columns = ['Method','Hyperparameters']).sort_values(by='Method')\n", + "\n", + "column_format = \"l p{37em}\"\n", + "pd.set_option(\"display.max_colwidth\", 1000)\n", + "sr_df = pd.DataFrame(sr_frames, columns=[\"Method\", \"Hyperparameters\"]).sort_values(\n", + " by=\"Method\"\n", + ")\n", "methods = sr_df.Method.unique()\n", - "for i in [1,2]:\n", - " print_methods = methods[:ceil(len(methods)/2)] if i == 1 else methods[ceil(len(methods)/2):]\n", + "for i in [1, 2]:\n", + " print_methods = (\n", + " methods[: ceil(len(methods) / 2)]\n", + " if i == 1\n", + " else methods[ceil(len(methods) / 2) :]\n", + " )\n", " df = sr_df.loc[sr_df.Method.isin(print_methods)]\n", - " df.loc[df['Method'] == df['Method'].shift(1), 'Method'] = ''\n", + " df.loc[df[\"Method\"] == df[\"Method\"].shift(1), \"Method\"] = \"\"\n", " sr_ltx = df.to_latex(\n", - " index=False,\n", - " column_format=column_format,\n", - " float_format='{:0.1f}',\n", - " formatters={\n", - " 'Hyperparameters': hp_format\n", - " }\n", - " )\n", + " index=False,\n", + " column_format=column_format,\n", + " float_format=\"{:0.1f}\",\n", + " formatters={\"Hyperparameters\": hp_format},\n", + " )\n", " # insert midrule between methods\n", " sr_ltx = add_midrules(sr_ltx)\n", - "# for s in sr_ltx.splitlines(): print(s[:80])\n", - " with open('../../paper/tables/sr_methods_hp_table'+str(i)+'.tex','w') as f:\n", + " # for s in sr_ltx.splitlines(): print(s[:80])\n", + " with open(\"../../paper/tables/sr_methods_hp_table\" + str(i) + \".tex\", \"w\") as f:\n", " f.write(sr_ltx)\n", - " \n", + "\n", "##########\n", "# ML table\n", "##########\n", - "ml_df = pd.DataFrame(ml_frames, columns = ['Method','Hyperparameters'])\n", - "ml_df.loc[ml_df['Method'] == ml_df['Method'].shift(1), 'Method'] = ''\n", + "ml_df = pd.DataFrame(ml_frames, columns=[\"Method\", \"Hyperparameters\"])\n", + "ml_df.loc[ml_df[\"Method\"] == ml_df[\"Method\"].shift(1), \"Method\"] = \"\"\n", "ml_ltx = ml_df.to_latex(\n", - " index=False,\n", - " column_format=column_format,\n", - " float_format='{:0.1f}',\n", - " formatters={\n", - " 'Hyperparameters': hp_format\n", - " }\n", - " )\n", + " index=False,\n", + " column_format=column_format,\n", + " float_format=\"{:0.1f}\",\n", + " formatters={\"Hyperparameters\": hp_format},\n", + ")\n", "# insert midrule between methods\n", "ml_ltx2 = ml_ltx.splitlines()\n", - "for i in range(len(ml_ltx),0):\n", - " ml_ltx2.insert(i,'\\midrule') \n", - "ml_ltx = '\\n'.join(ml_ltx2)\n", - " #add_midrules(sr_ltx)\n", - "with open('../../paper/tables/ml_methods_hp_table.tex','w') as f:\n", + "for i in range(len(ml_ltx), 0):\n", + " ml_ltx2.insert(i, \"\\midrule\")\n", + "ml_ltx = \"\\n\".join(ml_ltx2)\n", + "# add_midrules(sr_ltx)\n", + "with open(\"../../paper/tables/ml_methods_hp_table.tex\", \"w\") as f:\n", " f.write(ml_ltx)" ] }, @@ -172,7 +175,8 @@ "metadata": {}, "outputs": [], "source": [ - "for s in ml_ltx.splitlines(): print(s[:80])" + "for s in ml_ltx.splitlines():\n", + " print(s[:80])" ] }, { @@ -181,8 +185,7 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "add_midrules(sr_ltx)\n" + "add_midrules(sr_ltx)" ] }, { diff --git a/experiment/metrics/evaluation.py b/experiment/metrics/evaluation.py index 75c7a2035..36fdf9772 100644 --- a/experiment/metrics/evaluation.py +++ b/experiment/metrics/evaluation.py @@ -1,7 +1,8 @@ +import signal + import numpy as np -from sklearn.metrics import accuracy_score, mean_squared_error import sympy as sp -from sympy.parsing.sympy_parser import parse_expr +from sklearn.metrics import mean_squared_error """ Timeout handling @@ -9,30 +10,34 @@ MAXTIME = 60 -import signal + class SimplifyTimeOutException(Exception): pass + def alarm_handler(signum, frame): - print(f"raising SimplifyTimeOutException") + print("raising SimplifyTimeOutException") raise SimplifyTimeOutException """ For all of these, higher is better """ + + def accuracy(est, X, y): pred = est.predict(X) mse = mean_squared_error(y, pred) y_var = np.var(y) - if(y_var == 0.0): + if y_var == 0.0: y_var = 1e-9 r2 = 1 - mse / y_var r2 = np.round(r2, 3) return r2 + """ Utilities """ @@ -49,79 +54,77 @@ def round_floats(ex1): return ex2 - def get_symbolic_model(pred_model, local_dict): # TODO: update namespace for exact_formula runs sp_model = sp.parse_expr(pred_model, local_dict=local_dict) sp_model = round_floats(sp_model) signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: sp_model = sp.simplify(sp_model) except Exception as e: - print('Warning: simplify failed. Msg:',e) + print("Warning: simplify failed. Msg:", e) pass return sp_model + def simplicity(pred_model, feature_names): - local_dict = {f:sp.Symbol(f) for f in feature_names} + local_dict = {f: sp.Symbol(f) for f in feature_names} sp_model = get_symbolic_model(pred_model, local_dict) # compute num. components num_components = 0 for _ in sp.preorder_traversal(sp_model): num_components += 1 # compute simplicity as per juding criteria - simplicity = -np.round(np.log(num_components)/np.log(5), 1) + simplicity = -np.round(np.log(num_components) / np.log(5), 1) return simplicity + """ Problem specific """ + + def symbolic_equivalence(true_model, pred_model, local_dict): """Check whether symbolic model is equivalent to the ground truth model.""" sp_model = get_symbolic_model(pred_model, local_dict) sym_diff = round_floats(true_model - sp_model) - sym_frac = round_floats(sp_model/true_model) - print('true_model:',true_model, '; \npred_model:',sp_model) + sym_frac = round_floats(sp_model / true_model) + print("true_model:", true_model, "; \npred_model:", sp_model) try: - diff_const=sym_diff.is_constant(simplify=False) - frac_const=sym_frac.is_constant(simplify=False) + diff_const = sym_diff.is_constant(simplify=False) + frac_const = sym_frac.is_constant(simplify=False) # check if we can skip simplification if not diff_const and not frac_const: signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(MAXTIME) # maximum time, defined above + signal.alarm(MAXTIME) # maximum time, defined above try: if not diff_const: sym_diff = sp.simplify(sym_diff) - diff_const=sym_diff.is_constant() + diff_const = sym_diff.is_constant() if not frac_const: sym_frac = sp.simplify(sym_frac) - frac_const=sym_frac.is_constant() + frac_const = sym_frac.is_constant() except Exception as e: - print('Warning: simplify failed. Msg:',e) + print("Warning: simplify failed. Msg:", e) pass - except Exception as e: - print('const checking failed.') - diff_const=False - frac_const=False + except Exception: + print("const checking failed.") + diff_const = False + frac_const = False pass - result = dict( - equivalent = ( - str(sym_diff) == '0' - or diff_const - or frac_const - ), - sym_diff = str(sym_diff), - sym_frac = str(sym_frac), - true_model = str(true_model), - pred_model = str(sp_model) - ) - - print('result:',result) + equivalent=(str(sym_diff) == "0" or diff_const or frac_const), + sym_diff=str(sym_diff), + sym_frac=str(sym_frac), + true_model=str(true_model), + pred_model=str(sp_model), + ) + + print("result:", result) return result diff --git a/experiment/metrics/test_evaluation.py b/experiment/metrics/test_evaluation.py index 4917b17ef..b080f0c93 100644 --- a/experiment/metrics/test_evaluation.py +++ b/experiment/metrics/test_evaluation.py @@ -1,81 +1,101 @@ -from .evaluation import * +import sympy as sp + +from .evaluation import get_symbolic_model, simplicity, symbolic_equivalence # define local namespace dictionary SEIR_VARS = {} -for s in ['dS', 'dE', 'dI', 'dR', - 'x_S', 'x_E', 'x_I', 'x_R', 'x_rzero', 'x_gamma', - 'x_sigma', 'x_c', 'x_N', 'x_Aux']: - SEIR_VARS.update({ s: sp.Symbol(s) }) +for s in [ + "dS", + "dE", + "dI", + "dR", + "x_S", + "x_E", + "x_I", + "x_R", + "x_rzero", + "x_gamma", + "x_sigma", + "x_c", + "x_N", + "x_Aux", +]: + SEIR_VARS.update({s: sp.Symbol(s)}) seir_gts = { - "dS": get_symbolic_model('-x_gamma*x_rzero*x_S*x_I/x_N',SEIR_VARS), - "dE": get_symbolic_model('x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E', SEIR_VARS), - "dI": get_symbolic_model('x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N', SEIR_VARS), - "dR": get_symbolic_model('x_gamma*x_I-x_c*x_R*x_I/x_N', SEIR_VARS) + "dS": get_symbolic_model("-x_gamma*x_rzero*x_S*x_I/x_N", SEIR_VARS), + "dE": get_symbolic_model("x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E", SEIR_VARS), + "dI": get_symbolic_model("x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N", SEIR_VARS), + "dR": get_symbolic_model("x_gamma*x_I-x_c*x_R*x_I/x_N", SEIR_VARS), } + def test_seir_true(): """SEIR models evaluate as solutions""" pred_models = [ - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N'), - ("dE", 'x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E'), - ("dI", 'x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N'), - ("dR", 'x_gamma*x_I-x_c*x_R*x_I/x_N'), - ("dE", 'x_E*(-x_sigma) + x_gamma*x_S*x_I/(x_N/x_rzero)'), - ("dE", '(-x_E*x_sigma**2 + x_gamma*x_sigma*x_rzero*x_S*x_I/x_N)/x_sigma') + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N"), + ("dE", "x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E"), + ("dI", "x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N"), + ("dR", "x_gamma*x_I-x_c*x_R*x_I/x_N"), + ("dE", "x_E*(-x_sigma) + x_gamma*x_S*x_I/(x_N/x_rzero)"), + ("dE", "(-x_E*x_sigma**2 + x_gamma*x_sigma*x_rzero*x_S*x_I/x_N)/x_sigma"), ] - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_offset(): """SEIR models evaluate as solutions when offset""" pred_models = [ - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N+1.80'), - ("dE", 'x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E - 3.12'), - ("dI", 'x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N + 678.0'), - ("dR", 'x_gamma*x_I-x_c*x_R*x_I/x_N-0.043'), + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N+1.80"), + ("dE", "x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E - 3.12"), + ("dI", "x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N + 678.0"), + ("dR", "x_gamma*x_I-x_c*x_R*x_I/x_N-0.043"), ] - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_scale(): """SEIR models evaluate as solutions when scaled""" pred_models = { - ("dS", '-x_gamma*x_rzero*x_S*x_I/x_N*1.80'), - ("dE", '(x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E) * 3.12'), - ("dI", '(x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N)/678.0'), - ("dR", '(x_gamma*x_I-x_c*x_R*x_I/x_N)*0.043'), - ("dE", 'x_E*(-3*x_sigma) + x_gamma*x_S*x_I/(0.33333*x_N/x_rzero)'), + ("dS", "-x_gamma*x_rzero*x_S*x_I/x_N*1.80"), + ("dE", "(x_gamma*x_rzero*x_S*x_I/x_N - x_sigma*x_E) * 3.12"), + ("dI", "(x_sigma*x_E-x_gamma*x_I+x_c*x_R*x_I/x_N)/678.0"), + ("dR", "(x_gamma*x_I-x_c*x_R*x_I/x_N)*0.043"), + ("dE", "x_E*(-3*x_sigma) + x_gamma*x_S*x_I/(0.33333*x_N/x_rzero)"), } - for k,v in pred_models: + for k, v in pred_models: result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_seir_true_round(): """SEIR models evaluate as solutions when rounded""" pred_models = { - "dE": '(x_gamma*x_rzero*x_S*x_I/x_N*1.0003 - x_sigma*x_E*0.9999)', - "dI": 'x_sigma*x_E*0.9999-x_gamma*x_I+x_c*x_R*x_I/x_N*1.0001', - "dR": '(x_gamma*x_I*1.0004-x_c*x_R*x_I/x_N)*0.043', + "dE": "(x_gamma*x_rzero*x_S*x_I/x_N*1.0003 - x_sigma*x_E*0.9999)", + "dI": "x_sigma*x_E*0.9999-x_gamma*x_I+x_c*x_R*x_I/x_N*1.0001", + "dR": "(x_gamma*x_I*1.0004-x_c*x_R*x_I/x_N)*0.043", } - for k,v in pred_models.items(): + for k, v in pred_models.items(): result = symbolic_equivalence(seir_gts[k], v, SEIR_VARS) - assert result['equivalent'] + assert result["equivalent"] + def test_simplicity_comparator(): """Test comparison has the right sign""" - big = '2.35*( 0.4 * x1 * x2 - 1.5 * x1 + 2.5 * x2 + 1 + log(30 * x3**2))' - small = '2.35*( 0.4 * x1 * x2 - 1.5 * x1 )' - features = ['x1','x2','x3','x4','x5'] - assert simplicity(big, features) < simplicity(small, features) \ No newline at end of file + big = "2.35*( 0.4 * x1 * x2 - 1.5 * x1 + 2.5 * x2 + 1 + log(30 * x3**2))" + small = "2.35*( 0.4 * x1 * x2 - 1.5 * x1 )" + features = ["x1", "x2", "x3", "x4", "x5"] + assert simplicity(big, features) < simplicity(small, features) diff --git a/experiment/read_file.py b/experiment/read_file.py index da901c5a3..b9f7e7800 100755 --- a/experiment/read_file.py +++ b/experiment/read_file.py @@ -1,30 +1,31 @@ -import pandas as pd import numpy as np +import pandas as pd -def read_file(filename, label='target', use_dataframe=True, sep=None): + +def read_file(filename, label="target", use_dataframe=True, sep=None): if filename is None: - raise ValueError('filename is required') - - if filename.endswith('gz'): - compression = 'gzip' + raise ValueError("filename is required") + + if filename.endswith("gz"): + compression = "gzip" else: compression = None - - print('compression:',compression) - print('filename:',filename) + + print("compression:", compression) + print("filename:", filename) if sep is None: - if filename.endswith(('.tsv', '.tsv.gz')): - sep = '\t' - elif filename.endswith(('.csv', '.csv.gz')): - sep = ',' + if filename.endswith((".tsv", ".tsv.gz")): + sep = "\t" + elif filename.endswith((".csv", ".csv.gz")): + sep = "," else: - sep = ',' + sep = "," input_data = pd.read_csv(filename, sep=sep, compression=compression) - + # clean up column names - clean_names = {k:k.strip().replace('.','_') for k in input_data.columns} + clean_names = {k: k.strip().replace(".", "_") for k in input_data.columns} input_data = input_data.rename(columns=clean_names) feature_names = [x for x in input_data.columns.values if x != label] @@ -35,8 +36,6 @@ def read_file(filename, label='target', use_dataframe=True, sep=None): X = X.values y = input_data[label].values - assert(X.shape[1] == feature_names.shape[0]) + assert X.shape[1] == feature_names.shape[0] return X, y, feature_names - - diff --git a/experiment/seeds.py b/experiment/seeds.py index 385ad33a8..915b0cce0 100755 --- a/experiment/seeds.py +++ b/experiment/seeds.py @@ -1,13 +1,103 @@ # fixed seeds for the experiment SEEDS = [ - 23654, 15795, 860, 5390, 16850, 29910, 4426, 21962, 14423, 28020, - 29802, 21575, 11964, 11284, 22118, 6265, 11363, 27495, 16023, 8322, - 1685, 32052, 769, 26967, 30187, 32157, 23333, 2433, 5311, 5051, - 6420, 17568, 20939, 19769, 28693, 6396, 29419, 27480, 32304, 8666, - 25658, 18942, 24233, 18431, 32219, 2747, 25551, 26382, 189, 31677, - 19118, 3005, 21042, 1899, 24118, 1267, 31551, 17912, 11394, 3556, - 3890, 8838, 30740, 27464, 14502, 21777, 10627, 8792, 10555, 10253, - 8433, 10233, 11016, 23897, 2612, 23425, 25939, 22619, 21870, 23483, - 26054, 15787, 27132, 17159, 12206, 8226, 14541, 3152, 26531, 1585, - 3943, 23939, 19457, 1021, 11653, 10805, 13417, 20227, 7989, 9692 + 23654, + 15795, + 860, + 5390, + 16850, + 29910, + 4426, + 21962, + 14423, + 28020, + 29802, + 21575, + 11964, + 11284, + 22118, + 6265, + 11363, + 27495, + 16023, + 8322, + 1685, + 32052, + 769, + 26967, + 30187, + 32157, + 23333, + 2433, + 5311, + 5051, + 6420, + 17568, + 20939, + 19769, + 28693, + 6396, + 29419, + 27480, + 32304, + 8666, + 25658, + 18942, + 24233, + 18431, + 32219, + 2747, + 25551, + 26382, + 189, + 31677, + 19118, + 3005, + 21042, + 1899, + 24118, + 1267, + 31551, + 17912, + 11394, + 3556, + 3890, + 8838, + 30740, + 27464, + 14502, + 21777, + 10627, + 8792, + 10555, + 10253, + 8433, + 10233, + 11016, + 23897, + 2612, + 23425, + 25939, + 22619, + 21870, + 23483, + 26054, + 15787, + 27132, + 17159, + 12206, + 8226, + 14541, + 3152, + 26531, + 1585, + 3943, + 23939, + 19457, + 1021, + 11653, + 10805, + 13417, + 20227, + 7989, + 9692, ] diff --git a/experiment/submit_jobs.py b/experiment/submit_jobs.py index c923722d2..f92a5bcd6 100755 --- a/experiment/submit_jobs.py +++ b/experiment/submit_jobs.py @@ -1,42 +1,42 @@ -from glob import glob -import os -import sys import argparse +import os +from glob import glob + from pmlb import regression_dataset_names -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Submit long jobs.", - add_help=False) - parser.add_argument('DATA_PATH',type=str) - parser.add_argument('-ml',action='store',dest='mls', type=str, - default='') - parser.add_argument('-q',action='store',dest='Q', type=str, - default='moore_normal') - parser.add_argument('-n_trials',action='store',dest='TRIALS', default=1) - parser.add_argument('-results',action='store',dest='RDIR', - default='results',type=str,help='Results directory') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Submit long jobs.", add_help=False) + parser.add_argument("DATA_PATH", type=str) + parser.add_argument("-ml", action="store", dest="mls", type=str, default="") + parser.add_argument( + "-q", action="store", dest="Q", type=str, default="moore_normal" + ) + parser.add_argument("-n_trials", action="store", dest="TRIALS", default=1) + parser.add_argument( + "-results", + action="store", + dest="RDIR", + default="results", + type=str, + help="Results directory", + ) args = parser.parse_args() -datapath = args.DATA_PATH +datapath = args.DATA_PATH -if args.mls == '': - ml_files = glob('./methods/*.py') - mls = ','.join([m[10:-3] for m in ml_files if '__' not in m]) - print('No MLs passed. Running everything in methods/*.py:', - mls) +if args.mls == "": + ml_files = glob("./methods/*.py") + mls = ",".join([m[10:-3] for m in ml_files if "__" not in m]) + print("No MLs passed. Running everything in methods/*.py:", mls) else: mls = args.mls -lpc_options = '-q {Q} -m 12000 -n_jobs 1'.format(Q=args.Q) +lpc_options = "-q {Q} -m 12000 -n_jobs 1".format(Q=args.Q) for dataset in regression_dataset_names: - f = args.DATA_PATH + '/' + dataset + '/' + dataset + '.tsv.gz' - jobline = ('python analyze.py {DATA} ' - '-ml {ML} ' - '-results {RDIR} -n_trials {NT} {LPC}').format(DATA=f, - LPC=lpc_options, - ML=mls, - RDIR=args.RDIR, - NT=args.TRIALS) + f = args.DATA_PATH + "/" + dataset + "/" + dataset + ".tsv.gz" + jobline = ( + "python analyze.py {DATA} -ml {ML} -results {RDIR} -n_trials {NT} {LPC}" + ).format(DATA=f, LPC=lpc_options, ML=mls, RDIR=args.RDIR, NT=args.TRIALS) print(jobline) os.system(jobline) diff --git a/experiment/symbolic_utils.py b/experiment/symbolic_utils.py index daec4aed3..3d53670b6 100644 --- a/experiment/symbolic_utils.py +++ b/experiment/symbolic_utils.py @@ -1,19 +1,22 @@ +import ast import pdb +import re + import pandas as pd -from yaml import load, Loader import sympy +from read_file import read_file + # from sympy import * -from sympy import Symbol, simplify, factor, Float, preorder_traversal, Integer +from sympy import Float, Integer, Symbol, preorder_traversal, simplify from sympy.parsing.sympy_parser import parse_expr -from read_file import read_file -import re -import ast +from yaml import Loader, load ############################################################################### # fn definitions from the algorithms, written in sympy operators -def sub(x,y): - return sympy.Add(x,-y) +def sub(x, y): + return sympy.Add(x, -y) + # def division(x,y): # print('division') @@ -26,30 +29,37 @@ def sub(x,y): # # return result # return sympy.Mul(x,1/y) -#TODO: handle protected division -def div(x,y): - return sympy.Mul(x,1/y) + +# TODO: handle protected division +def div(x, y): + return sympy.Mul(x, 1 / y) + def square(x): - return sympy.Pow(x,2) + return sympy.Pow(x, 2) + def cube(x): - return sympy.Pow(x,3) + return sympy.Pow(x, 3) + def quart(x): - return sympy.Pow(x,4) + return sympy.Pow(x, 4) + def PLOG(x, base=None): if isinstance(x, sympy.Float): if x < 0: x = sympy.Abs(x) if base: - return sympy.log(x,base) + return sympy.log(x, base) else: return sympy.log(x) + def PLOG10(x): - return PLOG(x,10) + return PLOG(x, 10) + def PSQRT(x): if isinstance(x, sympy.Float): @@ -57,209 +67,248 @@ def PSQRT(x): return sympy.sqrt(sympy.Abs(x)) return sympy.sqrt(x) + ############################################################################### + def complexity(expr): - c=0 + c = 0 for arg in preorder_traversal(expr): c += 1 return c - + + def round_floats(ex1): ex2 = ex1 for a in preorder_traversal(ex1): if isinstance(a, Float): if abs(a) < 0.0001: - ex2 = ex2.subs(a,Integer(0)) + ex2 = ex2.subs(a, Integer(0)) else: - ex2 = ex2.subs(a, Float(round(a, 3),3)) + ex2 = ex2.subs(a, Float(round(a, 3), 3)) return ex2 + ################################################################################ # currently the MRGP model is put together incorrectly. this set of functions # corrects the MRGP model form so that it can be fed to sympy and simplified. ################################################################################ def add_commas(model): - return ''.join([m + ',' if not m.endswith('(') else m - for m in model.split()])[:-1] + return "".join([m + "," if not m.endswith("(") else m for m in model.split()])[:-1] + def decompose_mrgp_model(model_str): """split mrgp model into its betas and model parts""" - new_model=[] # get betas - betas = [float(b[0]) for b in re.findall( - r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*', - model_str)] - print('betas:',betas) + betas = [ + float(b[0]) + for b in re.findall(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*", model_str) + ] + print("betas:", betas) # get form - submodel = re.sub(pattern=r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*', - repl=r'', - string=model_str) - return betas, submodel #new_model + submodel = re.sub( + pattern=r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\*", repl=r"", string=model_str + ) + return betas, submodel # new_model + def print_model(node): - if hasattr(node, 'func'): - model_str = node.func.id + '(' - elif hasattr(node, 'id'): - # model_str = node.id + if hasattr(node, "func"): + model_str = node.func.id + "(" + elif hasattr(node, "id"): + # model_str = node.id return node.id else: pdb.set_trace() - if hasattr(node, 'args'): + if hasattr(node, "args"): i = 0 for arg in node.args: model_str += print_model(arg) i += 1 if i < len(node.args): - model_str += ',' - model_str += ')' + model_str += "," + model_str += ")" # print('print_model::',model_str) return model_str + def add_betas(node, betas): beta = betas[0] betas.pop(0) if float(beta) > 0: - model_str = str(beta) + '*' + print_model(node) + model_str = str(beta) + "*" + print_model(node) i = 1 else: # print('filtering fn w beta=',beta) - model_str = '' + model_str = "" i = 0 - if hasattr(node, 'args'): + if hasattr(node, "args"): for arg in node.args: submodel = add_betas(arg, betas) - if submodel != '': - model_str += '+' if i != 0 else '' - model_str += submodel + if submodel != "": + model_str += "+" if i != 0 else "" + model_str += submodel i += 1 # print('add_betas::',model_str) return model_str + + ################################################################################ + def clean_pred_model(model_str, dataset, est_name): - mrgp = 'MRGP' in est_name - - model_str = model_str.strip() + mrgp = "MRGP" in est_name + + model_str = model_str.strip() if mrgp: - model_str = model_str.replace('+','add') + model_str = model_str.replace("+", "add") model_str = add_commas(model_str) betas, model_str = decompose_mrgp_model(model_str) - X, labels, features = read_file(dataset) - - local_dict = {k:Symbol(k) for k in features} + + local_dict = {k: Symbol(k) for k in features} new_model_str = model_str - # rename features - for i,f in reversed(list(enumerate(features))): - print('replacing feature',i,'with',f) - if any([n in est_name.lower() for n in ['mrgp','operon','dsr']]): - i = i + 1 - new_model_str = new_model_str.replace('x'+str(i),f) - new_model_str = new_model_str.replace('x_'+str(i),f) - new_model_str = new_model_str.replace('X_'+str(i),f) - new_model_str = new_model_str.replace('X'+str(i),f) - new_model_str = new_model_str.replace('x[:,{}]'.format(i),f) - new_model_str = new_model_str.replace('x[{}]'.format(i),f) + # Rename generic feature placeholders. Model strings are expected to use + # dataset feature names, and some datasets genuinely have names such as + # ``x1`` and ``x2``. In that case, treating the names as placeholders would + # corrupt an already-correct model. + placeholder_pattern = r"\b(?:x_|X_|x|X)(\d+)\b" + placeholder_tokens = { + match.group(0) for match in re.finditer(placeholder_pattern, new_model_str) + } + feature_names = {str(feature) for feature in features} + + index_offset = int( # Some methods start counting features at 1 + any(name in est_name.lower() for name in ["mrgp", "operon", "dsr"]) + ) + + def feature_name(index): + feature_index = index - index_offset + if 0 <= feature_index < len(features): + return str(features[feature_index]) + return None + + # If every placeholder-looking identifier is an actual feature name, the + # string is already in the documented dataset-name form. This intentionally + # favors preserving a compliant model in the otherwise ambiguous x1/x2 case. + if not placeholder_tokens.issubset(feature_names): + + def replace_placeholder(match): + replacement = feature_name(int(match.group(1))) + return replacement if replacement is not None else match.group(0) + + new_model_str = re.sub(placeholder_pattern, replace_placeholder, new_model_str) + + # Array-style placeholders cannot be dataset feature names, so they can + # always be normalized. Keep this separate from identifier replacement. + def replace_array_placeholder(match): + replacement = feature_name(int(match.group(1))) + return replacement if replacement is not None else match.group(0) + + new_model_str = re.sub( + r"x\[:,\s*(\d+)\s*\]", replace_array_placeholder, new_model_str + ) + new_model_str = re.sub( + r"x\[\s*(\d+)\s*\]", replace_array_placeholder, new_model_str + ) # operators - new_model_str = new_model_str.replace('^','**') - #GP-GOMEA - new_model_str = new_model_str.replace('p/','/') - new_model_str = new_model_str.replace('plog','PLOG') - new_model_str = new_model_str.replace('aq','/') + new_model_str = new_model_str.replace("^", "**") + # GP-GOMEA + new_model_str = new_model_str.replace("p/", "/") + new_model_str = new_model_str.replace("plog", "PLOG") + new_model_str = new_model_str.replace("aq", "/") # MRGP - new_model_str = new_model_str.replace('mylog','PLOG') + new_model_str = new_model_str.replace("mylog", "PLOG") # ITEA - new_model_str = new_model_str.replace('sqrtAbs','PSQRT') + new_model_str = new_model_str.replace("sqrtAbs", "PSQRT") # new_model_str = re.sub(pattern=r'sqrtAbs\((.*?)\)', # repl=r'sqrt(abs(\1))', # string=new_model_str # ) - new_model_str = new_model_str.replace('np.','') + new_model_str = new_model_str.replace("np.", "") # ellyn & FEAT - new_model_str = new_model_str.replace('|','') - new_model_str = new_model_str.replace('log','PLOG') - new_model_str = new_model_str.replace('sqrt','PSQRT') + new_model_str = new_model_str.replace("|", "") + new_model_str = new_model_str.replace("log", "PLOG") + new_model_str = new_model_str.replace("sqrt", "PSQRT") # AIFeynman - new_model_str = new_model_str.replace('pi','3.1415926535') - - local_dict.update({ - 'add':sympy.Add, - 'mul':sympy.Mul, - 'max':sympy.Max, - 'min':sympy.Min, - 'sub':sub, - 'div':div, - 'square':square, - 'cube':cube, - 'quart':quart, - 'PLOG':PLOG, - 'PLOG10':PLOG, - 'PSQRT':PSQRT - }) + new_model_str = new_model_str.replace("pi", "3.1415926535") + + local_dict.update( + { + "add": sympy.Add, + "mul": sympy.Mul, + "max": sympy.Max, + "min": sympy.Min, + "sub": sub, + "div": div, + "square": square, + "cube": cube, + "quart": quart, + "PLOG": PLOG, + "PLOG10": PLOG, + "PSQRT": PSQRT, + } + ) # BSR # get rid of square brackets - new_model_str = new_model_str.replace('[','').replace(']','') + new_model_str = new_model_str.replace("[", "").replace("]", "") - print('parsing',new_model_str) + print("parsing", new_model_str) if mrgp: - mrgp_ast = ast.parse(new_model_str, "","eval") - new_model_str = add_betas(mrgp_ast.body,betas) - assert(len(betas)==0) + mrgp_ast = ast.parse(new_model_str, "", "eval") + new_model_str = add_betas(mrgp_ast.body, betas) + assert len(betas) == 0 print(local_dict) - model_sym = parse_expr(new_model_str, local_dict = local_dict) - print('round_floats') + model_sym = parse_expr(new_model_str, local_dict=local_dict) + print("round_floats") model_sym = round_floats(model_sym) - print('rounded:',model_sym) - print('simplify...') + print("rounded:", model_sym) + print("simplify...") model_sym = simplify(model_sym, ratio=1) - print('simplified:',model_sym) + print("simplified:", model_sym) return model_sym def get_sym_model(dataset, return_str=True): """return sympy model from dataset metadata""" metadata = load( - open('/'.join(dataset.split('/')[:-1])+'/metadata.yaml','r'), - Loader=Loader + open("/".join(dataset.split("/")[:-1]) + "/metadata.yaml", "r"), Loader=Loader ) - df = pd.read_csv(dataset,sep='\t') - features = [c for c in df.columns if c != 'target'] -# print('features:',df.columns) - description = metadata['description'].split('\n') - model_str = [ms for ms in description if '=' in ms][0].split('=')[-1] - model_str = model_str.replace('pi','3.1415926535') + df = pd.read_csv(dataset, sep="\t") + features = [c for c in df.columns if c != "target"] + # print('features:',df.columns) + description = metadata["description"].split("\n") + model_str = [ms for ms in description if "=" in ms][0].split("=")[-1] + model_str = model_str.replace("pi", "3.1415926535") if return_str: return model_str # pdb.set_trace() # handle feynman problem constants -# print('model:',model_str) - model_sym = parse_expr(model_str, - local_dict = {k:Symbol(k) for k in features}) + # print('model:',model_str) + model_sym = parse_expr(model_str, local_dict={k: Symbol(k) for k in features}) model_sym = round_floats(model_sym) -# print('sym model:',model_sym) + # print('sym model:',model_sym) return model_sym + def get_sympy_model(model_str, dataset): - model_str = model_str.replace('pi', '3.1415926535') + model_str = model_str.replace("pi", "3.1415926535") - df = pd.read_csv(dataset, sep='\t') - features = [c for c in df.columns if c != 'target'] + df = pd.read_csv(dataset, sep="\t") + features = [c for c in df.columns if c != "target"] - model_sym = parse_expr( - model_str, - local_dict={k: Symbol(k) for k in features} - ) + model_sym = parse_expr(model_str, local_dict={k: Symbol(k) for k in features}) model_sym = round_floats(model_sym) return model_sym + def rewrite_AIFeynman_model_size(model_str): """AIFeynman complexity was incorrect prior to version , update it here""" return complexity(parse_expr(model_str)) diff --git a/experiment/test_algorithm.py b/experiment/test_algorithm.py index 089674a29..ac7c5c74c 100644 --- a/experiment/test_algorithm.py +++ b/experiment/test_algorithm.py @@ -1,92 +1,146 @@ -import pytest -import sys -import os +import importlib import json +import os +import sys import types -from os.path import dirname as d from os.path import abspath -root_dir = d(abspath(__file__)) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') +from os.path import dirname as d + +import numpy as np +import pandas as pd +import pytest +sys.path.append(d(abspath(__file__))) + +import symbolic_utils from evaluate_model import evaluate_model -import importlib +from read_file import read_file +from symbolic_utils import ( + complexity, + round_floats, +) +from sympy import Symbol, simplify + # symbolic model stuff from sympy.parsing.sympy_parser import parse_expr -from symbolic_utils import (complexity, round_floats, - sub, div, square, cube, quart, - PLOG, PLOG10, PSQRT) -from read_file import read_file -from sympy import Symbol -if 'OMP_NUM_THREADS' not in os.environ.keys(): - os.environ['OMP_NUM_THREADS'] = '1' -if 'OPENBLAS_NUM_THREADS' not in os.environ.keys(): - os.environ['OPENBLAS_NUM_THREADS'] = '1' -if 'MKL_NUM_THREADS' not in os.environ.keys(): - os.environ['MKL_NUM_THREADS'] = '1' +if "OMP_NUM_THREADS" not in os.environ.keys(): + os.environ["OMP_NUM_THREADS"] = "1" +if "OPENBLAS_NUM_THREADS" not in os.environ.keys(): + os.environ["OPENBLAS_NUM_THREADS"] = "1" +if "MKL_NUM_THREADS" not in os.environ.keys(): + os.environ["MKL_NUM_THREADS"] = "1" # shared variables -dataset = 'test/192_vineyard_small.tsv.gz' -results_path = 'tmp_results' +dataset = "test/192_vineyard_small.tsv.gz" +results_path = "tmp_results" random_state = 42 + +def _use_toy_dataframe(monkeypatch, feature_names): + """Make clean_pred_model see a small dataset without a fixture on disk.""" + dataframe = pd.DataFrame(np.zeros((2, len(feature_names))), columns=feature_names) + dataframe["target"] = 0.0 + + def read_toy_file(_dataset): + return ( + dataframe.drop(columns="target"), + dataframe["target"].to_numpy(), + dataframe.drop(columns="target").columns.to_numpy(), + ) + + monkeypatch.setattr(symbolic_utils, "read_file", read_toy_file) + + +def test_clean_pred_model_preserves_x_named_features(monkeypatch, ml): + feature_names = ["x1", "x2", "x3", "y1", "y2", "y3"] + model = "x1*y1 + x2*y2 + x3*y3" + _use_toy_dataframe(monkeypatch, feature_names) + + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) + + assert simplify(cleaned - parse_expr(model)) == 0 + + +def test_clean_pred_model_preserves_mixed_x_named_features(monkeypatch, ml): + feature_names = ["m1", "m2", "G", "x1", "x2", "y1", "y2", "z1", "z2"] + model = "x1**2 - 2*x1*x2 + x2**2" + _use_toy_dataframe(monkeypatch, feature_names) + + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) + + assert simplify(cleaned - parse_expr(model)) == 0 + + +def test_clean_pred_model_still_normalizes_generic_placeholders(monkeypatch, ml): + _use_toy_dataframe(monkeypatch, ["mass", "velocity"]) + + is_one_based = any(name in ml.lower() for name in ["mrgp", "operon", "dsr"]) + model = "x1 + x2" if is_one_based else "x0 + x1" + cleaned = symbolic_utils.clean_pred_model(model, "toy.tsv", ml) + + assert simplify(cleaned - parse_expr("mass + velocity")) == 0 + + def get_algorithm(ml): - algorithm = importlib.__import__('methods.'+ml+'.regressor',globals(), - locals(), - ['*']) - print('algorithm imported:',algorithm) + algorithm = importlib.__import__( + "methods." + ml + ".regressor", globals(), locals(), ["*"] + ) + print("algorithm imported:", algorithm) return algorithm + def test_import(ml): """Check algorithm imports""" algorithm = get_algorithm(ml) - assert 'est' in dir(algorithm) - assert 'model' in dir(algorithm) + assert "est" in dir(algorithm) + assert "model" in dir(algorithm) eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs eval_kwarg_types = { - 'test_params':dict, - 'max_train_samples':int, - 'scale_x':bool, - 'scale_y':bool, - 'pre_train':types.FunctionType, - 'use_dataframe':bool + "test_params": dict, + "max_train_samples": int, + "scale_x": bool, + "scale_y": bool, + "pre_train": types.FunctionType, + "use_dataframe": bool, } - for k,v in eval_kwargs.items(): - assert k in ['test_params', - 'max_train_samples', - 'scale_x', - 'scale_y', - 'pre_train', - 'use_dataframe' - ] + for k, v in eval_kwargs.items(): + assert k in [ + "test_params", + "max_train_samples", + "scale_x", + "scale_y", + "pre_train", + "use_dataframe", + ] assert isinstance(v, eval_kwarg_types[k]) + # TODO: need to update evaluate model to handle this format def test_evaluate(ml): """Dataset evaluation""" algorithm = get_algorithm(ml) - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs else: eval_kwargs = {} - print(eval_kwargs) + print(eval_kwargs) json_file = evaluate_model( - dataset=dataset, # input file - results_path=results_path, # results directory - random_state=random_state, # random state seed - est_name=ml, # estimator name - est=algorithm.est, # estimator class + dataset=dataset, # input file + results_path=results_path, # results directory + random_state=random_state, # random state seed + est_name=ml, # estimator name + est=algorithm.est, # estimator class algorithm=algorithm, - model=algorithm.model, # method to extract the model from the estimator - test=True, # testing - **eval_kwargs + model=algorithm.model, # method to extract the model from the estimator + test=True, # testing + **eval_kwargs, ) print(json_file) print("hello") @@ -98,35 +152,36 @@ def test_sympy(ml): algorithm = get_algorithm(ml) - dataset_name = dataset.split('/')[-1][:-7] - json_file = (results_path + '/' + dataset_name + '_' + ml + '_' - + str(random_state) + '.json') - + dataset_name = dataset.split("/")[-1][:-7] + json_file = ( + results_path + "/" + dataset_name + "_" + ml + "_" + str(random_state) + ".json" + ) + if os.path.exists(json_file): - r = json.load(open(json_file, 'r')) + r = json.load(open(json_file, "r")) else: - raise FileNotFoundError(json_file+' not found') + raise FileNotFoundError(json_file + " not found") - est_name = r['algorithm'] + r["algorithm"] - raw_model = r['symbolic_model'] - print('raw_model:',raw_model) + raw_model = r["symbolic_model"] + print("raw_model:", raw_model) X, labels, features = read_file(dataset) - local_dict = {k:Symbol(k) for k in features} + local_dict = {k: Symbol(k) for k in features} - model_sym = parse_expr(raw_model, local_dict = local_dict) + model_sym = parse_expr(raw_model, local_dict=local_dict) model_sym = round_floats(model_sym) - print('sym model:',model_sym) + print("sym model:", model_sym) # Should pass - if hasattr(algorithm, 'complexity'): + if hasattr(algorithm, "complexity"): try: model_complexity = algorithm.complexity(model_sym) - print('model complexity (author provided):', model_complexity) + print("model complexity (author provided):", model_complexity) except Exception as exc: - print('algorithm complexity failed; using fallback:', exc) + print("algorithm complexity failed; using fallback:", exc) model_complexity = complexity(model_sym) - print('model complexity (using our implementation):', model_complexity) - else: # Given that the string is sympy compatible, this should work + print("model complexity (using our implementation):", model_complexity) + else: # Given that the string is sympy compatible, this should work model_complexity = complexity(model_sym) - print('model complexity (using our implementation):', model_complexity) + print("model complexity (using our implementation):", model_complexity) diff --git a/experiment/test_evaluate_model.py b/experiment/test_evaluate_model.py index d4fd00c17..1d8bcdd53 100755 --- a/experiment/test_evaluate_model.py +++ b/experiment/test_evaluate_model.py @@ -1,36 +1,40 @@ +import importlib import sys -from os.path import dirname as d from os.path import abspath -root_dir = d(d(abspath(__file__))) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') +from os.path import dirname as d + +sys.path.append(d(d(abspath(__file__)))) + from evaluate_model import evaluate_model -import importlib + def test_evaluate_model(ml): - print('running test_evaluate_model with ml=',ml) - dataset = 'test/192_vineyard_small.tsv.gz' - results_path = 'tmp_results' + print("running test_evaluate_model with ml=", ml) + dataset = "test/192_vineyard_small.tsv.gz" + results_path = "tmp_results" random_state = 42 - algorithm = importlib.__import__(f'methods.{ml}.regressor',globals(), - locals(), - ['est','hyper_params','complexity']) + algorithm = importlib.__import__( + f"methods.{ml}.regressor", + globals(), + locals(), + ["est", "hyper_params", "complexity"], + ) - print('algorithm imported:',algorithm) + print("algorithm imported:", algorithm) eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs evaluate_model( - dataset=dataset, - results_path=results_path, - random_state=random_state, + dataset=dataset, + results_path=results_path, + random_state=random_state, est_name=ml, est=algorithm.est, model=algorithm.model, algorithm=algorithm, - test=True, # testing - **eval_kwargs + test=True, # testing + **eval_kwargs, ) diff --git a/experiment/test_tuned_models.py b/experiment/test_tuned_models.py index 96b652a26..3f2515ee9 100644 --- a/experiment/test_tuned_models.py +++ b/experiment/test_tuned_models.py @@ -1,54 +1,50 @@ +import importlib import sys +from os.path import abspath from os.path import dirname as d -from os.path import abspath, join -root_dir = d(d(abspath(__file__))) -sys.path.append(root_dir) -print('appended',root_dir,'to sys.path') -import pytest -from glob import glob +sys.path.append(d(d(abspath(__file__)))) + + from evaluate_model import evaluate_model -import importlib + def test_tuned_models(ml): - print('running test_evaluate_model with ml=',ml) - dataset = 'test/strogatz_shearflow1.tsv.gz' - results_path = 'tmp_results' + print("running test_evaluate_model with ml=", ml) + dataset = "test/strogatz_shearflow1.tsv.gz" + results_path = "tmp_results" random_state = 42 - algorithm = importlib.__import__('methods.'+ml, - globals(), - locals(), - ['*'] - ) + algorithm = importlib.__import__("methods." + ml, globals(), locals(), ["*"]) - print('algorithm:',algorithm.est) - if 'hyper_params' not in dir(algorithm): - algorithm.hyper_params = {} - print('hyperparams:',algorithm.hyper_params) + print("algorithm:", algorithm.est) + if "hyper_params" not in dir(algorithm): + setattr(algorithm, "hyper_params", {}) + print("hyperparams:", algorithm.hyper_params) # optional keyword arguments passed to evaluate eval_kwargs = {} - if 'eval_kwargs' in dir(algorithm): + if "eval_kwargs" in dir(algorithm): eval_kwargs = algorithm.eval_kwargs # check for conflicts btw cmd line args and eval_kwargs # if args.SYM_DATA: - eval_kwargs['scale_x'] = False - eval_kwargs['scale_y'] = False - eval_kwargs['skip_tuning'] = True - eval_kwargs['sym_data'] = True - - evaluate_model(dataset, - results_path, - random_state, - ml, - algorithm.est, - algorithm.hyper_params, - algorithm.complexity, - algorithm.model, - target_noise=0, - feature_noise=0, - test=True, # testing - **eval_kwargs - ) + eval_kwargs["scale_x"] = False + eval_kwargs["scale_y"] = False + eval_kwargs["skip_tuning"] = True + eval_kwargs["sym_data"] = True + + evaluate_model( + dataset, + results_path, + random_state, + ml, + algorithm.est, + algorithm.hyper_params, + algorithm.complexity, + algorithm.model, + target_noise=0, + feature_noise=0, + test=True, # testing + **eval_kwargs, + ) diff --git a/experiment/utils.py b/experiment/utils.py index 2b27ab694..2a6e57cef 100755 --- a/experiment/utils.py +++ b/experiment/utils.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd + def jsonify(d): """recursively formats dicts for json serialization""" if isinstance(d, list): @@ -13,18 +14,17 @@ def jsonify(d): d[k] = jsonify(d[k]) elif isinstance(d, np.ndarray): return d.tolist() - elif d.__class__.__name__.startswith('int'): + elif d.__class__.__name__.startswith("int"): return int(d) - elif d.__class__.__name__.startswith('float'): + elif d.__class__.__name__.startswith("float"): return float(d) elif isinstance(d, pd.DataFrame) or isinstance(d, pd.Series): return d.values.tolist() elif isinstance(d, bool): return d - elif d == None: + elif d is None: return None elif not isinstance(d, str): - print("WARNING: attempting to store ",d,"as a str for json") + print("WARNING: attempting to store ", d, "as a str for json") return str(d) return d - diff --git a/postprocessing/blackbox_results.ipynb b/postprocessing/blackbox_results.ipynb index 1bdf5ae31..b92573262 100644 --- a/postprocessing/blackbox_results.ipynb +++ b/postprocessing/blackbox_results.ipynb @@ -15,67 +15,66 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import json\n", + "import os\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from glob import glob\n", - "from tqdm import tqdm\n", + "import pandas as pd\n", "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib\n", - "matplotlib.rc('pdf', fonttype=42)\n", - "import os\n", - "sns.set(font_scale=1.25, style='whitegrid')\n", "\n", - "rdir = '../results/'\n", - "# figdir = 'figs/black-box' \n", - "figdir = '../../paper/figs/results_pmlb_r1/'\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", + "\n", + "sns.set(font_scale=1.25, style=\"whitegrid\")\n", + "\n", + "rdir = \"../results/\"\n", + "# figdir = 'figs/black-box'\n", + "figdir = \"../../paper/figs/results_pmlb_r1/\"\n", "# Set to path to PMLB datasets\n", - "datadir = '../../../pmlb/datasets/'\n", - "print('figdir:',figdir)\n", - "def save(name='tmp',h=None):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "datadir = \"../../../pmlb/datasets/\"\n", + "print(\"figdir:\", figdir)\n", + "\n", + "\n", + "def save(name=\"tmp\", h=None):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',figdir+'/'+name+'.pdf')\n", + " print(\"saving\", figdir + \"/\" + name + \".pdf\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.pdf', bbox_inches='tight')\n", - " \n", + " plt.savefig(figdir + \"/\" + name + \".pdf\", bbox_inches=\"tight\")\n", + "\n", + "\n", "symbolic_algs = [\n", - " 'AFP', \n", - " 'AFP_FE',\n", - " 'BSR',\n", - " 'DSR',\n", - " 'FFX',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'GP-GOMEA',\n", - " 'gplearn',\n", - " 'ITEA', \n", - " 'MRGP', \n", - " 'Operon',\n", - " 'SBP-GP',\n", - " 'AIFeynman'\n", - "]\n", - "nongp_algs = [\n", - " 'BSR',\n", - " 'DSR',\n", - " 'AIFeynman'\n", + " \"AFP\",\n", + " \"AFP_FE\",\n", + " \"BSR\",\n", + " \"DSR\",\n", + " \"FFX\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"GP-GOMEA\",\n", + " \"gplearn\",\n", + " \"ITEA\",\n", + " \"MRGP\",\n", + " \"Operon\",\n", + " \"SBP-GP\",\n", + " \"AIFeynman\",\n", "]\n", + "nongp_algs = [\"BSR\", \"DSR\", \"AIFeynman\"]\n", "gp_algs = [\n", - " 'AFP', \n", - " 'AFP_FE',\n", - " 'FFX',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'GP-GOMEA',\n", - " 'gplearn',\n", - " 'ITEA', \n", - " 'MRGP', \n", - " 'Operon',\n", - " 'SBP-GP',\n", + " \"AFP\",\n", + " \"AFP_FE\",\n", + " \"FFX\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"GP-GOMEA\",\n", + " \"gplearn\",\n", + " \"ITEA\",\n", + " \"MRGP\",\n", + " \"Operon\",\n", + " \"SBP-GP\",\n", "]" ] }, @@ -92,7 +91,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_results = pd.read_feather('../results/black-box_results.feather')" + "df_results = pd.read_feather(\"../results/black-box_results.feather\")" ] }, { @@ -108,33 +107,40 @@ "metadata": {}, "outputs": [], "source": [ - "df_results2 = df_results.merge(df_results.groupby('dataset')['algorithm'].nunique().reset_index(),\n", - " on='dataset',suffixes=('','_count'))\n", + "df_results2 = df_results.merge(\n", + " df_results.groupby(\"dataset\")[\"algorithm\"].nunique().reset_index(),\n", + " on=\"dataset\",\n", + " suffixes=(\"\", \"_count\"),\n", + ")\n", + "\n", "\n", - " \n", "# rankings per trial per dataset\n", - "for col in [c for c in df_results2.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_results2[col+'_rank_per_trial']=df_results2.groupby(['dataset','random_state'])[col].apply(lambda x: \n", - " round(x,3).rank(\n", - " ascending=ascending))\n", - " \n", - "df_sum = df_results2.groupby(['algorithm','dataset'],as_index=False).median()\n", - "df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt)\n", - "df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x))\n", + "for col in [c for c in df_results2.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_results2[col + \"_rank_per_trial\"] = df_results2.groupby(\n", + " [\"dataset\", \"random_state\"]\n", + " )[col].apply(lambda x: round(x, 3).rank(ascending=ascending))\n", + "\n", + "df_sum = df_results2.groupby([\"algorithm\", \"dataset\"], as_index=False).median()\n", + "df_sum[\"rmse_test\"] = df_sum[\"mse_test\"].apply(np.sqrt)\n", + "df_sum[\"log_mse_test\"] = df_sum[\"mse_test\"].apply(lambda x: np.log(1 + x))\n", "df_results = df_results2\n", "\n", "# rankings and normalized scores per dataset\n", - "for col in [c for c in df_sum.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_sum[col+'_rank']=df_sum.groupby(['dataset'])[col].apply(lambda x: \n", - " round(x,3).rank(ascending=ascending)\n", - " )\n", - " df_sum[col+'_norm'] = df_sum.groupby('dataset')[col].apply(lambda x: (x-x.min())/(x.max()-x.min()))\n", + "for col in [c for c in df_sum.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_sum[col + \"_rank\"] = df_sum.groupby([\"dataset\"])[col].apply(\n", + " lambda x: round(x, 3).rank(ascending=ascending)\n", + " )\n", + " df_sum[col + \"_norm\"] = df_sum.groupby(\"dataset\")[col].apply(\n", + " lambda x: (x - x.min()) / (x.max() - x.min())\n", + " )\n", "###################\n", "# save summary data\n", "###################\n", - "df_sum.to_csv(rdir+'results_black-box_summary.csv.gz',compression='gzip',index=False)\n", + "df_sum.to_csv(\n", + " rdir + \"results_black-box_summary.csv.gz\", compression=\"gzip\", index=False\n", + ")\n", "\n", "df_sum" ] @@ -152,39 +158,48 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "def compare(df_compare, x='r2_test',y='algorithm', row=None, col=None, scale=None, xlim=[],**kwargs):\n", + "def compare(\n", + " df_compare,\n", + " x=\"r2_test\",\n", + " y=\"algorithm\",\n", + " row=None,\n", + " col=None,\n", + " scale=None,\n", + " xlim=[],\n", + " **kwargs,\n", + "):\n", " plt.figure()\n", " order = df_compare.groupby(y)[x].median().sort_values(ascending=False).index\n", - " if scale=='log' and len(xlim)>0 and xlim[0] == 0:\n", - " df_compare.loc[:,x] += 1\n", + " if scale == \"log\" and len(xlim) > 0 and xlim[0] == 0:\n", + " df_compare.loc[:, x] += 1\n", " xlim[0] = 1\n", - " xnew = '1 + '+x\n", - " df_compare=df_compare.rename(columns={x:xnew})\n", + " xnew = \"1 + \" + x\n", + " df_compare = df_compare.rename(columns={x: xnew})\n", " x = xnew\n", - " \n", - " sns.catplot(data=df_compare, \n", - " kind='point',\n", - "# color='w',\n", - " y=y,\n", - " x=x,\n", - " order=order,\n", - " fliersize=0,\n", - "# notch=True,\n", - " row=row,\n", - " col=col,\n", - " palette='flare_r',\n", - " **kwargs\n", - " )\n", - " plt.ylabel('')\n", - " if len(xlim)>0:\n", - " plt.xlim(xlim[0],xlim[1])\n", + "\n", + " sns.catplot(\n", + " data=df_compare,\n", + " kind=\"point\",\n", + " # color='w',\n", + " y=y,\n", + " x=x,\n", + " order=order,\n", + " fliersize=0,\n", + " # notch=True,\n", + " row=row,\n", + " col=col,\n", + " palette=\"flare_r\",\n", + " **kwargs,\n", + " )\n", + " plt.ylabel(\"\")\n", + " if len(xlim) > 0:\n", + " plt.xlim(xlim[0], xlim[1])\n", " if scale:\n", " plt.gca().set_xscale(scale)\n", - " \n", - " save(name='_'.join(['cat-boxplot',x + '-by-'+ y]) )\n", + "\n", + " save(name=\"_\".join([\"cat-boxplot\", x + \"-by-\" + y]))\n", " if col:\n", - " save(name='_'.join(['cat-boxplot',x + '-by-'+ y] + [col]) )" + " save(name=\"_\".join([\"cat-boxplot\", x + \"-by-\" + y] + [col]))" ] }, { @@ -202,65 +217,75 @@ "source": [ "# Make the PairGrid\n", "sns.set_style(\"whitegrid\")\n", - "df_plot=df_sum.copy()\n", + "df_plot = df_sum.copy()\n", "\n", - "df_plot['*algorithm*'] = df_plot['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x)\n", + "df_plot[\"*algorithm*\"] = df_plot[\"algorithm\"].apply(\n", + " lambda x: \"*\" + x if x in symbolic_algs else x\n", + ")\n", "# different options\n", - "x_vars=[\n", - "# 'rmse_test',\n", - "# 'log_mse_test',\n", - "# 'r2_test_norm',\n", - " 'r2_test',\n", - "# 'r2_test_rank',\n", - " 'model_size',\n", - "# 'model_size_rank',\n", - " 'training time (s)',\n", + "x_vars = [\n", + " # 'rmse_test',\n", + " # 'log_mse_test',\n", + " # 'r2_test_norm',\n", + " \"r2_test\",\n", + " # 'r2_test_rank',\n", + " \"model_size\",\n", + " # 'model_size_rank',\n", + " \"training time (s)\",\n", "]\n", - "order = df_plot.groupby('*algorithm*')[x_vars[0]].median().sort_values(\n", - " ascending='r2' not in x_vars[0] or 'rank' in x_vars[0]).index\n", - "g = sns.PairGrid(df_plot, \n", - " x_vars=x_vars,\n", - " y_vars=['*algorithm*'],\n", - " height=6.5, \n", - " aspect=0.6,\n", - "# hue='symbolic_dataset'\n", - " )\n", - "\n", - "# Draw a dot plot \n", - "g.map(sns.pointplot, \n", - " size=10,\n", - " orient=\"h\",\n", - " jitter=False,\n", - " order=order,\n", - " palette=\"flare_r\",\n", - " linewidth=1,\n", - " markeredgecolor='w',\n", - " join=False,\n", - " estimator=np.median,\n", - " n_boot=1000,\n", - " ci=95\n", - " )\n", + "order = (\n", + " df_plot.groupby(\"*algorithm*\")[x_vars[0]]\n", + " .median()\n", + " .sort_values(ascending=\"r2\" not in x_vars[0] or \"rank\" in x_vars[0])\n", + " .index\n", + ")\n", + "g = sns.PairGrid(\n", + " df_plot,\n", + " x_vars=x_vars,\n", + " y_vars=[\"*algorithm*\"],\n", + " height=6.5,\n", + " aspect=0.6,\n", + " # hue='symbolic_dataset'\n", + ")\n", + "\n", + "# Draw a dot plot\n", + "g.map(\n", + " sns.pointplot,\n", + " size=10,\n", + " orient=\"h\",\n", + " jitter=False,\n", + " order=order,\n", + " palette=\"flare_r\",\n", + " linewidth=1,\n", + " markeredgecolor=\"w\",\n", + " join=False,\n", + " estimator=np.median,\n", + " n_boot=1000,\n", + " ci=95,\n", + ")\n", "# # Use semantically meaningful titles for the columns\n", - "titles = [x.replace('_',' ').title().replace('(S)','(s)').replace('R2','$R^2$') for x in x_vars]\n", - "g.axes.flat[0].set_ylabel('')\n", + "titles = [\n", + " x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\").replace(\"R2\", \"$R^2$\")\n", + " for x in x_vars\n", + "]\n", + "g.axes.flat[0].set_ylabel(\"\")\n", "for ax, title in zip(g.axes.flat, titles):\n", - "\n", " # Set a different title for each axes\n", " ax.set(title=title)\n", - " ax.set_xlabel('')\n", - " \n", - " if any([n in title.lower() for n in ['size','time']]):\n", - " ax.set_xscale('log')\n", - " \n", - " if title == '$R^2$ Test':\n", - " ax.set_xlim([-.25,1])\n", + " ax.set_xlabel(\"\")\n", + "\n", + " if any([n in title.lower() for n in [\"size\", \"time\"]]):\n", + " ax.set_xscale(\"log\")\n", + "\n", + " if title == \"$R^2$ Test\":\n", + " ax.set_xlim([-0.25, 1])\n", "\n", " # Make the grid horizontal instead of vertical\n", " ax.yaxis.grid(True)\n", "\n", "sns.despine(left=True, bottom=True)\n", "\n", - "save(name='_'.join(['pairgrid-pointplot']+x_vars))" + "save(name=\"_\".join([\"pairgrid-pointplot\"] + x_vars))" ] }, { @@ -278,76 +303,89 @@ "source": [ "# Make the PairGrid\n", "sns.set_style(\"whitegrid\")\n", - "df_plot=df_sum.copy()\n", - "df_plot['*algorithm*'] = df_plot['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x)\n", - "df_plot = df_plot[['*algorithm*','r2_test','r2_test_norm','r2_test_rank','friedman_dataset']]\n", - "\n", - "x='r2_test'\n", - "markers = ['o','s','x']\n", - "colors = sns.color_palette('flare',n_colors=3)\n", - "for x in ['r2_test_norm']:\n", - " h = plt.figure(figsize=(12,6))\n", + "df_plot = df_sum.copy()\n", + "df_plot[\"*algorithm*\"] = df_plot[\"algorithm\"].apply(\n", + " lambda x: \"*\" + x if x in symbolic_algs else x\n", + ")\n", + "df_plot = df_plot[\n", + " [\"*algorithm*\", \"r2_test\", \"r2_test_norm\", \"r2_test_rank\", \"friedman_dataset\"]\n", + "]\n", + "\n", + "x = \"r2_test\"\n", + "markers = [\"o\", \"s\", \"x\"]\n", + "colors = sns.color_palette(\"flare\", n_colors=3)\n", + "for x in [\"r2_test_norm\"]:\n", + " h = plt.figure(figsize=(12, 6))\n", " ax = plt.gca()\n", - " order = df_plot.groupby('*algorithm*')[x].median().sort_values(\n", - " ascending='r2' not in x or 'rank' in x).index\n", + " order = (\n", + " df_plot.groupby(\"*algorithm*\")[x]\n", + " .median()\n", + " .sort_values(ascending=\"r2\" not in x or \"rank\" in x)\n", + " .index\n", + " )\n", " i = 0\n", " axes = []\n", - " g = sns.pointplot(ax=ax,\n", - " data=df_plot,\n", - " x=x,\n", - " y='*algorithm*',\n", - " hue='friedman_dataset',\n", - " size=10,\n", - " orient=\"h\",\n", - " order=order,\n", - " markers=['s','x'],\n", - " palette=\"flare_r\",\n", - " join=False,\n", - " estimator=np.median,\n", - " legend_out=True\n", - " )\n", + " g = sns.pointplot(\n", + " ax=ax,\n", + " data=df_plot,\n", + " x=x,\n", + " y=\"*algorithm*\",\n", + " hue=\"friedman_dataset\",\n", + " size=10,\n", + " orient=\"h\",\n", + " order=order,\n", + " markers=[\"s\", \"x\"],\n", + " palette=\"flare_r\",\n", + " join=False,\n", + " estimator=np.median,\n", + " legend_out=True,\n", + " )\n", " # all PMLB\n", - " g2 = sns.pointplot(ax=ax,\n", - " data=df_plot,\n", - " x=x,\n", - " y='*algorithm*',\n", - "# hue='friedman_dataset',\n", - " size=10,\n", - " orient=\"h\",\n", - " order=order,\n", - " color='r',\n", - "# palette=\"flare_r\",\n", - " join=False,\n", - " estimator=np.median,\n", - " markers = ['o'],\n", - " legend_out=True\n", - " )\n", + " g2 = sns.pointplot(\n", + " ax=ax,\n", + " data=df_plot,\n", + " x=x,\n", + " y=\"*algorithm*\",\n", + " # hue='friedman_dataset',\n", + " size=10,\n", + " orient=\"h\",\n", + " order=order,\n", + " color=\"r\",\n", + " # palette=\"flare_r\",\n", + " join=False,\n", + " estimator=np.median,\n", + " markers=[\"o\"],\n", + " legend_out=True,\n", + " )\n", " # add all PMLB to legend\n", " handles, labels = ax.get_legend_handles_labels()\n", - " #Create custom artists\n", - " both = plt.Line2D((0,1),(0,0), color='r', marker='o', linestyle='-', label='All')\n", + " # Create custom artists\n", + " both = plt.Line2D((0, 1), (0, 0), color=\"r\", marker=\"o\", linestyle=\"-\", label=\"All\")\n", " handles.append(both)\n", - " plt.legend(title='Data Subset',\n", - " handles=handles, \n", - " labels=['No Friedman','Only Friedman','All'])\n", - " \n", - " if x=='r2_test':\n", - " plt.xlim([-.5,1.0])\n", - " if 'norm' in x:\n", - " x='normalized_R2_test'\n", + " plt.legend(\n", + " title=\"Data Subset\",\n", + " handles=handles,\n", + " labels=[\"No Friedman\", \"Only Friedman\", \"All\"],\n", + " )\n", + "\n", + " if x == \"r2_test\":\n", + " plt.xlim([-0.5, 1.0])\n", + " if \"norm\" in x:\n", + " x = \"normalized_R2_test\"\n", " sns.despine(left=True, bottom=True)\n", - "# plt.title(prefix.replace('_',' ').title())\n", + " # plt.title(prefix.replace('_',' ').title())\n", " axes.append(ax)\n", - " i+=1\n", + " i += 1\n", " plt.tight_layout()\n", " for ax in axes:\n", - " ax.set_ylabel('')\n", - " ax.set_xlabel(x.replace('_',' ').title().replace('(S)','(s)').replace('R2','$R^2$'))\n", + " ax.set_ylabel(\"\")\n", + " ax.set_xlabel(\n", + " x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\").replace(\"R2\", \"$R^2$\")\n", + " )\n", " ax.yaxis.grid(True)\n", " ax.xaxis.grid(True)\n", "\n", - "\n", - " save(name='friedman_comparison_'+'_'.join(['pairgrid-pointplot']+[x]))" + " save(name=\"friedman_comparison_\" + \"_\".join([\"pairgrid-pointplot\"] + [x]))" ] }, { @@ -364,14 +402,22 @@ "outputs": [], "source": [ "import pandas as pd\n", + "\n", + "\n", "# for dataset, dfg in df_sum.groupby('dataset'):\n", - "def get_size(dataset,i):\n", - " df = pd.read_csv(datadir+dataset+'/'+dataset+'.tsv.gz',sep='\\t') \n", + "def get_size(dataset, i):\n", + " df = pd.read_csv(datadir + dataset + \"/\" + dataset + \".tsv.gz\", sep=\"\\t\")\n", " return df.shape[i]\n", - "df_sum.loc[:,'dataset_nsamples'] = df_sum['dataset'].apply(lambda x: get_size(x,0))\n", - "df_sum.loc[:,'dataset_nsamples_train'] = df_sum['dataset'].apply(lambda x: min(get_size(x,0),10000))\n", - "df_sum.loc[:,'dataset_nfeatures'] = df_sum['dataset'].apply(lambda x: get_size(x,1))\n", - "df_sum.loc[:,'dataset_npoints'] = df_sum['dataset_nsamples']*df_sum['dataset_nfeatures']\n", + "\n", + "\n", + "df_sum.loc[:, \"dataset_nsamples\"] = df_sum[\"dataset\"].apply(lambda x: get_size(x, 0))\n", + "df_sum.loc[:, \"dataset_nsamples_train\"] = df_sum[\"dataset\"].apply(\n", + " lambda x: min(get_size(x, 0), 10000)\n", + ")\n", + "df_sum.loc[:, \"dataset_nfeatures\"] = df_sum[\"dataset\"].apply(lambda x: get_size(x, 1))\n", + "df_sum.loc[:, \"dataset_npoints\"] = (\n", + " df_sum[\"dataset_nsamples\"] * df_sum[\"dataset_nfeatures\"]\n", + ")\n", "df_sum" ] }, @@ -381,39 +427,61 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum['Method Family'] = df_sum['algorithm'].apply(lambda x: 'Operon' if x=='Operon' else 'XGB' if x=='XGB' else 'Other Non-GP SR' if x in nongp_algs else 'Other GP SR' if x in gp_algs else 'Other ML')\n", - "sns.set_style('whitegrid')\n", - "for col in ['dataset_nsamples_train','dataset_nsamples','dataset_nfeatures','dataset_npoints']:\n", - " plt.figure(figsize=(8,8))\n", + "df_sum[\"Method Family\"] = df_sum[\"algorithm\"].apply(\n", + " lambda x: (\n", + " \"Operon\"\n", + " if x == \"Operon\"\n", + " else \"XGB\"\n", + " if x == \"XGB\"\n", + " else \"Other Non-GP SR\"\n", + " if x in nongp_algs\n", + " else \"Other GP SR\"\n", + " if x in gp_algs\n", + " else \"Other ML\"\n", + " )\n", + ")\n", + "sns.set_style(\"whitegrid\")\n", + "for col in [\n", + " \"dataset_nsamples_train\",\n", + " \"dataset_nsamples\",\n", + " \"dataset_nfeatures\",\n", + " \"dataset_npoints\",\n", + "]:\n", + " plt.figure(figsize=(8, 8))\n", " ax = sns.lineplot(\n", - " data=df_sum.loc[df_sum.algorithm.isin(['Operon',\n", - " 'FEAT',\n", - " 'EPLEX',\n", - " 'SBP-GP',\n", - " 'XGB',\n", - " 'DSR',\n", - " ])],\n", - " x = col,\n", - " y = 'training time (hr)',\n", - "# hue='Method Family',\n", - "# style='Method Family',\n", - " hue='algorithm',\n", - " style='algorithm',\n", + " data=df_sum.loc[\n", + " df_sum.algorithm.isin(\n", + " [\n", + " \"Operon\",\n", + " \"FEAT\",\n", + " \"EPLEX\",\n", + " \"SBP-GP\",\n", + " \"XGB\",\n", + " \"DSR\",\n", + " ]\n", + " )\n", + " ],\n", + " x=col,\n", + " y=\"training time (hr)\",\n", + " # hue='Method Family',\n", + " # style='Method Family',\n", + " hue=\"algorithm\",\n", + " style=\"algorithm\",\n", " markers=True,\n", - " palette='cividis',\n", + " palette=\"cividis\",\n", " markersize=10,\n", " markerfacecolor=None,\n", - " err_style='bars',\n", - " alpha=0.85\n", - "# facecolors=None\n", - "# scatter_kws = {'facecolors':'none'}\n", + " err_style=\"bars\",\n", + " alpha=0.85,\n", + " # facecolors=None\n", + " # scatter_kws = {'facecolors':'none'}\n", " )\n", - " ax.set_yscale('log')\n", - " ax.set_xscale('log')\n", - "# plt.legend(loc=[1.01,0])\n", - " plt.xlabel(col.replace('_',' ').title())\n", - " plt.ylabel('Training Time (hr)')\n", - " save('training_time_'+col)" + " ax.set_yscale(\"log\")\n", + " ax.set_xscale(\"log\")\n", + " # plt.legend(loc=[1.01,0])\n", + " plt.xlabel(col.replace(\"_\", \" \").title())\n", + " plt.ylabel(\"Training Time (hr)\")\n", + " save(\"training_time_\" + col)" ] }, { @@ -439,16 +507,18 @@ "outputs": [], "source": [ "np.random.seed(42)\n", - "def bootstrap(val, n = 1000, fn=np.mean):\n", + "\n", + "\n", + "def bootstrap(val, n=1000, fn=np.mean):\n", " val_samples = []\n", " for i in range(n):\n", - " sample = np.random.randint(0,len(val)-1, size=len(val))\n", - " val_samples.append( fn(val[sample]) )\n", + " sample = np.random.randint(0, len(val) - 1, size=len(val))\n", + " val_samples.append(fn(val[sample]))\n", " m = np.mean(val_samples)\n", " sd = np.std(val_samples)\n", - " ci_upper = np.quantile(val_samples,0.95)\n", - " ci_lower = np.quantile(val_samples,0.05)\n", - " return m, sd, ci_upper,ci_lower" + " ci_upper = np.quantile(val_samples, 0.95)\n", + " ci_lower = np.quantile(val_samples, 0.05)\n", + " return m, sd, ci_upper, ci_lower" ] }, { @@ -457,132 +527,137 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", "from pareto_utils import front\n", - "sns.set(font_scale=1, style='whitegrid')\n", - "labelsize=18\n", - "plt.figure(figsize=(7,7))\n", - "data = df_sum.copy() #.loc[df_sum.algorithm.isin(symbolic_algs)]\n", - "data['*algorithm*'] = data['algorithm'].apply(lambda x: x+'*' if x in symbolic_algs else x)\n", - "xcol = 'r2_test_rank'\n", + "\n", + "sns.set(font_scale=1, style=\"whitegrid\")\n", + "labelsize = 18\n", + "plt.figure(figsize=(7, 7))\n", + "data = df_sum.copy() # .loc[df_sum.algorithm.isin(symbolic_algs)]\n", + "data[\"*algorithm*\"] = data[\"algorithm\"].apply(\n", + " lambda x: x + \"*\" if x in symbolic_algs else x\n", + ")\n", + "xcol = \"r2_test_rank\"\n", "# xcol = 'r2_test'\n", - "ycol = 'model_size_rank'\n", + "ycol = \"model_size_rank\"\n", "# ycol = 'model_size'\n", - "palette = 'viridis'\n", + "palette = \"viridis\"\n", "# outline pareto front\n", - "pareto_data = data.groupby('*algorithm*').median()\n", + "pareto_data = data.groupby(\"*algorithm*\").median()\n", "\n", - "objs = pareto_data[[xcol,ycol]].values\n", + "objs = pareto_data[[xcol, ycol]].values\n", "# reverse R2 (objs are minimized)\n", "# objs[:,0] = -objs[:,0]\n", "levels = 6\n", - "styles=['-','-.','--',':', ':', ':']\n", + "styles = [\"-\", \"-.\", \"--\", \":\", \":\", \":\"]\n", "PFs = []\n", "pareto_ranks = -np.ones(len(pareto_data))\n", "for el in range(levels):\n", - "# pdb.set_trace()\n", - " PF = front(objs[:,0],objs[:,1])\n", - " if len(PF)>0:\n", - " print('PF:',PF)\n", + " # pdb.set_trace()\n", + " PF = front(objs[:, 0], objs[:, 1])\n", + " if len(PF) > 0:\n", + " print(\"PF:\", PF)\n", " pareto_ranks[PF] = el\n", - " objs[PF,:] = np.inf\n", + " objs[PF, :] = np.inf\n", " PFs.append(PF)\n", "i = 0\n", - "pareto_data.loc[:,'pareto_rank'] = pareto_ranks\n", + "pareto_data.loc[:, \"pareto_rank\"] = pareto_ranks\n", "for pfset in PFs:\n", - " xset,yset = [],[]\n", + " xset, yset = [], []\n", "\n", " for pf in pfset:\n", " xset.append(pareto_data[xcol].values[pf])\n", - " yset.append( pareto_data[ycol].values[pf] )\n", - " linestyle=styles[i]\n", - " plt.plot(xset,yset, styles[i]+'k', alpha=0.5, zorder=1)\n", - "# plt.gca().set_zorder(10)\n", - " i+=1\n", + " yset.append(pareto_data[ycol].values[pf])\n", + " linestyle = styles[i]\n", + " plt.plot(xset, yset, styles[i] + \"k\", alpha=0.5, zorder=1)\n", + " # plt.gca().set_zorder(10)\n", + " i += 1\n", "\n", - "cmap = sns.color_palette(palette=palette, \n", - " n_colors=pareto_data.pareto_rank.nunique(), \n", - " desat=None, \n", - " as_cmap=False)\n", + "cmap = sns.color_palette(\n", + " palette=palette,\n", + " n_colors=pareto_data.pareto_rank.nunique(),\n", + " desat=None,\n", + " as_cmap=False,\n", + ")\n", "\n", "ax = sns.scatterplot(\n", - " ax = plt.gca(),\n", - "# ax = g.ax_joint,\n", - "# data = data.groupby('*algorithm*').median(),\n", - " data = pareto_data,\n", - " x = xcol,\n", - " y = ycol,\n", - "# style='*algorithm*',\n", - "# style='pareto_rank',\n", - " hue = 'pareto_rank',\n", + " ax=plt.gca(),\n", + " # ax = g.ax_joint,\n", + " # data = data.groupby('*algorithm*').median(),\n", + " data=pareto_data,\n", + " x=xcol,\n", + " y=ycol,\n", + " # style='*algorithm*',\n", + " # style='pareto_rank',\n", + " hue=\"pareto_rank\",\n", " s=250,\n", - "# palette=palette,\n", + " # palette=palette,\n", " # edgecolor='k'\n", " legend=False,\n", - " palette=cmap\n", + " palette=cmap,\n", ")\n", "ax.set_zorder(2)\n", - "xoff = .5\n", + "xoff = 0.5\n", "yoff = 0.3\n", "# xoff, yoff = 0, 0\n", - "for idx,row in pareto_data.iterrows():\n", - " x=row[xcol]-xoff\n", - " y=row[ycol]-yoff\n", - " ha='right'\n", - " \n", - " if idx in ['Linear','AFP_FE*','MLP','MRGP*']:\n", - " x=row[xcol]+xoff\n", - " ha='left'\n", - " elif idx == 'Operon*':\n", - " x = row[xcol]+1\n", + "for idx, row in pareto_data.iterrows():\n", + " x = row[xcol] - xoff\n", + " y = row[ycol] - yoff\n", + " ha = \"right\"\n", + "\n", + " if idx in [\"Linear\", \"AFP_FE*\", \"MLP\", \"MRGP*\"]:\n", + " x = row[xcol] + xoff\n", + " ha = \"left\"\n", + " elif idx == \"Operon*\":\n", + " x = row[xcol] + 1\n", " y -= yoff\n", - " elif idx in ['gplearn*','FEAT*']:\n", - "# x -= xoff\n", - "# x=row[xcol]+xoff\n", - " y = row[ycol]+1\n", - "# ha='left'\n", - " \n", - " plt.text(s=idx,\n", - " x = x, \n", - " y = y,\n", - " ha=ha,\n", - " va='top',\n", - " bbox=dict(facecolor='w', edgecolor='b', boxstyle='round', alpha=1)\n", - " )\n", - " \n", + " elif idx in [\"gplearn*\", \"FEAT*\"]:\n", + " # x -= xoff\n", + " # x=row[xcol]+xoff\n", + " y = row[ycol] + 1\n", + " # ha='left'\n", + "\n", + " plt.text(\n", + " s=idx,\n", + " x=x,\n", + " y=y,\n", + " ha=ha,\n", + " va=\"top\",\n", + " bbox=dict(facecolor=\"w\", edgecolor=\"b\", boxstyle=\"round\", alpha=1),\n", + " )\n", + "\n", "# confidence intervals\n", "i = 0\n", - "for alg, dg in data.groupby('*algorithm*'):\n", - " x = dg[xcol].median() \n", + "for alg, dg in data.groupby(\"*algorithm*\"):\n", + " x = dg[xcol].median()\n", " y = dg[ycol].median()\n", " _, sdx, ciux, cilx = bootstrap(dg[xcol].values, fn=np.median, n=1000)\n", " _, sdy, ciuy, cily = bootstrap(dg[ycol].values, fn=np.median, n=1000)\n", " plt.plot(\n", - " [cilx,ciux],\n", - " [y, y],\n", - " alpha=0.5,\n", - " color=cmap[int(pareto_data.loc[alg,'pareto_rank'])]\n", - "# color='b'\n", - " )\n", + " [cilx, ciux],\n", + " [y, y],\n", + " alpha=0.5,\n", + " color=cmap[int(pareto_data.loc[alg, \"pareto_rank\"])],\n", + " # color='b'\n", + " )\n", " plt.plot(\n", - " [x, x],\n", - " [cily,ciuy],\n", - " alpha=0.5,\n", - " color=cmap[int(pareto_data.loc[alg,'pareto_rank'])]\n", - "# color='b'\n", - " )\n", + " [x, x],\n", + " [cily, ciuy],\n", + " alpha=0.5,\n", + " color=cmap[int(pareto_data.loc[alg, \"pareto_rank\"])],\n", + " # color='b'\n", + " )\n", " i += 1\n", "ax.set_aspect(1.0)\n", "ax.set_xlim(left=0)\n", "ax.set_ylim(bottom=0)\n", - "ticksize=16\n", - "plt.xticks([0,5,10,15,20],fontsize=ticksize)\n", - "plt.yticks([0,5,10,15,20],fontsize=ticksize)\n", + "ticksize = 16\n", + "plt.xticks([0, 5, 10, 15, 20], fontsize=ticksize)\n", + "plt.yticks([0, 5, 10, 15, 20], fontsize=ticksize)\n", "# ax.set_yscale('log')\n", - "plt.xlabel(xcol.replace('_',' ').replace('r2','$R^2$').title(), fontsize=labelsize)\n", - "plt.ylabel(ycol.replace('_',' ').title(), fontsize=labelsize)\n", + "plt.xlabel(xcol.replace(\"_\", \" \").replace(\"r2\", \"$R^2$\").title(), fontsize=labelsize)\n", + "plt.ylabel(ycol.replace(\"_\", \" \").title(), fontsize=labelsize)\n", "sns.despine(left=True, bottom=True)\n", - "save(name=\"pareto_plot_\"+xcol+'_'+ycol)" + "save(name=\"pareto_plot_\" + xcol + \"_\" + ycol)" ] }, { @@ -600,7 +675,7 @@ "metadata": {}, "outputs": [], "source": [ - "pareto_data['pareto_rank']" + "pareto_data[\"pareto_rank\"]" ] } ], diff --git a/postprocessing/collate_blackbox_results.py b/postprocessing/collate_blackbox_results.py index 17a78be6e..045d37567 100644 --- a/postprocessing/collate_blackbox_results.py +++ b/postprocessing/collate_blackbox_results.py @@ -7,140 +7,144 @@ ################################################################################ # Black-box problems ################################################################################ -import pandas as pd import json -import numpy as np +import sys from glob import glob + +import pandas as pd from tqdm import tqdm -import os -import sys -rdir = '../results_pmlb_r1/' +rdir = "../results_pmlb_r1/" if len(sys.argv) > 1: rdir = sys.argv[1] else: - print('no rdir provided, using',rdir) -print('reading results from directory', rdir) - + print("no rdir provided, using", rdir) +print("reading results from directory", rdir) + symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', - 'AIFeynman' -] -nongp_algs = [ - 'BSR', - 'DSR', - 'AIFeynman' + "AFP", + "AFP_FE", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", + "AIFeynman", ] +nongp_algs = ["BSR", "DSR", "AIFeynman"] gp_algs = [ - 'AFP', - 'AFP_FE', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', + "AFP", + "AFP_FE", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", ] ########## # load data from json ########## frames = [] comparison_cols = [ - 'dataset', - 'algorithm', - 'random_state', - 'time_time', - 'model_size', - 'symbolic_model', - 'r2_test', - 'mse_test', - 'mae_test', - 'params' + "dataset", + "algorithm", + "random_state", + "time_time", + "model_size", + "symbolic_model", + "r2_test", + "mse_test", + "mae_test", + "params", ] fails = [] -import pdb -for f in tqdm(glob(rdir + '/*/*.json')): - if 'cv_results' in f: +for f in tqdm(glob(rdir + "/*/*.json")): + if "cv_results" in f: continue # leave out symbolic data - if 'feynman_' in f or 'strogatz_' in f: + if "feynman_" in f or "strogatz_" in f: continue # leave out LinearReg, Lasso (we have SGD with penalty) - if any([m in f for m in ['LinearRegression','Lasso','EHCRegressor']]): + if any([m in f for m in ["LinearRegression", "Lasso", "EHCRegressor"]]): continue - try: - r = json.load(open(f,'r')) - if isinstance(r['symbolic_model'],list): -# print(f) - sm = ['B'+str(i)+'*'+ri for i, ri in enumerate(r['symbolic_model'])] - sm = '+'.join(sm) - r['symbolic_model'] = sm - - sub_r = {k:v for k,v in r.items() if k in comparison_cols} - # df = pd.DataFrame(sub_r) - frames.append(sub_r) + try: + r = json.load(open(f, "r")) + if isinstance(r["symbolic_model"], list): + # print(f) + sm = ["B" + str(i) + "*" + ri for i, ri in enumerate(r["symbolic_model"])] + r["symbolic_model"] = "+".join(sm) + + sub_r = {k: v for k, v in r.items() if k in comparison_cols} + # df = pd.DataFrame(sub_r) + frames.append(sub_r) # print(f) # print(r.keys()) except Exception as e: - fails.append([f,e]) + fails.append([f, e]) pass - -print(len(fails),'fails:',fails) + +print(len(fails), "fails:", fails) # df_results = pd.concat(frames) df_results = pd.DataFrame.from_records(frames) -df_results['params_str'] = df_results['params'].apply(str) -df_results = df_results.drop(columns=['params']) +df_results["params_str"] = df_results["params"].apply(str) +df_results = df_results.drop(columns=["params"]) ########## # cleanup ########## -df_results = df_results.rename(columns={'time_time':'training time (s)'}) -df_results.loc[:,'training time (hr)'] = df_results['training time (s)']/3600 +df_results = df_results.rename(columns={"time_time": "training time (s)"}) +df_results.loc[:, "training time (hr)"] = df_results["training time (s)"] / 3600 # remove regressor from names -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('Regressor','')) -#Rename SGD to Linear -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: 'Linear' if x=='SGD' else x) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("Regressor", "") +) +# Rename SGD to Linear +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: "Linear" if x == "SGD" else x +) # rename sembackpropgp to SBP -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('sembackpropgp','SBP-GP')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("sembackpropgp", "SBP-GP") +) # rename FE_AFP to AFP_FE -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('FE_AFP','AFP_FE')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("FE_AFP", "AFP_FE") +) # rename GPGOMEA to GP-GOMEA -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('GPGOMEA','GP-GOMEA')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("GPGOMEA", "GP-GOMEA") +) # add modified R2 with 0 floor -df_results['r2_zero_test'] = df_results['r2_test'].apply(lambda x: max(x,0)) +df_results["r2_zero_test"] = df_results["r2_test"].apply(lambda x: max(x, 0)) # label friedman ddatasets -df_results.loc[:,'friedman_dataset'] = df_results['dataset'].str.contains('_fri_') -print('loaded',len(df_results),'results') +df_results.loc[:, "friedman_dataset"] = df_results["dataset"].str.contains("_fri_") +print("loaded", len(df_results), "results") # additional metadata -df_results['symbolic_alg'] = df_results['algorithm'].apply(lambda x: x in symbolic_algs) +df_results["symbolic_alg"] = df_results["algorithm"].apply(lambda x: x in symbolic_algs) -for col in ['algorithm','dataset']: - print(df_results[col].nunique(), col+'s') +for col in ["algorithm", "dataset"]: + print(df_results[col].nunique(), col + "s") ########## # save results ########## -df_results.to_feather('../results/black-box_results.feather') -print('results saved to ../results/black-box_results.feather') +df_results.to_feather("../results/black-box_results.feather") +print("results saved to ../results/black-box_results.feather") ######## -print('mean trial count:') -print(df_results.groupby('algorithm')['dataset'].count().sort_values() - / df_results.dataset.nunique()) - +print("mean trial count:") +print( + df_results.groupby("algorithm")["dataset"].count().sort_values() + / df_results.dataset.nunique() +) diff --git a/postprocessing/collate_groundtruth_results.py b/postprocessing/collate_groundtruth_results.py index 92e0885e8..10a20193d 100644 --- a/postprocessing/collate_groundtruth_results.py +++ b/postprocessing/collate_groundtruth_results.py @@ -7,107 +7,127 @@ ################################################################################ # Ground-truth problems ################################################################################ -import pandas as pd import json -import numpy as np -from glob import glob -from tqdm import tqdm import os import sys +from glob import glob + +import pandas as pd +from tqdm import tqdm -rdir = '../results_sym_data/' +rdir = "../results_sym_data/" if len(sys.argv) > 1: rdir = sys.argv[1] else: - print('no rdir provided, using',rdir) -print('reading results from directory', rdir) + print("no rdir provided, using", rdir) +print("reading results from directory", rdir) ########## # load data from json ########## frames = [] -excluded_datasets = [ - 'feynman_test_10', - 'feynman_I_26_2', - 'feynman_I_30_5' -] -excluded_cols = [ - 'params' -] +excluded_datasets = ["feynman_test_10", "feynman_I_26_2", "feynman_I_30_5"] +excluded_cols = ["params"] fails = [] bad_bsr = [] updated = 0 -for f in tqdm(glob(rdir + '/*/*.json')): - if os.path.exists(f+'.updated'): - f += '.updated' +for f in tqdm(glob(rdir + "/*/*.json")): + if os.path.exists(f + ".updated"): + f += ".updated" updated += 1 - if 'cv_results' in f: + if "cv_results" in f: continue - if 'EHC' in f: + if "EHC" in f: continue if any([ed in f for ed in excluded_datasets]): continue - try: - r = json.load(open(f,'r')) - if isinstance(r['symbolic_model'],list): - print('WARNING: list returned for model:',f) + try: + r = json.load(open(f, "r")) + if isinstance(r["symbolic_model"], list): + print("WARNING: list returned for model:", f) bad_bsr.append(f) - sm = ['B'+str(i)+'*'+ri for i, ri in enumerate(r['symbolic_model'])] - sm = '+'.join(sm) - r['symbolic_model'] = sm - - sub_r = {k:v for k,v in r.items() if k not in excluded_cols} - # df = pd.DataFrame(sub_r) - frames.append(sub_r) + sm = ["B" + str(i) + "*" + ri for i, ri in enumerate(r["symbolic_model"])] + r["symbolic_model"] = "+".join(sm) + + sub_r = {k: v for k, v in r.items() if k not in excluded_cols} + # df = pd.DataFrame(sub_r) + frames.append(sub_r) # print(f) # print(r.keys()) except Exception as e: - fails.append([f,e]) + fails.append([f, e]) pass - -print('{} results files loaded, {} ({:.1f}%) of which are ' - 'updated'.format(len(frames), updated, updated/len(frames)*100)) -print(len(fails),'fails:') -for f in fails: + +print( + "{} results files loaded, {} ({:.1f}%) of which are updated".format( + len(frames), updated, updated / len(frames) * 100 + ) +) +print(len(fails), "fails:") +for f in fails: print(f[0]) -print('bad bsr:',bad_bsr) +print("bad bsr:", bad_bsr) df_results = pd.DataFrame.from_records(frames) ########## # cleanup ########## -df_results = df_results.rename(columns={'time_time':'training time (s)'}) -df_results.loc[:,'training time (hr)'] = df_results['training time (s)']/3600 +df_results = df_results.rename(columns={"time_time": "training time (s)"}) +df_results.loc[:, "training time (hr)"] = df_results["training time (s)"] / 3600 # add modified R2 with 0 floor -df_results['r2_zero_test'] = df_results['r2_test'].apply(lambda x: max(x,0)) -for col in ['symbolic_error_is_zero', 'symbolic_error_is_constant', 'symbolic_fraction_is_constant']: - df_results.loc[:,col] = df_results[col].fillna(False) -print(','.join(df_results.algorithm.unique())) +df_results["r2_zero_test"] = df_results["r2_test"].apply(lambda x: max(x, 0)) +for col in [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", +]: + df_results.loc[:, col] = df_results[col].fillna(False) +print(",".join(df_results.algorithm.unique())) # remove 'Regressor' from names -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('Regressor','')) -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('tuned.','')) -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('sembackpropgp','SBP-GP')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("Regressor", "") +) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("tuned.", "") +) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("sembackpropgp", "SBP-GP") +) # rename FE_AFP to AFP_FE -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('FE_AFP','AFP_FE')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("FE_AFP", "AFP_FE") +) # rename GPGOMEA to GP-GOMEA -df_results['algorithm'] = df_results['algorithm'].apply(lambda x: x.replace('GPGOMEA','GP-GOMEA')) +df_results["algorithm"] = df_results["algorithm"].apply( + lambda x: x.replace("GPGOMEA", "GP-GOMEA") +) # indicator of strogatz or feynman -df_results['data_group'] = df_results['dataset'].apply(lambda x: 'Feynman' if 'feynman' in x else 'Strogatz') +df_results["data_group"] = df_results["dataset"].apply( + lambda x: "Feynman" if "feynman" in x else "Strogatz" +) ########## # compute symbolic solutions ########## -df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero', - 'symbolic_error_is_constant', - 'symbolic_fraction_is_constant'] - ].apply(any,raw=True, axis=1) -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0') -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan') +df_results.loc[:, "symbolic_solution"] = df_results[ + [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", + ] +].apply(any, raw=True, axis=1) +df_results.loc[:, "symbolic_solution"] = ( + df_results["symbolic_solution"] & ~df_results["simplified_symbolic_model"].isna() +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "0" +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "nan" +) ########## # save results ########## -df_results.to_feather('../results/ground-truth_results.feather') -print('results saved to ../results/ground-truth_results.feather') - +df_results.to_feather("../results/ground-truth_results.feather") +print("results saved to ../results/ground-truth_results.feather") diff --git a/postprocessing/genCSVs.py b/postprocessing/genCSVs.py index c451aff78..daada54ac 100644 --- a/postprocessing/genCSVs.py +++ b/postprocessing/genCSVs.py @@ -1,103 +1,142 @@ +from glob import glob + import numpy as np import pandas as pd -from glob import glob # CSV with datasets information -datadir = '../../../pmlb/datasets/' +datadir = "../../../pmlb/datasets/" frames = [] -for f in glob(datadir+'/*/*.tsv.gz'): - df = pd.read_csv(f,sep='\t') - group = 'feynman' if 'feynman' in f else 'strogatz' if 'strogatz' in f else 'black-box' - frames.append(dict( - name=f.split('/')[-1][:-7], - nsamples = df.shape[0], - nfeatures = df.shape[1], - npoints = df.shape[0]*df.shape[1], - Group=group - )) - +for f in glob(datadir + "/*/*.tsv.gz"): + df = pd.read_csv(f, sep="\t") + group = ( + "feynman" if "feynman" in f else "strogatz" if "strogatz" in f else "black-box" + ) + frames.append( + dict( + name=f.split("/")[-1][:-7], + nsamples=df.shape[0], + nfeatures=df.shape[1], + npoints=df.shape[0] * df.shape[1], + Group=group, + ) + ) + df = pd.DataFrame.from_records(frames) df.to_csv("../docs/csv/datasets_info.csv") # CSV for the blackbox results symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', - 'AIFeynman' + "AFP", + "AFP_FE", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", + "AIFeynman", ] -x_vars=[ 'rmse_test', - 'log_mse_test', - 'r2_test', - 'model_size', - 'training time (s)'] - -df_results = pd.read_feather('../results/black-box_results.feather') -df_blackbox = df_results.merge(df_results.groupby('dataset')['algorithm'].nunique().reset_index(), - on='dataset',suffixes=('','_count')) - -df_sum = df_blackbox.groupby(['algorithm','dataset'],as_index=False).median() -df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt) -df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x)) - -df_sum['algorithm'] = df_sum['algorithm'].apply(lambda x: '*'+x if x in symbolic_algs else x) -(df_sum - .groupby(["algorithm","dataset"]) - .median()[x_vars] - .to_csv("../docs/csv/blackbox_results.csv") +x_vars = ["rmse_test", "log_mse_test", "r2_test", "model_size", "training time (s)"] + +df_results = pd.read_feather("../results/black-box_results.feather") +df_blackbox = df_results.merge( + df_results.groupby("dataset")["algorithm"].nunique().reset_index(), + on="dataset", + suffixes=("", "_count"), +) + +df_sum = df_blackbox.groupby(["algorithm", "dataset"], as_index=False).median() +df_sum["rmse_test"] = df_sum["mse_test"].apply(np.sqrt) +df_sum["log_mse_test"] = df_sum["mse_test"].apply(lambda x: np.log(1 + x)) + +df_sum["algorithm"] = df_sum["algorithm"].apply( + lambda x: "*" + x if x in symbolic_algs else x +) +( + df_sum.groupby(["algorithm", "dataset"]) + .median()[x_vars] + .to_csv("../docs/csv/blackbox_results.csv") ) # Aggregate -df_sum.merge(df, left_on="dataset", right_on="name").to_csv("../docs/csv/blackbox_results_datasets.csv") +df_sum.merge(df, left_on="dataset", right_on="name").to_csv( + "../docs/csv/blackbox_results_datasets.csv" +) # Ground Truth results -df_results = pd.read_feather('../results/ground-truth_results.feather') -df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero', - 'symbolic_error_is_constant', - 'symbolic_fraction_is_constant'] - ].apply(any,raw=True, axis=1) +df_results = pd.read_feather("../results/ground-truth_results.feather") +df_results.loc[:, "symbolic_solution"] = df_results[ + [ + "symbolic_error_is_zero", + "symbolic_error_is_constant", + "symbolic_fraction_is_constant", + ] +].apply(any, raw=True, axis=1) # clean up any corner cases (constant models, failures) -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0') -df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan') -df_results2 = df_results.merge(df_results.groupby(['dataset','target_noise'])['algorithm'].nunique().reset_index(), - on=['dataset','target_noise'],suffixes=('','_count')) +df_results.loc[:, "symbolic_solution"] = ( + df_results["symbolic_solution"] & ~df_results["simplified_symbolic_model"].isna() +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "0" +) +df_results.loc[:, "symbolic_solution"] = df_results["symbolic_solution"] & ~( + df_results["simplified_symbolic_model"] == "nan" +) +df_results2 = df_results.merge( + df_results.groupby(["dataset", "target_noise"])["algorithm"] + .nunique() + .reset_index(), + on=["dataset", "target_noise"], + suffixes=("", "_count"), +) # count repeat trials df_results2 = df_results2.merge( - df_results2.groupby(['algorithm','dataset','target_noise'])['random_state'].nunique().reset_index(), - on=['algorithm','dataset','target_noise'],suffixes=('','_repeats')) + df_results2.groupby(["algorithm", "dataset", "target_noise"])["random_state"] + .nunique() + .reset_index(), + on=["algorithm", "dataset", "target_noise"], + suffixes=("", "_repeats"), +) -# accuracy-based exact solutions -df_results2['accuracy_solution'] = df_results2['r2_test'].apply(lambda x: x > 0.999).astype(float) +# accuracy-based exact solutions +df_results2["accuracy_solution"] = ( + df_results2["r2_test"].apply(lambda x: x > 0.999).astype(float) +) -# get mean solution rates for algs on datasets at specific noise levels, averaged over trials -for soln in ['accuracy_solution','symbolic_solution']: +# get mean solution rates for algs on datasets at specific noise levels, averaged over trials +for soln in ["accuracy_solution", "symbolic_solution"]: df_results2 = df_results2.merge( - df_results2.groupby(['algorithm','dataset','target_noise'])[soln].mean().reset_index(), - on=['algorithm','dataset', 'target_noise'],suffixes=('','_rate')) - -df_sum = df_results2.groupby(['algorithm','dataset','target_noise','data_group'],as_index=False).median() - -for soln in ['accuracy_solution','symbolic_solution']: - df_sum[soln +'_rate_(%)'] = df_sum[soln+'_rate'].apply(lambda x: x*100) -df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt) -df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x)) - -columns = ['algorithm','dataset','target_noise','data_group' - ,'symbolic_solution_rate_(%)','accuracy_solution' - ,'rmse_test', 'log_mse_test' - ] -df_sum[columns].to_csv('../docs/csv/groundtruth.csv', index=False) + df_results2.groupby(["algorithm", "dataset", "target_noise"])[soln] + .mean() + .reset_index(), + on=["algorithm", "dataset", "target_noise"], + suffixes=("", "_rate"), + ) + +df_sum = df_results2.groupby( + ["algorithm", "dataset", "target_noise", "data_group"], as_index=False +).median() +for soln in ["accuracy_solution", "symbolic_solution"]: + df_sum[soln + "_rate_(%)"] = df_sum[soln + "_rate"].apply(lambda x: x * 100) +df_sum["rmse_test"] = df_sum["mse_test"].apply(np.sqrt) +df_sum["log_mse_test"] = df_sum["mse_test"].apply(lambda x: np.log(1 + x)) + +columns = [ + "algorithm", + "dataset", + "target_noise", + "data_group", + "symbolic_solution_rate_(%)", + "accuracy_solution", + "rmse_test", + "log_mse_test", +] +df_sum[columns].to_csv("../docs/csv/groundtruth.csv", index=False) diff --git a/postprocessing/groundtruth_results.ipynb b/postprocessing/groundtruth_results.ipynb index 6b60bb4bc..47e63fb2d 100644 --- a/postprocessing/groundtruth_results.ipynb +++ b/postprocessing/groundtruth_results.ipynb @@ -19,33 +19,33 @@ "metadata": {}, "outputs": [], "source": [ - "from tabulate import tabulate\n", - "import pandas as pd\n", - "import json\n", + "import os\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from glob import glob\n", - "from tqdm import tqdm\n", + "import pandas as pd\n", "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib\n", - "matplotlib.rc('pdf', fonttype=42)\n", - "import os\n", + "\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", + "\n", "sns.set(font_scale=1.2)\n", - "rdir = '../results/'\n", + "rdir = \"../results/\"\n", "# figdir = 'figs/ground-truth/'\n", - "figdir = '../../paper/figs/results_sym_data/'\n", + "figdir = \"../../paper/figs/results_sym_data/\"\n", + "\n", + "print(\"figdir:\", figdir)\n", "\n", - "print('figdir:',figdir)\n", - "def save(h=None,name='tmp'):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "\n", + "def save(h=None, name=\"tmp\"):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',name+'.pdf')\n", + " print(\"saving\", name + \".pdf\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.pdf', dpi=400, bbox_inches='tight')\n", - " " + " plt.savefig(figdir + \"/\" + name + \".pdf\", dpi=400, bbox_inches=\"tight\")" ] }, { @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_results = pd.read_feather(rdir+'/ground-truth_results.feather')" + "df_results = pd.read_feather(rdir + \"/ground-truth_results.feather\")" ] }, { @@ -77,17 +77,26 @@ "metadata": {}, "outputs": [], "source": [ - "df_results.loc[:,'symbolic_solution'] = df_results[['symbolic_error_is_zero',\n", - " 'symbolic_error_is_constant',\n", - " 'symbolic_fraction_is_constant']\n", - " ].apply(any,raw=True, axis=1)\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\n", + " [\n", + " \"symbolic_error_is_zero\",\n", + " \"symbolic_error_is_constant\",\n", + " \"symbolic_fraction_is_constant\",\n", + " ]\n", + "].apply(any, raw=True, axis=1)\n", "# clean up any corner cases (constant models, failures)\n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~df_results['simplified_symbolic_model'].isna() \n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == '0')\n", - "df_results.loc[:,'symbolic_solution'] = df_results['symbolic_solution'] & ~(df_results['simplified_symbolic_model'] == 'nan')\n", + "df_results.loc[:, \"symbolic_solution\"] = (\n", + " df_results[\"symbolic_solution\"] & ~df_results[\"simplified_symbolic_model\"].isna()\n", + ")\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\"symbolic_solution\"] & ~(\n", + " df_results[\"simplified_symbolic_model\"] == \"0\"\n", + ")\n", + "df_results.loc[:, \"symbolic_solution\"] = df_results[\"symbolic_solution\"] & ~(\n", + " df_results[\"simplified_symbolic_model\"] == \"nan\"\n", + ")\n", "\n", "# save results for detailed tabulating\n", - "df_results.to_feather(rdir+'ground-truth_solns.feather')" + "df_results.to_feather(rdir + \"ground-truth_solns.feather\")" ] }, { @@ -103,43 +112,63 @@ "metadata": {}, "outputs": [], "source": [ - "df_results2 = df_results.merge(df_results.groupby(['dataset','target_noise'])['algorithm'].nunique().reset_index(),\n", - " on=['dataset','target_noise'],suffixes=('','_count'))\n", + "df_results2 = df_results.merge(\n", + " df_results.groupby([\"dataset\", \"target_noise\"])[\"algorithm\"]\n", + " .nunique()\n", + " .reset_index(),\n", + " on=[\"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_count\"),\n", + ")\n", "# count repeat trials\n", "df_results2 = df_results2.merge(\n", - " df_results2.groupby(['algorithm','dataset','target_noise'])['random_state'].nunique().reset_index(),\n", - " on=['algorithm','dataset','target_noise'],suffixes=('','_repeats'))\n", + " df_results2.groupby([\"algorithm\", \"dataset\", \"target_noise\"])[\"random_state\"]\n", + " .nunique()\n", + " .reset_index(),\n", + " on=[\"algorithm\", \"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_repeats\"),\n", + ")\n", "\n", - "# accuracy-based exact solutions \n", - "df_results2['accuracy_solution'] = df_results2['r2_test'].apply(lambda x: x > 0.999).astype(float)\n", + "# accuracy-based exact solutions\n", + "df_results2[\"accuracy_solution\"] = (\n", + " df_results2[\"r2_test\"].apply(lambda x: x > 0.999).astype(float)\n", + ")\n", "\n", - "# get mean solution rates for algs on datasets at specific noise levels, averaged over trials \n", - "for soln in ['accuracy_solution','symbolic_solution']:\n", + "# get mean solution rates for algs on datasets at specific noise levels, averaged over trials\n", + "for soln in [\"accuracy_solution\", \"symbolic_solution\"]:\n", " df_results2 = df_results2.merge(\n", - " df_results2.groupby(['algorithm','dataset','target_noise'])[soln].mean().reset_index(),\n", - " on=['algorithm','dataset', 'target_noise'],suffixes=('','_rate'))\n", - " \n", + " df_results2.groupby([\"algorithm\", \"dataset\", \"target_noise\"])[soln]\n", + " .mean()\n", + " .reset_index(),\n", + " on=[\"algorithm\", \"dataset\", \"target_noise\"],\n", + " suffixes=(\"\", \"_rate\"),\n", + " )\n", + "\n", "# # rankings\n", - "for col in [c for c in df_results2.columns if c.endswith('test') or c.endswith('size')]:\n", - " ascending = 'r2' not in col\n", - " df_results2[col+'_rank_per_trial']=df_results2.groupby(\n", - " ['dataset','target_noise','random_state'])[col].apply(\n", - " lambda x: \n", - " round(x,3).rank(\n", - " ascending=ascending).astype(int))\n", - " \n", - "df_sum = df_results2.groupby(['algorithm','dataset','target_noise','data_group'],as_index=False).median()\n", + "for col in [c for c in df_results2.columns if c.endswith(\"test\") or c.endswith(\"size\")]:\n", + " ascending = \"r2\" not in col\n", + " df_results2[col + \"_rank_per_trial\"] = df_results2.groupby(\n", + " [\"dataset\", \"target_noise\", \"random_state\"]\n", + " )[col].apply(lambda x: round(x, 3).rank(ascending=ascending).astype(int))\n", + "\n", + "df_sum = df_results2.groupby(\n", + " [\"algorithm\", \"dataset\", \"target_noise\", \"data_group\"], as_index=False\n", + ").median()\n", "# rankings and normalized scores per dataset\n", - "for col in [c for c in df_sum.columns if any([c.endswith(n) for n in ['test','size','rate']])]:\n", - " ascending = 'r2' not in col and 'solution' not in col\n", - " df_sum[col+'_rank']=df_sum.groupby(['dataset','target_noise'])[col].apply(\n", - " lambda x: round(x,3).rank(ascending=ascending).astype(int) )\n", - " df_sum[col+'_norm'] = df_sum.groupby(['dataset','target_noise'])[col].apply(lambda x: (x-x.min())/(x.max()-x.min()))\n", + "for col in [\n", + " c for c in df_sum.columns if any([c.endswith(n) for n in [\"test\", \"size\", \"rate\"]])\n", + "]:\n", + " ascending = \"r2\" not in col and \"solution\" not in col\n", + " df_sum[col + \"_rank\"] = df_sum.groupby([\"dataset\", \"target_noise\"])[col].apply(\n", + " lambda x: round(x, 3).rank(ascending=ascending).astype(int)\n", + " )\n", + " df_sum[col + \"_norm\"] = df_sum.groupby([\"dataset\", \"target_noise\"])[col].apply(\n", + " lambda x: (x - x.min()) / (x.max() - x.min())\n", + " )\n", "# df_sum['success_rate'] = df_results2.groupby(['algorithm','dataset'])['solution'].mean().reset_index()\n", - "for soln in ['accuracy_solution','symbolic_solution']:\n", - " df_sum[soln +'_rate_(%)'] = df_sum[soln+'_rate'].apply(lambda x: x*100)\n", - "df_sum['rmse_test'] = df_sum['mse_test'].apply(np.sqrt)\n", - "df_sum['log_mse_test'] = df_sum['mse_test'].apply(lambda x: np.log(1+x))\n", + "for soln in [\"accuracy_solution\", \"symbolic_solution\"]:\n", + " df_sum[soln + \"_rate_(%)\"] = df_sum[soln + \"_rate\"].apply(lambda x: x * 100)\n", + "df_sum[\"rmse_test\"] = df_sum[\"mse_test\"].apply(np.sqrt)\n", + "df_sum[\"log_mse_test\"] = df_sum[\"mse_test\"].apply(lambda x: np.log(1 + x))\n", "df_results = df_results2\n", "df_sum" ] @@ -158,7 +187,9 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum.to_csv(rdir+'/symbolic_dataset_results_sum.csv.gz',compression='gzip', index=False)" + "df_sum.to_csv(\n", + " rdir + \"/symbolic_dataset_results_sum.csv.gz\", compression=\"gzip\", index=False\n", + ")" ] }, { @@ -174,7 +205,9 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum.groupby(['target_noise','algorithm'])['symbolic_solution_rate_(%)'].mean().round(2).unstack().transpose()" + "df_sum.groupby([\"target_noise\", \"algorithm\"])[\n", + " \"symbolic_solution_rate_(%)\"\n", + "].mean().round(2).unstack().transpose()" ] }, { @@ -190,86 +223,98 @@ "metadata": {}, "outputs": [], "source": [ - "sns.set_style('whitegrid')\n", - "def compare(df_compare=None, x='r2_test',y='algorithm', row=None, col=None, scale=None, xlim=[], est=np.mean,\n", - " orient='h', hue=None, **kwargs):\n", + "sns.set_style(\"whitegrid\")\n", + "\n", + "\n", + "def compare(\n", + " df_compare=None,\n", + " x=\"r2_test\",\n", + " y=\"algorithm\",\n", + " row=None,\n", + " col=None,\n", + " scale=None,\n", + " xlim=[],\n", + " est=np.mean,\n", + " orient=\"h\",\n", + " hue=None,\n", + " **kwargs,\n", + "):\n", " df_compare = df_compare.copy()\n", - " if row==None and col == None:\n", - " aspect=1\n", + " if row is None and col is None:\n", + " aspect = 1\n", " else:\n", - " aspect=0.55\n", - "# plt.figure(figsize=(8,7))\n", - " tmp = df_compare.groupby(['target_noise',y])[x].apply(est).unstack().mean()\n", + " aspect = 0.55\n", + " # plt.figure(figsize=(8,7))\n", + " tmp = df_compare.groupby([\"target_noise\", y])[x].apply(est).unstack().mean()\n", " order = tmp.sort_values(ascending=False).index\n", - " \n", - " for c in [x,y,row,col]:\n", + "\n", + " for c in [x, y, row, col]:\n", " if c:\n", - " df_compare = df_compare.rename(columns={c:c.replace('_',' ').title()})\n", - " \n", - " x = x.replace('_',' ').title()\n", - " y = y.replace('_',' ').title()\n", + " df_compare = df_compare.rename(columns={c: c.replace(\"_\", \" \").title()})\n", + "\n", + " x = x.replace(\"_\", \" \").title()\n", + " y = y.replace(\"_\", \" \").title()\n", " if row:\n", - " row = row.replace('_',' ').title()\n", + " row = row.replace(\"_\", \" \").title()\n", " if col:\n", - " col = col.replace('_',' ').title()\n", - " \n", - " if scale=='log' and len(xlim)>0 and xlim[0] == 0:\n", - " df_compare.loc[:,x] += 1\n", + " col = col.replace(\"_\", \" \").title()\n", + "\n", + " if scale == \"log\" and len(xlim) > 0 and xlim[0] == 0:\n", + " df_compare.loc[:, x] += 1\n", " xlim[0] = 1\n", - " xnew = '1 + '+x\n", - " df_compare=df_compare.rename(columns={x:xnew})\n", + " xnew = \"1 + \" + x\n", + " df_compare = df_compare.rename(columns={x: xnew})\n", " x = xnew\n", - " if orient=='v':\n", + " if orient == \"v\":\n", " tmp = x\n", " x = y\n", " y = tmp\n", " if col and not row:\n", - " col_wrap = min(4, df_compare[col].nunique()) \n", + " col_wrap = min(4, df_compare[col].nunique())\n", " else:\n", - " col_wrap=None\n", - " \n", + " col_wrap = None\n", + "\n", " cat_args = dict(\n", - " data=df_compare, \n", - " kind='point',\n", - " y=y,\n", - " x=x,\n", - " order=order,\n", - " row=row,\n", - " col=col,\n", - " col_wrap=col_wrap,\n", - " palette='flare_r',\n", - " margin_titles=True,\n", - " aspect=aspect,\n", - " hue=hue,\n", - " legend_out=False,\n", + " data=df_compare,\n", + " kind=\"point\",\n", + " y=y,\n", + " x=x,\n", + " order=order,\n", + " row=row,\n", + " col=col,\n", + " col_wrap=col_wrap,\n", + " palette=\"flare_r\",\n", + " margin_titles=True,\n", + " aspect=aspect,\n", + " hue=hue,\n", + " legend_out=False,\n", " )\n", " cat_args.update(kwargs)\n", - " g = sns.catplot( **cat_args )\n", + " g = sns.catplot(**cat_args)\n", " if hue:\n", - " g._legend.remove() #(title=hue.replace('_',' ').title())\n", - " g.axes.flat[-1].legend(title=hue.replace('_',' ').title(),\n", - " fontsize=10\n", - " )\n", - " for ax in g.axes.flat: \n", + " g._legend.remove() # (title=hue.replace('_',' ').title())\n", + " g.axes.flat[-1].legend(title=hue.replace(\"_\", \" \").title(), fontsize=10)\n", + " for ax in g.axes.flat:\n", " ax.yaxis.grid(True)\n", - " ax.set_ylabel('')\n", - " ax.set_xlabel(ax.get_xlabel().replace('Symbolic ',''))\n", + " ax.set_ylabel(\"\")\n", + " ax.set_xlabel(ax.get_xlabel().replace(\"Symbolic \", \"\"))\n", " if col:\n", " ttl = ax.get_title()\n", - " ax.set_title(ttl.replace(col,'').replace('=',''))\n", + " ax.set_title(ttl.replace(col, \"\").replace(\"=\", \"\"))\n", "\n", - " \n", - " if len(xlim)>0:\n", - " plt.xlim(xlim[0],xlim[1])\n", + " if len(xlim) > 0:\n", + " plt.xlim(xlim[0], xlim[1])\n", " if scale:\n", " plt.gca().set_xscale(scale)\n", "\n", " sns.despine(left=True, bottom=True)\n", - " savename = '-'.join(['cat-'+cat_args['kind']+'plot',x+ '-by-'+ y])\n", - " if row: savename += '_'+row\n", - " if col: savename += '_'+col\n", - " \n", - " save(g, savename )" + " savename = \"-\".join([\"cat-\" + cat_args[\"kind\"] + \"plot\", x + \"-by-\" + y])\n", + " if row:\n", + " savename += \"_\" + row\n", + " if col:\n", + " savename += \"_\" + col\n", + "\n", + " save(g, savename)" ] }, { @@ -278,24 +323,25 @@ "metadata": {}, "outputs": [], "source": [ - "for metric in ['symbolic_solution_rate_(%)','r2_test','accuracy_solution']:\n", - "# for metric in ['r2_test']:\n", - " for kind in ['point']: #,'strip']:\n", - " args =dict(df_compare=df_sum, x=metric, est=np.mean, orient='h',\n", - " kind=kind) \n", - " if kind=='point': \n", - " args['join'] = False \n", - " args['markers']=['o','s','x','+']\n", - " if metric == 'r2_test':\n", - " args['xlim'] = [-1, 1]\n", - " compare(**args,\n", - " hue='target_noise', \n", - " col=None,\n", - " ) \n", - " compare(**args, \n", - " hue='target_noise', \n", - " col='data_group',\n", - " ) " + "for metric in [\"symbolic_solution_rate_(%)\", \"r2_test\", \"accuracy_solution\"]:\n", + " # for metric in ['r2_test']:\n", + " for kind in [\"point\"]: # ,'strip']:\n", + " args = dict(df_compare=df_sum, x=metric, est=np.mean, orient=\"h\", kind=kind)\n", + " if kind == \"point\":\n", + " args[\"join\"] = False\n", + " args[\"markers\"] = [\"o\", \"s\", \"x\", \"+\"]\n", + " if metric == \"r2_test\":\n", + " args[\"xlim\"] = [-1, 1]\n", + " compare(\n", + " **args,\n", + " hue=\"target_noise\",\n", + " col=None,\n", + " )\n", + " compare(\n", + " **args,\n", + " hue=\"target_noise\",\n", + " col=\"data_group\",\n", + " )" ] }, { @@ -306,66 +352,72 @@ "source": [ "# Make the PairGrid\n", "df_plot = df_sum.copy()\n", - "tmp = df_plot.groupby(['target_noise','algorithm'])['symbolic_solution_rate'].mean().unstack().mean()\n", + "tmp = (\n", + " df_plot.groupby([\"target_noise\", \"algorithm\"])[\"symbolic_solution_rate\"]\n", + " .mean()\n", + " .unstack()\n", + " .mean()\n", + ")\n", "order = tmp.sort_values(ascending=False).index\n", - "df_plot['size_diff'] = df_plot['model_size']-df_plot['simplified_complexity']+1\n", - "x_vars=[\n", - "# 'accuracy_solution_rate_(%)',\n", - "# 'mse_test',\n", - "# 'r2_test_rank',\n", - "# 'r2_test_norm',\n", - " 'symbolic_solution_rate_(%)',\n", - " 'r2_test',\n", - " 'simplified_complexity',\n", - "# 'size_diff',\n", - "# 'model_size',\n", - "# 'training time (s)',\n", - "# 'solution'\n", + "df_plot[\"size_diff\"] = df_plot[\"model_size\"] - df_plot[\"simplified_complexity\"] + 1\n", + "x_vars = [\n", + " # 'accuracy_solution_rate_(%)',\n", + " # 'mse_test',\n", + " # 'r2_test_rank',\n", + " # 'r2_test_norm',\n", + " \"symbolic_solution_rate_(%)\",\n", + " \"r2_test\",\n", + " \"simplified_complexity\",\n", + " # 'size_diff',\n", + " # 'model_size',\n", + " # 'training time (s)',\n", + " # 'solution'\n", "]\n", - "g = sns.PairGrid(df_plot, \n", - " x_vars=x_vars,\n", - " y_vars=['algorithm'],\n", - " height=6.5, \n", - " aspect=0.7,\n", - " hue='target_noise',\n", - "# hue_order=[0.01,0.001,0]\n", - "# hue='dataset'\n", - " )\n", - "g.map(sns.pointplot, \n", - "# size=10,\n", - " orient=\"h\",\n", - " jitter=False,\n", - " order=order,\n", - " palette=\"flare_r\",\n", - " errwidth=2,\n", - " linewidth=0.01,\n", - " markeredgecolor='w',\n", - " join=False,\n", - " estimator=np.mean,\n", - " n_boot=1000,\n", - " markers=['x','o','s','+'],\n", - " markeralpha=0.5\n", - " )\n", - "plt.legend(title='Target Noise')\n", - "titles = [x.replace('_',' ').title().replace('(S)','(s)') for x in x_vars]\n", + "g = sns.PairGrid(\n", + " df_plot,\n", + " x_vars=x_vars,\n", + " y_vars=[\"algorithm\"],\n", + " height=6.5,\n", + " aspect=0.7,\n", + " hue=\"target_noise\",\n", + " # hue_order=[0.01,0.001,0]\n", + " # hue='dataset'\n", + ")\n", + "g.map(\n", + " sns.pointplot,\n", + " # size=10,\n", + " orient=\"h\",\n", + " jitter=False,\n", + " order=order,\n", + " palette=\"flare_r\",\n", + " errwidth=2,\n", + " linewidth=0.01,\n", + " markeredgecolor=\"w\",\n", + " join=False,\n", + " estimator=np.mean,\n", + " n_boot=1000,\n", + " markers=[\"x\", \"o\", \"s\", \"+\"],\n", + " markeralpha=0.5,\n", + ")\n", + "plt.legend(title=\"Target Noise\")\n", + "titles = [x.replace(\"_\", \" \").title().replace(\"(S)\", \"(s)\") for x in x_vars]\n", "\n", "for ax, title in zip(g.axes.flat, titles):\n", - "\n", " # remove xlabel\n", - " ax.set_xlabel('')\n", - " ax.set_ylabel('')\n", + " ax.set_xlabel(\"\")\n", + " ax.set_ylabel(\"\")\n", " # Set a different title for each axes\n", " ax.set(title=title)\n", - " \n", - " if any([n in title.lower() for n in ['size','complexity','time']]):\n", - " ax.set_xscale('log')\n", - " if 'R2' in title and 'Rank' not in title:\n", - " ax.set(title=title.replace('R2','$R^2$'))\n", - " ax.set_xlim([0,1])\n", + "\n", + " if any([n in title.lower() for n in [\"size\", \"complexity\", \"time\"]]):\n", + " ax.set_xscale(\"log\")\n", + " if \"R2\" in title and \"Rank\" not in title:\n", + " ax.set(title=title.replace(\"R2\", \"$R^2$\"))\n", + " ax.set_xlim([0, 1])\n", "\n", " # Make the grid horizontal instead of vertical\n", " ax.yaxis.grid(True)\n", - "save(g, 'pairgrid_'+'_'.join(x_vars))" + "save(g, \"pairgrid_\" + \"_\".join(x_vars))" ] } ], diff --git a/postprocessing/paretoCSVs.py b/postprocessing/paretoCSVs.py index 73fb7fbd5..3be4d2d9d 100644 --- a/postprocessing/paretoCSVs.py +++ b/postprocessing/paretoCSVs.py @@ -1,59 +1,62 @@ -import pandas as pd import numpy as np -from glob import glob +import pandas as pd from pareto_utils import front np.random.seed(42) -def bootstrap(val, n = 1000, fn=np.mean): + +def bootstrap(val, n=1000, fn=np.mean): val_samples = [] for i in range(n): - sample = np.random.randint(0,len(val)-1, size=len(val)) - val_samples.append( fn(val[sample]) ) + sample = np.random.randint(0, len(val) - 1, size=len(val)) + val_samples.append(fn(val[sample])) m = np.mean(val_samples) sd = np.std(val_samples) - ci_upper = np.quantile(val_samples,0.95) - ci_lower = np.quantile(val_samples,0.05) - return m, sd, ci_upper,ci_lower + ci_upper = np.quantile(val_samples, 0.95) + ci_lower = np.quantile(val_samples, 0.05) + return m, sd, ci_upper, ci_lower + df = pd.read_csv("../docs/csv/blackbox_results.csv") -xcol = 'r2_test' -ycol = 'model_size' -zcol = 'training time (s)' +xcol = "r2_test" +ycol = "model_size" +zcol = "training time (s)" # outline pareto front -pareto_data = df.groupby('algorithm').median() +pareto_data = df.groupby("algorithm").median() + def create_front(df, xc, yc): - objs = df[[xc,yc]].values + objs = df[[xc, yc]].values levels = 5 PFs = [] for el in range(levels): - PF = front(-objs[:,0],objs[:,1]) - objs[PF,:] = 1e10 + PF = front(-objs[:, 0], objs[:, 1]) + objs[PF, :] = 1e10 PFs.append(PF) - if np.all(objs[:,0]==1e10): + if np.all(objs[:, 0] == 1e10): break front_alg = {} for i, pf in enumerate(PFs): print(f"FRONT {i}") for ix in pf: - name = df.iloc[ix].name + name = df.iloc[ix].name if name not in front_alg: front_alg[name] = i print(f"{name}") return front_alg -df['g1']=-1 -for k,v in create_front(pareto_data.copy(), xcol, ycol).items(): - df.loc[df.algorithm == k,'g1'] = v -df['g2']=-1 -for k,v in create_front(pareto_data.copy(), xcol, zcol).items(): - df.loc[df.algorithm == k,'g2'] = v -df['g3']=-1 -for k,v in create_front(pareto_data.copy(), ycol, zcol).items(): - df.loc[df.algorithm == k,'g3'] = v + +df["g1"] = -1 +for k, v in create_front(pareto_data.copy(), xcol, ycol).items(): + df.loc[df.algorithm == k, "g1"] = v +df["g2"] = -1 +for k, v in create_front(pareto_data.copy(), xcol, zcol).items(): + df.loc[df.algorithm == k, "g2"] = v +df["g3"] = -1 +for k, v in create_front(pareto_data.copy(), ycol, zcol).items(): + df.loc[df.algorithm == k, "g3"] = v df.groupby(by="algorithm").median().to_csv("../docs/csv/pareto.csv") diff --git a/postprocessing/pareto_utils.py b/postprocessing/pareto_utils.py index 0838db7c8..4e251d111 100644 --- a/postprocessing/pareto_utils.py +++ b/postprocessing/pareto_utils.py @@ -1,53 +1,55 @@ import numpy as np + + # PARETO FRONT TOOLS -def check_dominance(p1,p2): +def check_dominance(p1, p2): flag1 = 0 flag2 = 0 - for o1,o2 in zip(p1,p2): + for o1, o2 in zip(p1, p2): if o1 < o2: flag1 = 1 elif o1 > o2: flag2 = 1 - if flag1==1 and flag2 == 0: + if flag1 == 1 and flag2 == 0: return 1 - elif flag1==0 and flag2 == 1: + elif flag1 == 0 and flag2 == 1: return -1 else: return 0 -def front(obj1,obj2): + +def front(obj1, obj2): """return indices from x and y that are on the Pareto front.""" - rank = [] - assert(len(obj1)==len(obj2)) + assert len(obj1) == len(obj2) n_inds = len(obj1) front = [] for i in np.arange(n_inds): - p = (obj1[i],obj2[i]) + p = (obj1[i], obj2[i]) dcount = 0 dom = [] for j in np.arange(n_inds): - q = (obj1[j],obj2[j]) - compare = check_dominance(p,q) + q = (obj1[j], obj2[j]) + compare = check_dominance(p, q) if compare == 1: dom.append(j) -# print(p,'dominates',q) + # print(p,'dominates',q) elif compare == -1: - dcount = dcount +1 -# print(p,'dominated by',q) + dcount = dcount + 1 + # print(p,'dominated by',q) if dcount == 0: -# print(p,'is on the front') + # print(p,'is on the front') front.append(i) -# f_obj1 = [obj1[f] for f in front] + # f_obj1 = [obj1[f] for f in front] f_obj2 = [obj2[f] for f in front] -# s1 = np.argsort(np.array(f_obj1)) + # s1 = np.argsort(np.array(f_obj1)) s2 = np.argsort(np.array(f_obj2)) -# front = [front[s] for s in s1] + # front = [front[s] for s in s1] front = [front[s] for s in s2] return front diff --git a/postprocessing/pick_best_params.py b/postprocessing/pick_best_params.py index e3a98c9ad..e6d7c5396 100644 --- a/postprocessing/pick_best_params.py +++ b/postprocessing/pick_best_params.py @@ -2,50 +2,48 @@ # coding: utf-8 # # saved tuned models -# -# Go thru PMLB results and pick the most common params in the best estimators. +# +# Go thru PMLB results and pick the most common params in the best estimators. # Use these parameter settings for the Feynman and Strogatz models. + import pandas as pd -import json -import numpy as np -import os symbolic_algs = [ - 'AFP', - 'AFP_FE', - 'AIFeynman', - 'BSR', - 'DSR', - 'FFX', - 'FEAT', - 'EPLEX', - 'GP-GOMEA', - 'gplearn', - 'ITEA', - 'MRGP', - 'Operon', - 'SBP-GP', + "AFP", + "AFP_FE", + "AIFeynman", + "BSR", + "DSR", + "FFX", + "FEAT", + "EPLEX", + "GP-GOMEA", + "gplearn", + "ITEA", + "MRGP", + "Operon", + "SBP-GP", ] sr_filenames = [ - 'AFPRegressor', - 'FE_AFPRegressor', - 'AIFeynman', - 'BSRRegressor', - 'DSRRegressor', - 'FFXRegressor', - 'FEATRegressor', - 'EPLEXRegressor', - 'GPGOMEARegressor', - 'gplearn', - 'ITEARegressor', - 'MRGPRegressor', - 'OperonRegressor', - 'sembackpropgp' - ] -sr_name_to_filename = {k:v for k,v in zip(symbolic_algs, sr_filenames)} + "AFPRegressor", + "FE_AFPRegressor", + "AIFeynman", + "BSRRegressor", + "DSRRegressor", + "FFXRegressor", + "FEATRegressor", + "EPLEXRegressor", + "GPGOMEARegressor", + "gplearn", + "ITEARegressor", + "MRGPRegressor", + "OperonRegressor", + "sembackpropgp", +] +sr_name_to_filename = {k: v for k, v in zip(symbolic_algs, sr_filenames)} # read data from feather -df_results = pd.read_feather('../results/black-box_results.feather') +df_results = pd.read_feather("../results/black-box_results.feather") # keep only symbolic regressors df_results = df_results.loc[df_results.algorithm.isin(symbolic_algs)] @@ -63,12 +61,12 @@ # df_results.params.apply(str) best_params = [] -for alg, dfg in df_results.groupby('algorithm'): - counts = dfg['params_str'].value_counts() - print(alg, 'mode of params:',counts.index[0]) +for alg, dfg in df_results.groupby("algorithm"): + counts = dfg["params_str"].value_counts() + print(alg, "mode of params:", counts.index[0]) best_params.append([alg, counts.index[0]]) -# best_params +# best_params # # write tuned model scripts for each algorithm @@ -76,9 +74,8 @@ # In[5]: -import os for alg, bp in best_params: -# os.system('cp ../experiment/methods/{ALG}.py ../experiment/methods/tuned/{ALG}.py'.format(ALG=alg)) + # os.system('cp ../experiment/methods/{ALG}.py ../experiment/methods/tuned/{ALG}.py'.format(ALG=alg)) param_file = sr_name_to_filename[alg] - with open(f'../experiment/methods/tuned/params/_{param_file}.py','w') as f: - f.write('params = {}\n'.format(bp)) + with open(f"../experiment/methods/tuned/params/_{param_file}.py", "w") as f: + f.write("params = {}\n".format(bp)) diff --git a/postprocessing/pmlb_plots.ipynb b/postprocessing/pmlb_plots.ipynb index 0f524135a..fe006ef21 100644 --- a/postprocessing/pmlb_plots.ipynb +++ b/postprocessing/pmlb_plots.ipynb @@ -6,26 +6,32 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", "from glob import glob\n", - "import seaborn as sns\n", + "\n", "import matplotlib\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", "%matplotlib inline\n", - "matplotlib.rc('pdf', fonttype=42)\n", + "matplotlib.rc(\"pdf\", fonttype=42)\n", "\n", - "datadir = '../../../pmlb/datasets/'\n", + "datadir = \"../../../pmlb/datasets/\"\n", "frames = []\n", - "for f in glob(datadir+'/*/*.tsv.gz'):\n", - " df = pd.read_csv(f,sep='\\t') \n", - " group = 'feynman' if 'feynman' in f else 'strogatz' if 'strogatz' in f else 'black-box'\n", - " frames.append(dict(\n", - " name=f.split('/')[-1][:-7],\n", - " nsamples = df.shape[0],\n", - " nfeatures = df.shape[1],\n", - " npoints = df.shape[0]*df.shape[1],\n", - " Group=group\n", - " ))\n", - " \n", + "for f in glob(datadir + \"/*/*.tsv.gz\"):\n", + " df = pd.read_csv(f, sep=\"\\t\")\n", + " group = (\n", + " \"feynman\" if \"feynman\" in f else \"strogatz\" if \"strogatz\" in f else \"black-box\"\n", + " )\n", + " frames.append(\n", + " dict(\n", + " name=f.split(\"/\")[-1][:-7],\n", + " nsamples=df.shape[0],\n", + " nfeatures=df.shape[1],\n", + " npoints=df.shape[0] * df.shape[1],\n", + " Group=group,\n", + " )\n", + " )\n", + "\n", "df = pd.DataFrame.from_records(frames)\n", "df" ] @@ -37,25 +43,25 @@ "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "sns.set_style('whitegrid')\n", + "\n", + "sns.set_style(\"whitegrid\")\n", "sns.despine(left=True, bottom=True)\n", "## PMLB dataset sizes\n", "g = sns.scatterplot(\n", " data=df,\n", - " x='nsamples',\n", - " y='nfeatures',\n", - " hue='Group',\n", + " x=\"nsamples\",\n", + " y=\"nfeatures\",\n", + " hue=\"Group\",\n", " alpha=0.7,\n", - " s=100, \n", + " s=100,\n", ")\n", "ax = plt.gca()\n", - "plt.legend(loc='upper left')\n", - "ax.set_xscale('log')\n", - "ax.set_yscale('log')\n", - "plt.xlabel('No. of Samples')\n", - "plt.ylabel('No. of Features')\n", - "plt.savefig('figs/pmlb_size.png',dpi=400, bbox_inches='tight')" + "plt.legend(loc=\"upper left\")\n", + "ax.set_xscale(\"log\")\n", + "ax.set_yscale(\"log\")\n", + "plt.xlabel(\"No. of Samples\")\n", + "plt.ylabel(\"No. of Features\")\n", + "plt.savefig(\"figs/pmlb_size.png\", dpi=400, bbox_inches=\"tight\")" ] } ], diff --git a/postprocessing/statistical_comparisons.ipynb b/postprocessing/statistical_comparisons.ipynb index 0c244cef3..b73da7673 100644 --- a/postprocessing/statistical_comparisons.ipynb +++ b/postprocessing/statistical_comparisons.ipynb @@ -15,26 +15,30 @@ "metadata": {}, "outputs": [], "source": [ - "from scipy.stats import wilcoxon \n", - "import pandas as pd\n", "import itertools as it\n", "import os\n", + "\n", + "import pandas as pd\n", "import seaborn as sns\n", - "sns.set(font_scale=1.5, style='white')\n", + "from scipy.stats import wilcoxon\n", + "\n", + "sns.set(font_scale=1.5, style=\"white\")\n", "\n", - "figdir = 'figs/stats/'\n", - "rdir = '../results/'\n", + "figdir = \"figs/stats/\"\n", + "rdir = \"../results/\"\n", "\n", - "print('figdir:',figdir)\n", - "def save(h=None,name='tmp'):\n", - " name = name.strip().replace(' ','-').replace('%','pct')\n", - " if h == None:\n", + "print(\"figdir:\", figdir)\n", + "\n", + "\n", + "def save(h=None, name=\"tmp\"):\n", + " name = name.strip().replace(\" \", \"-\").replace(\"%\", \"pct\")\n", + " if h is None:\n", " h = plt.gcf()\n", " h.tight_layout()\n", - " print('saving',figdir+'/'+name+'.png')\n", + " print(\"saving\", figdir + \"/\" + name + \".png\")\n", " if not os.path.exists(figdir):\n", " os.makedirs(figdir)\n", - " plt.savefig(figdir+'/'+name+'.png', bbox_inches='tight')" + " plt.savefig(figdir + \"/\" + name + \".png\", bbox_inches=\"tight\")" ] }, { @@ -43,79 +47,73 @@ "metadata": {}, "outputs": [], "source": [ - "import pdb\n", "import numpy as np\n", "\n", + "\n", "def pairwise_pval(df, metric, alg1, alg2):\n", - " df = df.loc[df.algorithm.isin([alg1, alg2]),:].copy()\n", - " x = df.loc[df.algorithm==alg1, metric].values \n", - " y = df.loc[df.algorithm==alg2, metric].values \n", - " eff_size = np.abs(np.nanmedian(x)/np.nanmedian(y))\n", - " if metric.endswith('norm'):\n", - " rmetric = metric.replace('norm','rank')\n", + " df = df.loc[df.algorithm.isin([alg1, alg2]), :].copy()\n", + " x = df.loc[df.algorithm == alg1, metric].values\n", + " y = df.loc[df.algorithm == alg2, metric].values\n", + " eff_size = np.abs(np.nanmedian(x) / np.nanmedian(y))\n", + " if metric.endswith(\"norm\"):\n", + " rmetric = metric.replace(\"norm\", \"rank\")\n", " else:\n", - " rmetric = metric+'_rank'\n", + " rmetric = metric + \"_rank\"\n", " if rmetric not in df.columns:\n", " rmetric = metric\n", - " \n", - " x_rank = df.loc[df.algorithm==alg1, rmetric].values \n", - " y_rank = df.loc[df.algorithm==alg2, rmetric].values \n", - "# pdb.set_trace()\n", + "\n", + " df.loc[df.algorithm == alg1, rmetric].values\n", + " df.loc[df.algorithm == alg2, rmetric].values\n", + " # pdb.set_trace()\n", " if len(x) != len(y):\n", " print(alg1, len(x))\n", " print(alg2, len(y))\n", " assert len(x) == len(y)\n", - "# w, p = mannwhitneyu(x, y)\n", - " if all(y==0) and all(x==0):\n", + " # w, p = mannwhitneyu(x, y)\n", + " if all(y == 0) and all(x == 0):\n", " return 1, 1\n", - " \n", + "\n", " w, p = wilcoxon(x, y)\n", " return p, eff_size\n", "\n", + "\n", "# return pstr, eff_size_str\n", "\n", + "\n", "def signif(pval, alpha, eff):\n", - " pstr = '{:1.2g}'.format(pval)\n", - " eff_size_str = '{:1.1f}X'.format(eff)\n", - " if pval == '-': return pval\n", + " pstr = \"{:1.2g}\".format(pval)\n", + " eff_size_str = \"{:1.1f}X\".format(eff)\n", + " if pval == \"-\":\n", + " return pval\n", " if float(pval) < alpha:\n", - " return 'textbf{'+pstr+'}', 'textbf{'+eff_size_str+'}'\n", - "# return pval+'*'\n", + " return \"textbf{\" + pstr + \"}\", \"textbf{\" + eff_size_str + \"}\"\n", + " # return pval+'*'\n", " else:\n", " return pstr, eff_size_str\n", - " \n", + "\n", + "\n", "def get_pval_df(df, metric, all_algs):\n", " df = df.copy()\n", - " n=0\n", + " n = 0\n", " pvals = []\n", " for alg1, alg2 in it.combinations(all_algs, 2):\n", " pval, eff_size = pairwise_pval(df, metric, alg1, alg2)\n", - " pvals.append(dict(\n", - " alg1 = alg1,\n", - " alg2 = alg2,\n", - " eff_size = eff_size,\n", - " pval = pval\n", - " ))\n", + " pvals.append(dict(alg1=alg1, alg2=alg2, eff_size=eff_size, pval=pval))\n", " n += 1\n", - " c_alpha = alpha/n\n", + " c_alpha = alpha / n\n", "\n", - "\n", - " print('n:',n,'c_alpha:',c_alpha) \n", + " print(\"n:\", n, \"c_alpha:\", c_alpha)\n", " df_pvals = pd.DataFrame.from_records(pvals)\n", " # df_pvals['pval_thresh'] = pd.cut(x=df_pvals['pval'], bins = [0, c_alpha, 1])\n", - " df_pvals['pval_thresh'] = df_pvals['pval'].apply(lambda x: bin_pval(x, c_alpha))\n", + " df_pvals[\"pval_thresh\"] = df_pvals[\"pval\"].apply(lambda x: bin_pval(x, c_alpha))\n", "\n", " # significance\n", - " df_pvals.loc[:,'pval_bold'] = df_pvals.apply(lambda x: signif(x['pval'], \n", - " c_alpha,\n", - " x['eff_size'])[0],\n", - " axis=1\n", - " )\n", - " df_pvals.loc[:,'eff_size_bold'] = df_pvals.apply(lambda x: signif(x['pval'], \n", - " c_alpha,\n", - " x['eff_size'])[1],\n", - " axis=1\n", - " )\n", + " df_pvals.loc[:, \"pval_bold\"] = df_pvals.apply(\n", + " lambda x: signif(x[\"pval\"], c_alpha, x[\"eff_size\"])[0], axis=1\n", + " )\n", + " df_pvals.loc[:, \"eff_size_bold\"] = df_pvals.apply(\n", + " lambda x: signif(x[\"pval\"], c_alpha, x[\"eff_size\"])[1], axis=1\n", + " )\n", " return df_pvals, c_alpha" ] }, @@ -125,66 +123,70 @@ "metadata": {}, "outputs": [], "source": [ - "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", "%matplotlib inline\n", - "from matplotlib.colors import LogNorm\n", + "\n", "\n", "def bin_pval(x, c_alpha):\n", "\n", - " for stars,level in zip([4,3,2,1],[1e-3, 1e-2, 1e-1, 1]):\n", - " if x < level*c_alpha: \n", - " return stars #level #*c_alpha\n", + " for stars, level in zip([4, 3, 2, 1], [1e-3, 1e-2, 1e-1, 1]):\n", + " if x < level * c_alpha:\n", + " return stars # level #*c_alpha\n", " return 0\n", "\n", + "\n", "def pval_heatmap(df, metric, problem, algs):\n", " df = df.copy()\n", - " n = 0 \n", - " pvals = []\n", "\n", " df_pvals, c_alpha = get_pval_df(df, metric, algs)\n", - " \n", + "\n", " # pd.cut(x=df_pvals['pval'], bins = [0, c_alpha, 1])\n", - " tbl = df_pvals.set_index(['alg1','alg2'])['pval_thresh'].unstack().transpose() #.fillna('-')\n", + " tbl = (\n", + " df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval_thresh\"].unstack().transpose()\n", + " ) # .fillna('-')\n", "\n", " mask = np.zeros_like(tbl, dtype=np.bool)\n", - " mask[np.triu_indices_from(mask,k=1)] = True\n", - "\n", - " h = plt.figure(figsize=(10,10))\n", - "\n", - " cmap = sns.color_palette('flare',n_colors=5)\n", - " cmap[0] = [.9, .9, .9]\n", - " ax = sns.heatmap(tbl, \n", - " linewidth=0.25,\n", - " mask=mask,\n", - " square=True,\n", - " cbar_kws=dict(\n", - " ticks = [0.4, 1.2, 2.0, 2.8, 3.6],\n", - " shrink=0.6,\n", - " ),\n", - " cmap=cmap,\n", - "\n", - " )\n", + " mask[np.triu_indices_from(mask, k=1)] = True\n", + "\n", + " h = plt.figure(figsize=(10, 10))\n", + "\n", + " cmap = sns.color_palette(\"flare\", n_colors=5)\n", + " cmap[0] = [0.9, 0.9, 0.9]\n", + " sns.heatmap(\n", + " tbl,\n", + " linewidth=0.25,\n", + " mask=mask,\n", + " square=True,\n", + " cbar_kws=dict(\n", + " ticks=[0.4, 1.2, 2.0, 2.8, 3.6],\n", + " shrink=0.6,\n", + " ),\n", + " cmap=cmap,\n", + " )\n", " cax = h.axes[-1]\n", " cbar_labels = [\n", - " 'no significance',\n", - " '$p<\\\\alpha$',\n", - " '$p<$1e-1$\\cdot \\\\alpha$',\n", - " '$p<$1e-2$\\cdot \\\\alpha$',\n", - " '$p<$1e-3$\\cdot \\\\alpha$',\n", - " ]\n", + " \"no significance\",\n", + " \"$p<\\\\alpha$\",\n", + " \"$p<$1e-1$\\cdot \\\\alpha$\",\n", + " \"$p<$1e-2$\\cdot \\\\alpha$\",\n", + " \"$p<$1e-3$\\cdot \\\\alpha$\",\n", + " ]\n", " cax.set_yticklabels(cbar_labels)\n", - " nice_metric = metric.replace('%','pct').replace('_',' ').replace('R2','$$R^2$$').title()\n", - " plt.title(('Wilcoxon signed-rank test, '\n", - " +nice_metric\n", - " +', $\\\\alpha =$ {:1.1e}').format(c_alpha)\n", - " )\n", - " plt.xlabel('')\n", - " plt.ylabel('')\n", - " savename = ('Pairwise comparison of '\n", - " +nice_metric\n", - " +' on '\n", - " +problem).replace(' ','_')\n", + " nice_metric = (\n", + " metric.replace(\"%\", \"pct\").replace(\"_\", \" \").replace(\"R2\", \"$$R^2$$\").title()\n", + " )\n", + " plt.title(\n", + " (\"Wilcoxon signed-rank test, \" + nice_metric + \", $\\\\alpha =$ {:1.1e}\").format(\n", + " c_alpha\n", + " )\n", + " )\n", + " plt.xlabel(\"\")\n", + " plt.ylabel(\"\")\n", + " savename = (\"Pairwise comparison of \" + nice_metric + \" on \" + problem).replace(\n", + " \" \", \"_\"\n", + " )\n", " save(h, savename)" ] }, @@ -208,17 +210,17 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum = pd.read_csv(rdir+'/symbolic_dataset_results_sum.csv.gz')\n", + "df_sum = pd.read_csv(rdir + \"/symbolic_dataset_results_sum.csv.gz\")\n", "algs = df_sum.algorithm.unique()\n", "datasets = df_sum.dataset.unique()\n", "noises = df_sum.target_noise.unique()\n", "\n", "df_sum\n", "cols = df_sum.columns\n", - "for col in [c for c in cols if 'rank' in c]:\n", - " df_sum.loc[:,col] = df_sum[col].fillna(len(algs)+1)\n", - "for col in [c for c in cols if 'solution_rate' in c]:\n", - " df_sum.loc[:,col] = df_sum[col].fillna(0.0)" + "for col in [c for c in cols if \"rank\" in c]:\n", + " df_sum.loc[:, col] = df_sum[col].fillna(len(algs) + 1)\n", + "for col in [c for c in cols if \"solution_rate\" in c]:\n", + " df_sum.loc[:, col] = df_sum[col].fillna(0.0)" ] }, { @@ -229,25 +231,23 @@ "source": [ "cols = df_sum.columns\n", "filler_frame = {\n", - " 'r2_test' : 0.0,\n", - " 'solution_rate_(%)' : 0.0,\n", + " \"r2_test\": 0.0,\n", + " \"solution_rate_(%)\": 0.0,\n", "}\n", "fill_frames = []\n", "\n", "for dataset in datasets:\n", " for noise in noises:\n", " for alg in algs:\n", - " df = df_sum.loc[df_sum.dataset == dataset].loc[df_sum.target_noise==noise]\n", + " df = df_sum.loc[df_sum.dataset == dataset].loc[df_sum.target_noise == noise]\n", " if alg not in df.algorithm.unique():\n", " new_fill = filler_frame.copy()\n", - " new_fill.update({\n", - " 'algorithm':alg,\n", - " 'dataset':dataset,\n", - " 'target_noise':noise\n", - " })\n", + " new_fill.update(\n", + " {\"algorithm\": alg, \"dataset\": dataset, \"target_noise\": noise}\n", + " )\n", " fill_frames.append(new_fill)\n", - "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys()) \n", - "df_sum_filled = df_sum.append(df_fill)\n" + "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys())\n", + "df_sum_filled = df_sum.append(df_fill)" ] }, { @@ -266,14 +266,16 @@ "alpha = 0.05\n", "SR_pvals = {}\n", "\n", - "for metric in ['r2_test','symbolic_solution_rate','simplified_complexity']:\n", - " for target_noise, dfg in df_sum_filled.groupby('target_noise'):\n", - " print(metric, 'noise=',target_noise)\n", - " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs) \n", + "for metric in [\"r2_test\", \"symbolic_solution_rate\", \"simplified_complexity\"]:\n", + " for target_noise, dfg in df_sum_filled.groupby(\"target_noise\"):\n", + " print(metric, \"noise=\", target_noise)\n", + " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs)\n", "\n", - " pval_tbl = df_pvals.set_index(['alg1','alg2'])['pval_bold'].unstack().fillna('-')\n", - " disp_tbl = df_pvals.set_index(['alg1','alg2'])['pval'].unstack().fillna('-')\n", - " SR_pvals[metric] = {target_noise:disp_tbl}\n", + " pval_tbl = (\n", + " df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval_bold\"].unstack().fillna(\"-\")\n", + " )\n", + " disp_tbl = df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval\"].unstack().fillna(\"-\")\n", + " SR_pvals[metric] = {target_noise: disp_tbl}\n", " display(disp_tbl)\n", "\n", " pval_tbl" @@ -292,11 +294,10 @@ "metadata": {}, "outputs": [], "source": [ - "for target_noise, dfg in df_sum_filled.groupby('target_noise'):\n", - " for metric in ['r2_test','symbolic_solution_rate','simplified_complexity']:\n", - " name = 'symbolic problems target_noise='+str(target_noise)\n", - " pval_heatmap(dfg, metric, name, algs)\n", - " " + "for target_noise, dfg in df_sum_filled.groupby(\"target_noise\"):\n", + " for metric in [\"r2_test\", \"symbolic_solution_rate\", \"simplified_complexity\"]:\n", + " name = \"symbolic problems target_noise=\" + str(target_noise)\n", + " pval_heatmap(dfg, metric, name, algs)" ] }, { @@ -319,7 +320,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_sum = pd.read_csv(rdir+'/results_black-box_summary.csv.gz')\n", + "df_sum = pd.read_csv(rdir + \"/results_black-box_summary.csv.gz\")\n", "algs = df_sum.algorithm.unique()\n", "datasets = df_sum.dataset.unique()" ] @@ -333,8 +334,8 @@ "# if a result is missing, assign it the worst ranking\n", "# cols = df_sum.columns\n", "filler_frame = {\n", - " 'r2_test_rank' : len(algs)+1,\n", - " 'model_size_rank' : len(algs)+1,\n", + " \"r2_test_rank\": len(algs) + 1,\n", + " \"model_size_rank\": len(algs) + 1,\n", "}\n", "fill_frames = []\n", "j = 0\n", @@ -342,19 +343,21 @@ " for alg in algs:\n", " df = df_sum.loc[df_sum.dataset == dataset]\n", " if alg not in df.algorithm.unique():\n", - " print('filling',alg,'missing for',dataset)\n", - "# for col in cols:\n", + " print(\"filling\", alg, \"missing for\", dataset)\n", + " # for col in cols:\n", " new_fill = filler_frame.copy()\n", - " new_fill.update({\n", - " 'algorithm':alg,\n", - " 'dataset':dataset,\n", - " })\n", + " new_fill.update(\n", + " {\n", + " \"algorithm\": alg,\n", + " \"dataset\": dataset,\n", + " }\n", + " )\n", " print(new_fill)\n", " fill_frames.append(new_fill)\n", " j += 1\n", - " \n", - "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys()) \n", - "df_sum_filled = df_sum.append(df_fill)\n" + "\n", + "df_fill = pd.DataFrame.from_records(fill_frames, columns=fill_frames[0].keys())\n", + "df_sum_filled = df_sum.append(df_fill)" ] }, { @@ -374,16 +377,16 @@ "alpha = 0.05\n", "BB_pvals = {}\n", "\n", - "for metric in ['r2_test','model_size']:\n", + "for metric in [\"r2_test\", \"model_size\"]:\n", " pvals = []\n", "\n", - " n = 0 \n", - " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs) \n", - " pval_tbl = df_pvals.set_index(['alg1','alg2'])['pval'].unstack() \n", - " BB_pvals[metric] = pval_tbl.copy() \n", - " pval_tbl = pval_tbl.fillna('-')\n", + " n = 0\n", + " df_pvals, c_alpha = get_pval_df(df_sum_filled, metric, algs)\n", + " pval_tbl = df_pvals.set_index([\"alg1\", \"alg2\"])[\"pval\"].unstack()\n", + " BB_pvals[metric] = pval_tbl.copy()\n", + " pval_tbl = pval_tbl.fillna(\"-\")\n", " display(pval_tbl)\n", - " \n", + "\n", " pval_tbl" ] }, @@ -393,8 +396,8 @@ "metadata": {}, "outputs": [], "source": [ - "for metric in ['r2_test','model_size']:\n", - " name = 'black-box problems'\n", + "for metric in [\"r2_test\", \"model_size\"]:\n", + " name = \"black-box problems\"\n", " pval_heatmap(df_sum_filled, metric, name, algs)" ] }, @@ -412,13 +415,13 @@ "outputs": [], "source": [ "# first fill the matrices so we can look at either pair\n", - "for k,v in BB_pvals.items():\n", + "for k, v in BB_pvals.items():\n", " algs = v.columns\n", - " for alg1, alg2 in it.combinations(algs,2):\n", - " if np.isnan(BB_pvals[k].loc[alg1,alg2]): \n", - " BB_pvals[k].loc[alg1,alg2] = BB_pvals[k].loc[alg2,alg1] \n", + " for alg1, alg2 in it.combinations(algs, 2):\n", + " if np.isnan(BB_pvals[k].loc[alg1, alg2]):\n", + " BB_pvals[k].loc[alg1, alg2] = BB_pvals[k].loc[alg2, alg1]\n", " else:\n", - " BB_pvals[k].loc[alg2,alg1] = BB_pvals[k].loc[alg1,alg2] " + " BB_pvals[k].loc[alg2, alg1] = BB_pvals[k].loc[alg1, alg2]" ] }, { @@ -434,7 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test']['Operon'].max() " + "BB_pvals[\"r2_test\"][\"Operon\"].max()" ] }, { @@ -450,7 +453,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['model_size'].loc['FEAT','SBP-GP']" + "BB_pvals[\"model_size\"].loc[\"FEAT\", \"SBP-GP\"]" ] }, { @@ -466,7 +469,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc['XGB',['Operon','SBP-GP']].max() #.replace('-','0').astype(float).max() #.round(2)" + "BB_pvals[\"r2_test\"].loc[\n", + " \"XGB\", [\"Operon\", \"SBP-GP\"]\n", + "].max() # .replace('-','0').astype(float).max() #.round(2)" ] }, { @@ -482,7 +487,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc['LGBM',['Operon','FEAT','SBP-GP']].max() #.replace('-','0').astype(float).max() #.round(2)" + "BB_pvals[\"r2_test\"].loc[\n", + " \"LGBM\", [\"Operon\", \"FEAT\", \"SBP-GP\"]\n", + "].max() # .replace('-','0').astype(float).max() #.round(2)" ] }, { @@ -499,11 +506,12 @@ "outputs": [], "source": [ "import itertools as it\n", + "\n", "import numpy as np\n", "\n", - "gps = ['Operon','FEAT','EPLEX','ITEA','GP-GOMEA']\n", - "trees = ['LGBM','XGB','RandomForest','AdaBoost']\n", - "BB_pvals['model_size'].loc[gps,trees].max().max()\n" + "gps = [\"Operon\", \"FEAT\", \"EPLEX\", \"ITEA\", \"GP-GOMEA\"]\n", + "trees = [\"LGBM\", \"XGB\", \"RandomForest\", \"AdaBoost\"]\n", + "BB_pvals[\"model_size\"].loc[gps, trees].max().max()" ] }, { @@ -512,7 +520,9 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc[['DSR','BSR','FFX','AIFeynman'],['DSR','BSR','FFX','AIFeynman']]" + "BB_pvals[\"r2_test\"].loc[\n", + " [\"DSR\", \"BSR\", \"FFX\", \"AIFeynman\"], [\"DSR\", \"BSR\", \"FFX\", \"AIFeynman\"]\n", + "]" ] }, { @@ -521,7 +531,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'].loc[['DSR','FFX'],['BSR','AIFeynman']]" + "BB_pvals[\"r2_test\"].loc[[\"DSR\", \"FFX\"], [\"BSR\", \"AIFeynman\"]]" ] }, { @@ -530,7 +540,7 @@ "metadata": {}, "outputs": [], "source": [ - "BB_pvals['r2_test'][['BSR','AIFeynman']]" + "BB_pvals[\"r2_test\"][[\"BSR\", \"AIFeynman\"]]" ] } ], diff --git a/postprocessing/symbolic_utils.py b/postprocessing/symbolic_utils.py index 4a1e59bad..66407dde7 100644 --- a/postprocessing/symbolic_utils.py +++ b/postprocessing/symbolic_utils.py @@ -1,28 +1,21 @@ -import pdb -import pandas as pd -from yaml import load, Loader -import sympy # from sympy import * -from sympy import Symbol, simplify, factor, Float, preorder_traversal, Integer -from sympy.parsing.sympy_parser import parse_expr -import re -import ast +from sympy import Float, Integer, preorder_traversal def complexity(expr): - c=0 + c = 0 for arg in preorder_traversal(expr): c += 1 return c - + + def round_floats(ex1): ex2 = ex1 for a in preorder_traversal(ex1): if isinstance(a, Float): if abs(a) < 0.0001: - ex2 = ex2.subs(a,Integer(0)) + ex2 = ex2.subs(a, Integer(0)) else: ex2 = ex2.subs(a, round(a, 3)) return ex2 - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..584509379 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +required-version = "==0.17.0" +target-version = "py37" +line-length = 88 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] + +[tool.mypy] +python_version = "3.10" +check_untyped_defs = true +ignore_missing_imports = true +follow_imports = "skip" +no_site_packages = true +warn_unused_ignores = true diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..907de0e53 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +# Ruff and mypy versions are pinned in .pre-commit-config.yaml. +pre-commit==4.6.2 diff --git a/scripts/check_method_layout.py b/scripts/check_method_layout.py index affb53578..9b84307cb 100644 --- a/scripts/check_method_layout.py +++ b/scripts/check_method_layout.py @@ -129,7 +129,9 @@ def check_algorithm(name: str) -> None: if parsed is not None or name not in METADATA_GRANDFATHERED: missing = [k for k in REQUIRED_METADATA_KEYS if k not in (parsed or {})] if missing and name not in METADATA_GRANDFATHERED: - error(name, f"{metadata} is missing required key(s): {', '.join(missing)}") + error( + name, f"{metadata} is missing required key(s): {', '.join(missing)}" + ) elif missing: warn(name, f"{metadata} is incomplete (missing {', '.join(missing)})") @@ -157,9 +159,14 @@ def check_algorithm(name: str) -> None: error(name, f"{regressor} does not define `{required}` at module level") if os.path.isfile(os.path.join(alg_path, "regressor.py")): - warn(name, f"{alg_path}/regressor.py is ignored; the harness imports {regressor}") + warn( + name, f"{alg_path}/regressor.py is ignored; the harness imports {regressor}" + ) if os.path.isfile(os.path.join(method_path, "metadata.yml")): - warn(name, f"{method_path}/metadata.yml is ignored; metadata.yml belongs in {alg_path}/") + warn( + name, + f"{method_path}/metadata.yml is ignored; metadata.yml belongs in {alg_path}/", + ) for stray in ("install.sh", "environment.yml", "requirements.txt", "Dockerfile"): if os.path.isfile(os.path.join(method_path, stray)): error( @@ -179,10 +186,14 @@ def check_orphan_method(name: str) -> None: def main() -> int: if not os.path.isdir(ALG_DIR) or not os.path.isdir(METHOD_DIR): - print(f"error: run this from the repository root (missing {ALG_DIR}/ or {METHOD_DIR}/)") + print( + f"error: run this from the repository root (missing {ALG_DIR}/ or {METHOD_DIR}/)" + ) return 1 - algorithms = sorted(d for d in os.listdir(ALG_DIR) if os.path.isdir(os.path.join(ALG_DIR, d))) + algorithms = sorted( + d for d in os.listdir(ALG_DIR) if os.path.isdir(os.path.join(ALG_DIR, d)) + ) methods = sorted( d for d in os.listdir(METHOD_DIR) @@ -199,13 +210,23 @@ def main() -> int: print(f"checked {len(algorithms)} algorithms and {len(methods)} method directories") for line in warnings: - print(f"::warning::{line}" if os.environ.get("GITHUB_ACTIONS") else f"warning: {line}") + print( + f"::warning::{line}" + if os.environ.get("GITHUB_ACTIONS") + else f"warning: {line}" + ) if errors: print() for line in errors: - print(f"::error::{line}" if os.environ.get("GITHUB_ACTIONS") else f"error: {line}") - print(f"\n{len(errors)} problem(s) found. See CONTRIBUTING.md for the expected layout.") + print( + f"::error::{line}" + if os.environ.get("GITHUB_ACTIONS") + else f"error: {line}" + ) + print( + f"\n{len(errors)} problem(s) found. See CONTRIBUTING.md for the expected layout." + ) return 1 print("method layout OK") diff --git a/scripts/typecheck.py b/scripts/typecheck.py new file mode 100644 index 000000000..6458018fb --- /dev/null +++ b/scripts/typecheck.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Type-check every Python file, including importlib-loaded hyphenated methods. + +Mypy's CLI rejects package names such as gp-elite and ps-tree. Supply explicit, +unique module names so these adapters are checked alongside the rest of the +repository without renaming the methods or excluding their source code. +""" + +import subprocess +from pathlib import Path + +from mypy.build import BuildSource, build +from mypy.errors import CompileError +from mypy.main import process_options + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + # Include tracked and new source files anywhere in the repo, honoring gitignore. + listing = subprocess.check_output( + [ + "git", + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + "*.py", + ], + cwd=root, + ) + paths = sorted({root / name.decode() for name in listing.split(b"\0") if name}) + paths = [path for path in paths if path.is_file()] + + sources = [] + for path in paths: + parts = list(path.relative_to(root).with_suffix("").parts) + if parts[-1] == "__init__": + parts.pop() + module = ".".join(part.replace("-", "_") for part in parts) or "srbench" + sources.append(BuildSource(str(path), module, None)) + + _, options = process_options( + ["--config-file", str(root / "pyproject.toml")], require_targets=False + ) + try: + result = build(sources=sources, options=options) + messages = result.errors + except CompileError as exc: + messages = exc.messages + for message in messages: + print(message) + if messages: + return 1 + print(f"Success: no issues found in {len(sources)} source files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())