diff --git a/.github/workflows/build-ete4-windows-wheels.yml b/.github/workflows/build-ete4-windows-wheels.yml new file mode 100644 index 0000000..f77ad39 --- /dev/null +++ b/.github/workflows/build-ete4-windows-wheels.yml @@ -0,0 +1,80 @@ +name: build-ete4-windows-wheels + +# Build Windows (win_amd64) wheels for ete4, which upstream does NOT ship for +# Windows (PyPI has an sdist only; conda-forge has no win-64 build). ete4 is a +# hard FunVIP dependency, so without a Windows wheel FunVIP cannot be pip/conda +# installed on Windows at all. This applies the small etetoolkit/ete PR #783 +# path-separator fix (tools/ete4-windows/ete4-windows.patch) and builds wheels +# with cibuildwheel. +# +# Trigger manually: Actions tab -> "build-ete4-windows-wheels" -> "Run workflow". +# The built wheels are uploaded as a downloadable artifact. +# +# ete4 is GPL-3.0-or-later; if you redistribute these wheels, also publish the +# corresponding (patched) source and keep ete4's LICENSE. See +# tools/ete4-windows/README.md. + +on: + workflow_dispatch: + inputs: + ete4_version: + description: "ete4 version to build Windows wheels for" + required: true + default: "4.4.0" + +jobs: + build-windows-wheels: + name: ete4 ${{ github.event.inputs.ete4_version }} win_amd64 + runs-on: windows-latest + steps: + - name: Checkout FunVIP (for the patch) + uses: actions/checkout@v4 + + - name: Set up host Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Download and unpack the ete4 source + shell: bash + run: | + set -euxo pipefail + VER="${{ github.event.inputs.ete4_version }}" + python -m pip install --upgrade pip + python -m pip download "ete4==${VER}" --no-deps --no-binary :all: -d sdist + tar -xzf "sdist/ete4-${VER}.tar.gz" + echo "ETE4_SRC=ete4-${VER}" >> "$GITHUB_ENV" + + - name: Apply the Windows build patch (etetoolkit/ete PR #783) + shell: bash + run: | + set -euxo pipefail + cd "$ETE4_SRC" + git init -q # give `git apply` a worktree to operate in + git apply --verbose "../tools/ete4-windows/ete4-windows.patch" + echo "--- patched lines ---" + grep -n "replace(sep" setup.py + grep -n "expanduser" ete4/config.py + + - name: Build wheels with cibuildwheel + shell: bash + env: + # FunVIP-supported CPythons; narrow this to speed up a first "does it + # compile?" check (e.g. just cp312-win_amd64). + CIBW_BUILD: "cp310-win_amd64 cp311-win_amd64 cp312-win_amd64 cp313-win_amd64" + CIBW_BUILD_VERBOSITY: "1" + # Import test loads the compiled Cython .pyd -- proves the wheel actually + # works on Windows, not merely that it compiled. Single-quoted Python + # strings keep this robust under Windows cmd. + CIBW_TEST_COMMAND: python -c "import ete4; from ete4 import Tree; print('ete4', ete4.__version__)" + run: | + set -euxo pipefail + python -m pip install cibuildwheel + python -m cibuildwheel --output-dir wheelhouse "$ETE4_SRC" + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ete4-${{ github.event.inputs.ete4_version }}-windows-wheels + path: wheelhouse/*.whl + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6de8f0f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,96 @@ +name: CI + +on: + push: + branches: [main, development] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Fast, broad matrix. The unit tests import only lightweight deps (pandas, + # numpy, matplotlib, biopython, pyyaml, psutil, unidecode) -- NOT ete4 -- so + # they run across Python versions and OSes without ete4's Cython build hurdle. + unit: + name: unit / py${{ matrix.python }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install unit-test dependencies + run: | + python -m pip install --upgrade pip + pip install pytest "pandas>=2.2.2,<4.0.0" "numpy<2" matplotlib biopython pyyaml psutil unidecode tabulate + - name: Run unit tests + run: pytest -q + env: + PYTHONPATH: . + + # One combo that exercises the real Linux install path: a fresh conda env, then a + # pip install of the package. ete4 is pulled and built from PyPI by pip (not conda: + # conda-forge's ete4 lags PyPI and does not have the pinned version), matching the + # documented install (conda for the external tools, pip for FunVIP + ete4). Then a + # full import + unit-test smoke test. + install-smoke: + name: full install (conda env + pip) / ubuntu / py3.12 + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + channels: conda-forge,bioconda + channel-priority: flexible + auto-activate-base: false + activate-environment: funvip-ci + - name: Install FunVIP with test extras (pip pulls + builds ete4 from PyPI) + run: pip install -e ".[test]" + - name: Smoke test (package imports) + unit tests + run: | + # The install step above is the real check: it proves ete4 builds from + # its PyPI sdist and the whole dependency set resolves in a fresh env. + # We intentionally do NOT import ete4 here -- ete4 eagerly imports PyQt6 + # (GUI), which needs Qt system libraries (libGL, libEGL, libX11, ...) that + # a headless runner may lack. That is a runtime/deployment concern, not an + # install one, and real users have those libs. funvip's package init is + # import-safe (no ete4/Qt), so this check stays reliable. + python -c "import funvip; print('funvip imports OK')" + pytest -q + + # Same as install-smoke, on Apple Silicon macOS. The macOS-specific value is + # confirming ete4 compiles from its PyPI sdist on osx-arm64 (conda-forge lags the + # pinned version, so ete4 comes via pip here too). Qt-free smoke, as on Linux + # (macOS provides Qt at runtime via conda's pyqt in the documented recipe). + install-smoke-macos: + name: full install (conda env + pip) / macos-arm64 / py3.12 + runs-on: macos-latest + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + channels: conda-forge,bioconda + channel-priority: flexible + auto-activate-base: false + activate-environment: funvip-ci + - name: Install FunVIP with test extras (pip pulls + builds ete4 from PyPI) + run: pip install -e ".[test]" + - name: Smoke test (package imports) + unit tests + run: | + python -c "import funvip; print('funvip imports OK')" + pytest -q diff --git a/.gitignore b/.gitignore index d51df1d..fd29ed3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,13 +12,11 @@ /*.fasta /*.nwk /funid/external/MAFFT_Windows/ -!/funid/external/mmseqs_Windows/bin/* -!/funid/external/mmseqs_Windows/bin/*.exe -!/funid/external/mmseqs_Windows/bin/*.dll -!/funip/external/mmseqs_Windows/bin/* -!/funip/external/mmseqs_Windows/bin/*.exe -!/funip/external/mmseqs_Windows/bin/*.dll -!/funvip/external/mmseqs_Windows/bin/* +/funvip/external/MAFFT_Windows/ +# mmseqs' bundled busybox self-installs a symlink per POSIX tool name (ls, cat, +# grep, ...) into this dir at runtime; only the real committed binaries should +# be tracked. +/funvip/external/mmseqs_Windows/bin/* !/funvip/external/mmseqs_Windows/bin/*.exe !/funvip/external/mmseqs_Windows/bin/*.dll /funid/__pycache__/*.pyc @@ -44,6 +42,10 @@ /DB/blast/*/*.nsq /DB/mmseqs/*/* !/DB/mmseqs/*/*.gitignore + +# Runtime BLAST/mmseqs db cache, built next to the installed package (see +# funvip/src/initialize.py: self.in_db) +/funvip/db/ /FunID-win32-x64/* /FunIP-win32-x64/* /FunVIP-win32-x64/* @@ -67,3 +69,15 @@ DB.fasta /funvip/FunVIP.egg-info/* CLAUDE.md .claude/ + +# Python + pytest caches +__pycache__/ +*.pyc +.pytest_cache/ +.coverage +htmlcov/ + +# Virtual environments +.venv/ +.venv*/ +venv/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..84afb55 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,71 @@ +# Changelog + +All notable changes to FunVIP are documented here. This project adheres to +[Semantic Versioning](https://semver.org/). + +## [1.0.0] - 2026-07-09 + +First stable release. The pipeline was ported from ete3 to ete4, made +substantially faster, hardened across the board, and given native Windows support. + +Upgrading from 0.5.x: 0.5.x is built on ete3 and 1.0 on ete4 (different packages), +so rebuild the conda environment instead of `pip install --upgrade`. See +`tutorial/installation.md` (Migrating from 0.5.x to 1.0). + +### Major +- **ete4 engine.** Migrated the whole tree layer from ete3 to ete4. Supported + Python is now 3.9-3.13. +- **Native Windows support.** ete4 has no Windows wheel on PyPI; FunVIP now bundles + prebuilt ete4 wheels (`funvip/_vendor/ete4_wheels`) and installs the matching one + on first run, falling back to building ete4 from source (patched for Windows) when + no wheel matches. The Windows install is now the same `pip install FunVIP` as + Linux/macOS. + +### Performance +- Tree interpretation roughly 13x faster (memoized tree search, per-tree set-cache + in `decide_type`, numpy `calculate_zero`, within-cluster `diff_min` instead of an + O(n^2) distance matrix). +- `seperate_clade` no longer re-deep-copies the resolved comb at every level + (dropped an O(depth^2) copy). +- Vectorized group clustering; `generate_dataset` pre-indexes sequences by group + (was O(groups x genes x N)). +- Pool workers share the sequence universe via fork copy-on-write instead of + per-task pickling. +- `concatenate` frees per-gene search tables after building the concatenated table + and dictionary-encodes its string columns. + +### Fixed +- **5.8S / conserved-gene interpretation crash.** The conserved-gene tree step used + to fail silently (RecursionError / pickle failure); it now interprets correctly. +- **Result-affecting preset bug.** An explicitly given cluster cutoff was discarded, + so runs used the wrong cutoff; presets and CLI cutoffs are now applied correctly. +- **TCS (t-coffee) no longer eats all system memory.** The bundled t-coffee + crash-loops on modern kernels (PID-indexed static array vs. large + `kernel.pid_max`); FunVIP no longer executes t-coffee just to detect it, caps and + times out the TCS run, and skips TCS cleanly on failure. See `tools/tcoffee/` for + a recipe to build a working t-coffee. +- Numerous correctness and robustness fixes: external-tool exit codes are checked, + option/preset validation bugs, several always-wrong guards, a list-mutation bug, + `--all` handling, `homogenize`, and the GenMine executable resolution. + +### Added +- Typed exception hierarchy with a top-level handler, plus a logging overhaul across + the core modules. +- Method-aware external-tool preflight (checks only the tools the chosen methods + need) and a more robust `Version` probe. +- Unit-test suite and CI (pip matrix on Python 3.10-3.13 + a conda install smoke + test), and a marked terrei end-to-end integration test. +- `environment.yml` and a `Dockerfile` for reproducible installs. +- Self-contained HTML report and per-group statistics. +- Parameter reference (`docs/parameters.md`), refreshed README and installation + docs, and build recipes under `tools/` (working t-coffee for TCS; ete4 Windows + wheels). + +### Changed (behavior) +- Cluster e-value default is now `1e-4` (a single clean preset key). +- Group-assignment ties are broken by top-bitscore majority. +- Missing per-gene bitscores in the concatenated table are filled by projecting onto + the fitted regression line. +- Removed confirmed-dead / broken code paths. + +[1.0.0]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b232df4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +# FunVIP in a self-contained conda environment (bundles the external tools). +# Useful on Windows/WSL or anywhere a native install is awkward. +# +# docker build -t funvip . +# docker run --rm -v "$PWD:/data" funvip --test terrei --email you@example.com --outdir /data/out +# +FROM condaforge/miniforge3:latest + +# Build the environment from environment.yml (external tools via conda + FunVIP +# via pip). Clean caches to keep the image small. +COPY environment.yml /tmp/environment.yml +RUN mamba env create -f /tmp/environment.yml && mamba clean -a -y + +# Headless Qt for ete4 tree rendering +ENV QT_QPA_PLATFORM=offscreen + +# Run FunVIP inside the env by default; arguments passed to `docker run` go to FunVIP. +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "funvip", "FunVIP"] +CMD ["--help"] diff --git a/README.md b/README.md index f73be15..8220362 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,6 @@ Bug reports are always welcomed -#### IMPORTANT NOTICE: The python dependency for Linux platform has changed from 3.12 to 3.11 for TCS inclusion. Please remake conda environment for FunVIP 0.3.25 update - ## Tutorial * [Part 1 - Getting started!](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial.md) * [Part 2 - Preparing database and query](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial2.md) @@ -34,6 +32,7 @@ Bug reports are always welcomed

## Documentation * See [Documentation](https://github.com/Changwanseo/FunVIP/blob/main/Documentation.md) for advanced usage ! +* See the [command-line parameter reference](docs/parameters.md) for every option, or run `FunVIP --help`

## Requirements - Conda or Mamba environment @@ -43,10 +42,23 @@ Bug reports are always welcomed \* Recently, Mamba is a lot faster than conda. See [here](https://github.com/conda-forge/miniforge?tab=readme-ov-file) to how to install mamba environment

## Installation -* [Windows](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Windows) -* [Mac - apple silicon](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Apple ) -* [Linux](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Linux) -* [from source](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Installation) +* [Windows](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#windows) +* [Mac - apple silicon](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#apple-silicon-mac) +* [Linux](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#linux) +* [from source](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#installation-from-source-for-developers) + +### Quick install (conda / mamba) +The external aligners and tree tools install via conda; FunVIP and its Python dependencies (including ete4) install via pip (which works on Linux/macOS). +```bash +conda env create -f environment.yml # or: mamba env create -f environment.yml +conda activate funvip +FunVIP --test terrei --email you@example.com +``` +Or run everything in a container (no local install; handy on Windows/WSL): +```bash +docker build -t funvip . +docker run --rm -v "$PWD:/data" funvip --test terrei --email you@example.com --outdir /data/out +```

## Usage ```FunVIP --db {Your database file} --query {Your query file} --email {Your email} --gene {Your genes} --preset {fast or accurate}``` @@ -54,6 +66,10 @@ Bug reports are always welcomed ### Example ```FunVIP --db Penicillium.xlsx --query Query.xlsx --email {Your email} --thread 8 --gene ITS BenA RPB2 CaM --preset fast``` +### Quick check +Verify your installation on a bundled dataset (needs an e-mail for the GenBank accessions): +```FunVIP --test terrei --email {Your email}``` + \* See documentation for detailed usage

@@ -88,11 +104,12 @@ tabular (```.xlsx```, ```.csv```, ```.parquet```, ```.ftr```) form * TREE_METHOD : fasttree is fastest, but least accurate (However, still a lot accurate than NJ tree). It is treated that iqtree is faster but slightly less accurate than raxml, but iqtree requires at least 1000 bootstrap. So in case of speed, raxml could be a little bit faster when low bootstrap selected--> ## Results -* ```Section Assignment.xlsx``` : Your clustering result is here. You can find which of your sequences are clustered to which section -* ```Identification_result.xlsx``` : Your final identification result. Shows how your sequences were assigned to species level through tree-based identification -* ```report.xlsx``` : overall statistics about the tree. If your find taxon ends with numbers, these taxon are found to be paraphyletic, so should be checked -* ```/Tree/{section}_{gene}.svg``` : Final collapsed tree in svg format. Can be edited in vector graphics programs, or in powerpoint (by ungroup) -* ```/Tree/{section}_{gene}_original.svg ``` : Uncollapsed tree for inspection +Outputs are written to `{outdir}/{runname}/`: +* ```{runname}.result.csv``` : Final identification result: how each query was assigned to the group and species level through tree-based identification +* ```{runname}.dataset.csv``` : Clustering result: which group (e.g. genus/section) each sequence was assigned to +* ```{runname}.report.txt``` : Run report with the options used and a per-step summary. Taxa ending with a number are paraphyletic and should be checked +* ```07_Tree/{runname}_{group}_{gene}.svg``` : Final collapsed tree (SVG), editable in vector-graphics programs or PowerPoint (ungroup) +* ```07_Tree/{runname}_{group}_{gene}_original.svg``` : Uncollapsed tree for inspection * Example output tree of FunVIP ![image](https://github.com/user-attachments/assets/7291c990-62d0-4579-8ae7-adc5d39a7fed) @@ -106,6 +123,9 @@ Will be tested by our lab memebers to fix bugs and advance features~~ Will be tested by peer taxonomists 3. Stable release (ver 1.0) +## Running the tests +```pytest``` runs the fast unit tests (no external tools needed). ```pytest -m integration``` runs the end-to-end terrei test (needs the external tools on PATH and ```FUNVIP_TEST_EMAIL``` set). See [tests/README.md](tests/README.md). + ## License [GPL 3.0](https://github.com/Changwanseo/FunVIP/blob/main/LICENSE) diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..f3d5e39 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,54 @@ +# Releasing FunVIP + +Maintainer checklist for cutting a release (e.g. 1.0.0). Run from a clean checkout +of `development` (or the release branch). + +## 1. Bundle the ete4 Windows wheels +ete4 has no Windows wheel on PyPI, so FunVIP ships them under +`funvip/_vendor/ete4_wheels/` and installs the right one on first Windows run. + +- [ ] Run the **build-ete4-windows-wheels** GitHub Action (Actions tab -> Run + workflow). It builds `cp310`-`cp313` `win_amd64` wheels and import-tests them. +- [ ] Download the artifact and place all four `.whl` files in + `funvip/_vendor/ete4_wheels/`, then commit them. +- [ ] Confirm the wheel version satisfies the ete4 pin in `pyproject.toml` + (`>=4.4.0,<4.5.0`). + +(The `cp312` wheel is already vendored and validated on a real Windows machine; +the Action produces the remaining versions on a clean runner.) + +## 2. Version and changelog +- [ ] Bump `version` in `pyproject.toml`. It is the single source of truth + (`FunVIP --version` reads it via `importlib.metadata`). +- [ ] Add/finish the release entry in `CHANGELOG.md` and set its date. + +## 3. Verify (do not skip) +- [ ] `pytest` passes (unit suite). +- [ ] Linux end-to-end: `FUNVIP_TEST_EMAIL= pytest -m integration` (or + `FunVIP --test terrei --email ` in a temp dir) produces a result CSV. +- [ ] Windows end-to-end: in a fresh Python 3.12 conda env, `pip install .` then + `FunVIP --test terrei --email ` -- confirm it prints + "installing bundled ete4" (uses the wheel, no compile) and completes. +- [ ] macOS: run the recipe once, or mark Intel Mac "experimental" in the docs. +- [ ] At least one real, dataset-scale run (the intended workload) completes. + +## 4. Build and publish +- [ ] Clean build: `python -m build` (produces sdist + a `py3-none-any` wheel; + confirm `funvip/_vendor/ete4_wheels/*.whl`, `funvip/external/*`, `funvip/data`, + and `funvip/preset` are inside the built artifacts). +- [ ] `twine check dist/*`. +- [ ] Upload: `twine upload dist/*` (test first on TestPyPI if unsure). Note: the + PyPI `FunVIP` was last published at 0.5.x (ete3); 1.0.0 is the ete4 line. +- [ ] Do NOT put a direct git/URL ete4 dependency in `pyproject.toml`; PyPI rejects + those. The Windows wheels ship inside the package, not as a URL dependency. + +## 5. Tag and announce +- [ ] Tag: `git tag v1.0.0 && git push origin v1.0.0`. +- [ ] Create the GitHub release from the tag; paste the CHANGELOG entry. +- [ ] Merge `development` into `main` if that is the release branch. + +## Known issues to mention in the release notes +- TCS is optional and Linux-only; the prebuilt bioconda t-coffee is broken (see + `tools/tcoffee/`). FunVIP skips TCS automatically when t-coffee is absent. +- On Windows, MAFFT's bundled bash prints a harmless "could not find /tmp" warning; + it does not affect results. diff --git a/docs/RELEASE_NOTES_v1.0.0.md b/docs/RELEASE_NOTES_v1.0.0.md new file mode 100644 index 0000000..e34fc23 --- /dev/null +++ b/docs/RELEASE_NOTES_v1.0.0.md @@ -0,0 +1,98 @@ +# FunVIP 1.0.0 + +First stable release. FunVIP moved from ete3 to ete4, became substantially faster, +was hardened across the board, and now installs natively on Windows with the same +one-line `pip install` as Linux and macOS. + +## Highlights + +- **New ete4 tree engine.** The whole tree layer was ported from ete3 to ete4. + Supported Python is now 3.9-3.13. +- **Native Windows support.** ete4 has no Windows wheel on PyPI, so FunVIP bundles + prebuilt ete4 wheels and installs the right one on first run (building from a + patched source only if no wheel matches). Windows now uses the same install steps + as the other platforms. +- **About 13x faster tree interpretation**, plus faster clustering, dataset + building, and concatenation. +- **Conserved-gene (5.8S) crash fixed** and a **result-affecting cluster-cutoff bug + fixed** (an explicit cutoff used to be discarded). +- **TCS no longer exhausts system memory.** The optional t-coffee step is now safely + contained and skipped when unavailable. +- **Test suite + CI, reproducible installs** (`environment.yml`, `Dockerfile`), and + a **self-contained HTML report**. + +## Install + +Linux (external tools via conda, FunVIP + ete4 via pip): +``` +conda create -n FunVIP python=3.12 +conda activate FunVIP +conda config --add channels conda-forge +conda install -c bioconda raxml iqtree "modeltest-ng==0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree +pip install FunVIP +FunVIP --test terrei --email +``` + +Windows is the same recipe (`conda create` / `activate` / `pip install FunVIP` / +`--test`); FunVIP installs the bundled ete4 automatically on first run. See +`tutorial/installation.md` for macOS and the one-file `environment.yml` recipe. + +## Upgrading from 0.5.x + +0.5.x is built on **ete3** and 1.0 on **ete4** (different packages, not two versions +of one). Do **not** `pip install --upgrade` across this jump; rebuild the conda +environment from scratch instead. Your input data, databases, and result folders are +untouched. Full steps are in `tutorial/installation.md`. + +## Changes + +### Performance +- Tree interpretation about 13x faster (memoized tree search, per-tree set-cache in + `decide_type`, numpy `calculate_zero`, within-cluster `diff_min` instead of an + O(n^2) distance matrix); removed an O(depth^2) copy in `seperate_clade`. +- Vectorized group clustering; `generate_dataset` pre-indexes sequences by group. +- Pool workers share the sequence universe via fork copy-on-write instead of + per-task pickling. +- `concatenate` frees per-gene tables and dictionary-encodes string columns. + +### Fixed +- Conserved-gene (5.8S) interpretation crash (silent RecursionError / pickle + failure) now interprets correctly. +- Cluster cutoff given explicitly (preset or CLI) is now applied instead of being + discarded. +- TCS (t-coffee) no longer hangs and consumes all memory: FunVIP never executes + t-coffee just to detect it, caps and times out the TCS run, and skips it cleanly + on failure. See `tools/tcoffee/` for a recipe to build a working t-coffee. +- Many correctness and robustness fixes: external-tool exit codes are checked, + option/preset validation bugs, several always-wrong guards, a list-mutation bug, + `--all` handling, `homogenize`, and GenMine executable resolution. + +### Added +- Typed exception hierarchy with a top-level handler and a logging overhaul. +- Method-aware external-tool preflight and a more robust version probe. +- Unit-test suite and CI (pip matrix on Python 3.10-3.13 plus a conda install smoke + test) and a marked terrei end-to-end integration test. +- `environment.yml` and a `Dockerfile`. +- Self-contained HTML report and per-group statistics. +- Parameter reference (`docs/parameters.md`), refreshed README and installation + docs, and build recipes under `tools/` (working t-coffee for TCS; ete4 Windows + wheels). + +### Changed (behavior) +- Cluster e-value default is now `1e-4`. +- Group-assignment ties are broken by top-bitscore majority. +- Missing per-gene bitscores in the concatenated table are filled by projecting onto + the fitted regression line. +- Removed confirmed-dead / broken code paths. + +## Known issues + +- TCS is optional and Linux-only. The prebuilt bioconda t-coffee is broken on modern + kernels (see `tools/tcoffee/`); FunVIP skips TCS automatically when t-coffee is + absent. +- On Windows, MAFFT's bundled shell prints a harmless "could not find /tmp" warning; + it does not affect results. + +## Citation + +If you use FunVIP, please cite the FunVIP publication (see the repository README). diff --git a/docs/gen_param_reference.py b/docs/gen_param_reference.py new file mode 100644 index 0000000..053cf90 --- /dev/null +++ b/docs/gen_param_reference.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +"""Generate docs/parameters.md from the live argparse parser in command.py. + +Run from the repo root: python docs/gen_param_reference.py > docs/parameters.md +This keeps the parameter reference in sync with funvip/src/command.py. +""" +import sys + +sys.argv = ["FunVIP"] # so the parser's parse_args() call sees no arguments + +from funvip.src.command import CommandParser + + +def _type_name(action): + if action.const is True or action.nargs == 0: + return "flag" + if action.type is not None: + return getattr(action.type, "__name__", str(action.type)) + if action.nargs in ("*", "+"): + return "list" + return "str" + + +def main(): + cp = CommandParser() + cp.get_args() # builds cp.parser (harmlessly parses no arguments) + parser = cp.parser + + out = [ + "# FunVIP command-line parameters", + "", + "_Auto-generated from `funvip/src/command.py` by `docs/gen_param_reference.py`;", + "do not edit by hand. Regenerate after changing the CLI._", + ] + for group in parser._action_groups: + actions = [a for a in group._group_actions if a.option_strings] + if not actions: + continue + title = (group.title or "options").strip() + out += ["", f"## {title}", "", "| Option | Type | Description |", "|---|---|---|"] + for a in actions: + opts = ", ".join(f"`{o}`" for o in a.option_strings) + help_ = (a.help or "").replace("|", r"\|").replace("\n", " ") + out.append(f"| {opts} | {_type_name(a)} | {help_} |") + print("\n".join(out)) + + +if __name__ == "__main__": + main() diff --git a/docs/parameters.md b/docs/parameters.md new file mode 100644 index 0000000..fdca8e9 --- /dev/null +++ b/docs/parameters.md @@ -0,0 +1,114 @@ +# FunVIP command-line parameters + +_Auto-generated from `funvip/src/command.py` by `docs/gen_param_reference.py`; +do not edit by hand. Regenerate after changing the CLI._ + +## options + +| Option | Type | Description | +|---|---|---| +| `-h`, `--help` | flag | show this help message and exit | +| `--email`, `-e` | str | E-mail notation to download data from GenBank | +| `--api`, `-a` | str | NCBI API strings to download data from GenBank | +| `--version` | flag | show program's version number and exit | + +## required + +| Option | Type | Description | +|---|---|---| +| `--query`, `-q` | str | Query fasta or table files, delimitated by space | +| `--db`, `-d` | str | Database table files, delimitated by space | +| `--gene`, `-g` | str | Gene names to be analyzed | + +## test + +| Option | Type | Description | +|---|---|---| +| `--test` | str | Run on a bundled test dataset, one of [penicillium, terrei, sanghuangporus]. Requires --email for the bundled GenBank accessions. | + +## run + +| Option | Type | Description | +|---|---|---| +| `--thread`, `-t` | int | Threads to be used for pipeline, default : system maximum | +| `--memory`, `-m` | str | Max memory limit in 'nG' form, ex: '16G', should be more than 4G, default : system maximum | +| `--outdir` | str | Out file location, default : current directory | +| `--runname` | str | Name prefix to current run : default : current timestamp | +| `--mode` | str | Analysis mode, one of [identification, validation]. default : identification | +| `--continue` | flag | Continue from previous run, default: False | +| `--step` | str | With --continue, resume from this pipeline step, one of [setup, search, cluster, align, trim, concatenate, modeltest, tree, visualize, report]. Ignored without a valid --continue. | +| `--level` | str | Taxonomic level at which each phylogenetic tree is built, one of [genus, subseries, series, subsection, section, subtribe, tribe, subfamily, family, suborder, order, subclass, class, subphylum, phylum, subdivision, division, subkingdom, kingdom]. default : genus | +| `--all` | flag | Run FunVIP for all database sequences, regardless of corresponding sequences exists in query, default : False | +| `--confident` | str | Skip blast analysis among database sequences, use it when your database sequences contains large number of misidentified sequences, default : False | +| `--maxoutgroup` | int | Maximum number of outgroup sequences to include in each phylogenetic tree, default : 3 | + +## method + +| Option | Type | Description | +|---|---|---| +| `--search` | str | Search method for selecting genes, groups and outgroups, one of [blast, mmseqs]. default : blast | +| `--alignment` | str | Multiple sequence alignment methods, [mafft], default : mafft | +| `--notcs` | flag | Skip T-COFFEE TCS(Transitive Consistency Score) for alignment validation. default: False | +| `--trim` | str | Trimming methods, [trimal, gblocks, none], default : trimal | +| `--modeltest` | str | Model test methods, [iqtree, modeltestng, none], default : none | +| `--tree` | str | Tree methods to build phylogenetic tree, [fasttree, iqtree, raxml], default : fasttree | + +## visualize + +| Option | Type | Description | +|---|---|---| +| `--bscutoff` | int | Bootstrap cutoff for visualize, default : 70 | +| `--highlight` | str | Color to highlight query sequences in tree visualization. Either in html svg recognizable string or hex code, default: #AA0000 | +| `--heightmultiplier` | float | Height multiplier in drawing collapsing nodes. Change it if you want to show collapse node more or less expanded. Default: 6 | +| `--maxwordlength` | int | Maximum letters to be shown in single line of tree annotation. Default: 48 | +| `--backgroundcolor` | str | Alternating background band colors in the tree, space-delimited hex codes in quotes. default: "#ffe0e0" "#ffefef". To remove the background use --backgroundcolor "#FFFFFF" "#FFFFFF". | +| `--outgroupcolor` | str | Background colors to indicate outgroup, default: #999999 | +| `--ftype` | str | Font to use for phylogenetic tree, default: Arial | +| `--fsize` | float | Font size for phylogenetic tree labels, default: 14 | +| `--fsize_bootstrap` | float | Font size to use for bootstrap support in phylogenetic tree, default: 9 | + +## advanced + +| Option | Type | Description | +|---|---|---| +| `--verbose`, `-v` | int | Verbosity level, 0: quiet, 1: info, 2: warning, 3: debug, default : 2 | +| `--collapsedistcutoff` | float | Maximum tree distance to be considered as same species, default : 0.01 | +| `--collapsebscutoff` | float | Minimum bootstrap support to collapse a clade as one species; the default 101 (above the 100 maximum) disables bootstrap-based collapsing. default : 101 | +| `--bootstrap` | int | Bootstrap replicates for tree analysis (ignored when tree method is fasttree). default : 100 (the accurate preset uses 1000) | +| `--nosolveflat` | flag | Do not detect 0 length branch and automatically solve them, default : False | +| `--regex` | str | Regex groups to parse strain numbers from your input. Maybe useful if your sequence descriptions are dirty. See documentation | +| `--cluster-cutoff` | float | Minimum percent identity to be considered as same group in clustering analysis. Should be between 0 and 1, default : 0.97 | +| `--cluster-evalue` | float | E-value cutoff for the blast/mmseqs clustering search; an explicit value overrides the preset. default : 0.0001 | +| `--cluster-wordsize` | int | Word size for blast/mmseqs search, default : 7 | +| `--cluster-max_target_seqs` | int | Max_target_seqs for blast/mmseqs search, increase this when expected search match is not found. default : 100 | +| `--mafft-algorithm` | str | MAFFT algorithm for alignment (see MAFFT docs). default : auto | +| `--mafft-op` | float | MAFFT op (gap opening penalty) value, default : 1.3 | +| `--mafft-ep` | float | MAFFT ep value, default : 0.1 | +| `--trimal-algorithm` | str | trimAl algorithm for trimming (see trimAl docs). default : gt | +| `--trimal-gt` | float | trimAl -gt (gap threshold), between 0 and 1. default : 0.2 | +| `--allow-innertrimming` | flag | Turn off FunVIP adjustment to not to trim inner alignment columns, default: False | +| `--criterion` | str | Model-selection criterion for modeltest, one of [AIC, AICc, BIC]. default : BIC | +| `--noavx` | flag | Do not use AVX for RAxML, default: False | +| `--outgroupoffset` | int | outgroupoffset value. Highering this value may select more distant outgroup, default : 20 | +| `--nosuspicious` | flag | Do not include suspicious samples for sequence-set. Mostly for metabarcoding analysis. May deduce inaccurate result with problematic database. | +| `--terminate` | flag | Terminate FunVIP run when critical error detected | + +## cache + +| Option | Type | Description | +|---|---|---| +| `--nocachedb` | flag | Disable caching current search database. Use it if your database is too big for system directory, default : True | +| `--usecache` | flag | Use cached search database, turn off if your cached database makes error, default : True | + +## save + +| Option | Type | Description | +|---|---|---| +| `--tableformat` | str | Default format for table files, [csv, xlsx], default : csv | +| `--nosearchresult` | flag | Do not save blast/mmseqs search matrix, use when dataset gets too big and generates IO bottleneck, default: False | + +## setting + +| Option | Type | Description | +|---|---|---| +| `--preset` | str | [fast, accurate], or json formatted option config file. Check documentation for each preset, default : fast | diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..6c4718c --- /dev/null +++ b/environment.yml @@ -0,0 +1,38 @@ +# Reproducible conda/mamba environment for FunVIP. +# +# conda env create -f environment.yml # or: mamba env create -f environment.yml +# conda activate funvip +# FunVIP --test terrei --email you@example.com +# +# The external aligners/tree tools come from conda (bioconda/conda-forge); FunVIP +# and its Python dependencies (incl. ete4) come from PyPI, which installs cleanly +# on Linux/macOS. To use the development version instead of the PyPI release, +# remove the `- FunVIP` pip line below and run `pip install -e .` from the repo. +name: funvip +channels: + - conda-forge + - bioconda +dependencies: + - python=3.12 + # External tools (tested versions in comments; unpinned to track conda updates) + - blast # 2.17.0 + - mmseqs2 # 18.8cc5c + - mafft # 7.526 + - trimal # 1.5.x + - fasttree # 2.2.0 + - iqtree # 3.1.2 + - raxml # 8.2.x + - modeltest-ng # 0.1.7 + # - t-coffee # OMITTED BY DEFAULT. TCS (optional alignment validation) needs + # t-coffee, but the bioconda build crash-loops and consumes all RAM + # on modern kernels: it indexes a static array by raw PID yet is + # compiled with MAX_N_PID=260000, so when kernel.pid_max is large + # (e.g. 4194304, the current default) its own PID overflows the + # array -> SIGSEGV -> infinite fault loop. The MAX_N_PID_4_TCOFFEE + # override is NOT compiled into that binary, so it cannot be raised + # at runtime. For working TCS use a t-coffee built with + # MAX_N_PID >= pid_max (or a host with pid_max < 260000). FunVIP + # caps + times out t-coffee and skips TCS cleanly when it fails. + - pip + - pip: + - FunVIP diff --git a/funvip/_vendor/__init__.py b/funvip/_vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/funvip/_vendor/ete4_wheels/LICENSE b/funvip/_vendor/ete4_wheels/LICENSE new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/funvip/_vendor/ete4_wheels/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/funvip/_vendor/ete4_wheels/README.md b/funvip/_vendor/ete4_wheels/README.md new file mode 100644 index 0000000..0ebbd7e --- /dev/null +++ b/funvip/_vendor/ete4_wheels/README.md @@ -0,0 +1,27 @@ +# Bundled ete4 Windows wheels + +FunVIP installs the matching wheel from this directory on the **first Windows run** +(see `_ensure_ete4` in `funvip/main.py`), because ete4 has no Windows wheel on PyPI +and cannot compile there. On Linux/macOS this directory is unused (ete4 installs +normally from PyPI). + +## What goes here + +`ete4--cp3XX-cp3XX-win_amd64.whl` for each supported CPython (3.10–3.13), +e.g. `ete4-4.4.0-cp312-cp312-win_amd64.whl`. + +## How to produce them + +Run the **build-ete4-windows-wheels** GitHub Actions workflow (see +`tools/ete4-windows/`), download the `ete4--windows-wheels` artifact, and +drop the `.whl` files here, then commit them. They are declared as package data in +`pyproject.toml`, so they ship inside the FunVIP wheel/sdist. + +The wheel version must satisfy FunVIP's ete4 pin (`>=4.4.0,<4.5.0`). + +## License + +These wheels are ete4, distributed under **GPL-3.0-or-later** (see `LICENSE` in this +directory). FunVIP bundles them unmodified except for the small Windows build fix in +`tools/ete4-windows/ete4-windows.patch`; the corresponding source is ete4's public +release plus that patch. FunVIP (also GPL-3.0) may redistribute them under the GPL. diff --git a/funvip/_vendor/ete4_wheels/ete4-4.4.0-cp312-cp312-win_amd64.whl b/funvip/_vendor/ete4_wheels/ete4-4.4.0-cp312-cp312-win_amd64.whl new file mode 100644 index 0000000..e04c154 Binary files /dev/null and b/funvip/_vendor/ete4_wheels/ete4-4.4.0-cp312-cp312-win_amd64.whl differ diff --git a/funvip/main.py b/funvip/main.py index 8d40ca5..efd8c92 100644 --- a/funvip/main.py +++ b/funvip/main.py @@ -18,7 +18,132 @@ def _ensure_deterministic_hash(): _ensure_deterministic_hash() -def main(): +# ete4 version fetched when building from source on Windows. Matches the pin in +# pyproject.toml and the bundled wheels; the source patch in _build_ete4_from_source +# targets this version. +_ETE4_SRC_VERSION = "4.4.0" + + +def _pip(*args): + import subprocess + + subprocess.check_call( + [sys.executable, "-m", "pip", "--disable-pip-version-check", *args] + ) + + +def _patch_line(path, old, new): + with open(path, encoding="utf-8") as fh: + text = fh.read() + if new in text: + return + if old not in text: + raise RuntimeError(f"expected text not found while patching {os.path.basename(path)}") + with open(path, "w", encoding="utf-8") as fh: + fh.write(text.replace(old, new)) + + +def _build_ete4_from_source(): + """Download the ete4 sdist, apply the Windows build fix (etetoolkit/ete PR #783), + and pip-install it. Needs a C/C++ compiler and internet; runs at most once.""" + import glob + import json + import tarfile + import tempfile + import urllib.request + + with tempfile.TemporaryDirectory() as tmp: + # Fetch the raw sdist tarball directly (not via `pip download`): pip would + # invoke the sdist's build backend to resolve metadata, which runs ete4's + # unpatched setup.py and hits the exact Windows path bug (PR #783) we are + # about to patch -- before we ever get a chance to apply the patch. + with urllib.request.urlopen( + f"https://pypi.org/pypi/ete4/{_ETE4_SRC_VERSION}/json" + ) as resp: + release = json.load(resp) + sdist_url = next( + f["url"] for f in release["urls"] if f["packagetype"] == "sdist" + ) + sdist = os.path.join(tmp, "ete4.tar.gz") + urllib.request.urlretrieve(sdist_url, sdist) + with tarfile.open(sdist) as tar: + try: + tar.extractall(tmp, filter="data") + except TypeError: # filter= added in Python 3.12 + tar.extractall(tmp) + src = next(d for d in glob.glob(os.path.join(tmp, "ete4-*")) if os.path.isdir(d)) + _patch_line( + os.path.join(src, "setup.py"), + "from os.path import isfile", "from os.path import isfile, sep", + ) + _patch_line( + os.path.join(src, "setup.py"), + "path.replace('/', '.')", "path.replace(sep, '.')", + ) + _patch_line( + os.path.join(src, "ete4", "config.py"), + "os.environ['HOME']", "os.path.expanduser('~')", + ) + _pip("install", "--no-deps", src) + + +def _ensure_ete4(): + """Make ete4 importable. ete4 has no Windows wheel on PyPI, so on Windows it is + not a pip dependency; instead FunVIP installs a prebuilt wheel bundled under + funvip/_vendor/ete4_wheels, or, if none is bundled for this Python, builds ete4 + from source (patched for Windows). No-op on Linux/macOS and once ete4 imports.""" + try: + import ete4 # noqa: F401 + + return + except ImportError: + pass + + if sys.platform != "win32": + # Off Windows ete4 is an ordinary dependency; a failed import is a genuine + # installation problem, so let it surface rather than masking it. + raise + + import glob + import importlib + + tag = f"cp{sys.version_info.major}{sys.version_info.minor}" + wheel_dir = os.path.join(os.path.dirname(__file__), "_vendor", "ete4_wheels") + wheels = sorted( + glob.glob(os.path.join(wheel_dir, f"ete4-*-{tag}-{tag}-win_amd64.whl")) + ) + try: + if wheels: + wheel = wheels[-1] + print( + f"[FunVIP] First Windows run: installing bundled ete4 " + f"({os.path.basename(wheel)}). This happens only once.", + flush=True, + ) + _pip("install", "--no-deps", wheel) + else: + print( + f"[FunVIP] First Windows run: no bundled ete4 wheel for Python {tag}; " + "building ete4 from source now (one-time; needs a C/C++ compiler and " + "internet, may take a few minutes).", + flush=True, + ) + _build_ete4_from_source() + except Exception as e: + raise SystemExit( + f"[FunVIP] Could not set up ete4 automatically: {e}\n" + "If the build failed for lack of a compiler, install one with:\n" + " conda install -c conda-forge cxx-compiler\n" + "then rerun. Otherwise place a prebuilt ete4 wheel in " + "funvip/_vendor/ete4_wheels (see tools/ete4-windows/)." + ) + + importlib.invalidate_caches() + import ete4 # noqa: F401 -- verify it imports now + + +def _run_funvip(): + _ensure_ete4() # install the bundled ete4 wheel on first Windows run (no-op elsewhere) from funvip.src import align from funvip.src import tree_interpretation_pipe from funvip.src import cluster @@ -39,7 +164,7 @@ def main(): from funvip.src.command import CommandParser from funvip.src.tool import index_step from funvip.src.opt_generator import opt_generator - from funvip.src.version import Version + from funvip.src.version import Version, preflight from time import time from time import sleep import pandas as pd @@ -52,7 +177,7 @@ def main(): import shutil import sys import logging - import PyQt5 + import PyQt6 # To prevent pyqt error os.environ["QT_QPA_PLATFORM"] = "offscreen" @@ -90,6 +215,10 @@ def main(): logger.setup_logging(list_info, list_warning, list_error, path, opt, tool) + # Fail fast (before any heavy work) with a clear message if a required + # external tool for the selected methods is not available on PATH. + preflight(opt) + # V contains all intermediate variables for FunVIP Run V = dataset.FunVIP_var() # R includes all reporting objects such as warnings, errors, statistics @@ -97,13 +226,23 @@ def main(): # Reload previous session from shelve if --continue selected if opt.continue_from_previous is True: + from funvip.src.exceptions import ConfigError + var = save.load_session(opt, savefile=path.save) - if "V" in var: - V = var["V"] - if "R" in var: - R = var["R"] - if "path" in var: - path = var["path"] + # A previous checkpoint that failed to save a required key (an unpicklable + # object logged only as a warning, an interrupted write, or a missing + # save.shelve) leaves these absent; resuming would silently run on an empty + # V and emit a 0-row result.csv reported as success. Fail clearly instead. + for _required in ("V", "R", "path"): + if _required not in var: + raise ConfigError( + f"--continue was requested but the saved session at {path.save} is " + f"missing required key '{_required}'. A previous checkpoint save was " + "incomplete or the session file is absent; re-run from an earlier --step." + ) + V = var["V"] + R = var["R"] + path = var["path"] if "model_dict" in var: model_dict = var["model_dict"] if "GenMine_flag" in var: @@ -429,7 +568,7 @@ def main(): try: logging.info(f"Time report generation: {round(time_end-time_visualize,3)}s") except: - logging.warning(f"Failed logging reoprt generation time") + logging.warning(f"Failed logging report generation time") # At the end of the run, print critical messages once again to be noticed with open(path.criticallog, "r") as frclog: @@ -451,3 +590,27 @@ def main(): for line in critical_logs: print(line) + + +def main(): + """Entry point: run FunVIP under a top-level handler so an expected + FunVIPError is reported cleanly and any unexpected crash is logged with a + full traceback, instead of dumping a bare stack trace at the user.""" + import logging + import sys + import traceback + from funvip.src.exceptions import FunVIPError + + try: + _run_funvip() + except FunVIPError as e: + logging.critical(f"FunVIP stopped: {e}") + sys.exit(1) + except KeyboardInterrupt: + logging.warning("FunVIP interrupted by user (KeyboardInterrupt)") + sys.exit(130) + except Exception as e: + logging.critical( + f"FunVIP crashed with an unexpected error: {e}\n{traceback.format_exc()}" + ) + sys.exit(1) diff --git a/funvip/preset/accurate.yaml b/funvip/preset/accurate.yaml index 56266a8..de027ea 100644 --- a/funvip/preset/accurate.yaml +++ b/funvip/preset/accurate.yaml @@ -1,7 +1,7 @@ MODE: identification CONTINUE: false QUERYONLY: true -CONFIDENT: fale +CONFIDENT: false MAXOUTGROUP: 3 SEARCH: blast ALIGNMENT: mafft @@ -9,7 +9,7 @@ TRIM: trimal MODELTEST: modelfinder TREE: raxml CLUSTER-CUTOFF: 0.97 -CLUSTER-EVALUE: 10 +CLUSTER-EVALUE: 1.0e-04 MAFFT-ALGORITHM: localpair TRIMAL-ALGORITHM: gt COLLAPSEDISTCUTOFF: 0.01 @@ -18,7 +18,6 @@ SOLVEFLAT: true VERBOSE: 2 MATRIXFORMAT: csv WORDSIZE: 7 -EVALUE: 1.0e-04 TRIMAL-gt: 0.2 MAFFT-op: 1.3 MAFFT-ep: 0.1 diff --git a/funvip/preset/fast.yaml b/funvip/preset/fast.yaml index 6f6bfdb..8612405 100644 --- a/funvip/preset/fast.yaml +++ b/funvip/preset/fast.yaml @@ -8,14 +8,13 @@ ALIGNMENT: mafft TRIM: trimal TREE: fasttree CLUSTER-CUTOFF: 0.97 -CLUSTER-EVALUE: 10 +CLUSTER-EVALUE: 1.0e-04 MAFFT-ALGORITHM: auto COLLAPSEDISTCUTOFF: 0.01 SOLVEFLAT: true VERBOSE: 2 MATRIXFORMAT: csv WORDSIZE: 7 -EVALUE: 1.0e-04 TRIMAL-gt: 0.2 MAFFT-op: 1.3 MAFFT-ep: 0.1 diff --git a/funvip/src/__pycache__/CATV_pipe.cpython-310.pyc b/funvip/src/__pycache__/CATV_pipe.cpython-310.pyc deleted file mode 100644 index 08330d5..0000000 Binary files a/funvip/src/__pycache__/CATV_pipe.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/CAT_V.cpython-310.pyc b/funvip/src/__pycache__/CAT_V.cpython-310.pyc deleted file mode 100644 index 22484be..0000000 Binary files a/funvip/src/__pycache__/CAT_V.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/CAT_V.cpython-39.pyc b/funvip/src/__pycache__/CAT_V.cpython-39.pyc deleted file mode 100644 index a0d28f1..0000000 Binary files a/funvip/src/__pycache__/CAT_V.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/CAT_V_pipe.cpython-39.pyc b/funvip/src/__pycache__/CAT_V_pipe.cpython-39.pyc deleted file mode 100644 index 07afed9..0000000 Binary files a/funvip/src/__pycache__/CAT_V_pipe.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/__init__.cpython-310.pyc b/funvip/src/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 4d2af06..0000000 Binary files a/funvip/src/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/__init__.cpython-39.pyc b/funvip/src/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 433b1a9..0000000 Binary files a/funvip/src/__pycache__/__init__.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/align.cpython-310.pyc b/funvip/src/__pycache__/align.cpython-310.pyc deleted file mode 100644 index e86567d..0000000 Binary files a/funvip/src/__pycache__/align.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/cluster.cpython-39.pyc b/funvip/src/__pycache__/cluster.cpython-39.pyc deleted file mode 100644 index 494db76..0000000 Binary files a/funvip/src/__pycache__/cluster.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/dataset.cpython-39.pyc b/funvip/src/__pycache__/dataset.cpython-39.pyc deleted file mode 100644 index f2f309e..0000000 Binary files a/funvip/src/__pycache__/dataset.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/ext.cpython-310.pyc b/funvip/src/__pycache__/ext.cpython-310.pyc deleted file mode 100644 index 4da9360..0000000 Binary files a/funvip/src/__pycache__/ext.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/ext.cpython-39.pyc b/funvip/src/__pycache__/ext.cpython-39.pyc deleted file mode 100644 index 2b79204..0000000 Binary files a/funvip/src/__pycache__/ext.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/hasher.cpython-39.pyc b/funvip/src/__pycache__/hasher.cpython-39.pyc deleted file mode 100644 index 7489147..0000000 Binary files a/funvip/src/__pycache__/hasher.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/initialize.cpython-39.pyc b/funvip/src/__pycache__/initialize.cpython-39.pyc deleted file mode 100644 index 83f076d..0000000 Binary files a/funvip/src/__pycache__/initialize.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/io.cpython-310.pyc b/funvip/src/__pycache__/io.cpython-310.pyc deleted file mode 100644 index 6ac42fd..0000000 Binary files a/funvip/src/__pycache__/io.cpython-310.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/io.cpython-39.pyc b/funvip/src/__pycache__/io.cpython-39.pyc deleted file mode 100644 index a279770..0000000 Binary files a/funvip/src/__pycache__/io.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/logger.cpython-39.pyc b/funvip/src/__pycache__/logger.cpython-39.pyc deleted file mode 100644 index bf0832d..0000000 Binary files a/funvip/src/__pycache__/logger.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/modeltest.cpython-39.pyc b/funvip/src/__pycache__/modeltest.cpython-39.pyc deleted file mode 100644 index b9465e8..0000000 Binary files a/funvip/src/__pycache__/modeltest.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/multigene.cpython-39.pyc b/funvip/src/__pycache__/multigene.cpython-39.pyc deleted file mode 100644 index f429bc1..0000000 Binary files a/funvip/src/__pycache__/multigene.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/ncbi.cpython-39.pyc b/funvip/src/__pycache__/ncbi.cpython-39.pyc deleted file mode 100644 index 5aeb8f4..0000000 Binary files a/funvip/src/__pycache__/ncbi.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/opt_generator.cpython-39.pyc b/funvip/src/__pycache__/opt_generator.cpython-39.pyc deleted file mode 100644 index 504df3c..0000000 Binary files a/funvip/src/__pycache__/opt_generator.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/reporter.cpython-39.pyc b/funvip/src/__pycache__/reporter.cpython-39.pyc deleted file mode 100644 index 37bae44..0000000 Binary files a/funvip/src/__pycache__/reporter.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/search.cpython-39.pyc b/funvip/src/__pycache__/search.cpython-39.pyc deleted file mode 100644 index cc2165c..0000000 Binary files a/funvip/src/__pycache__/search.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/tool.cpython-39.pyc b/funvip/src/__pycache__/tool.cpython-39.pyc deleted file mode 100644 index 0f26997..0000000 Binary files a/funvip/src/__pycache__/tool.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/validation.cpython-39.pyc b/funvip/src/__pycache__/validation.cpython-39.pyc deleted file mode 100644 index 67101d5..0000000 Binary files a/funvip/src/__pycache__/validation.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/__pycache__/visualize.cpython-39.pyc b/funvip/src/__pycache__/visualize.cpython-39.pyc deleted file mode 100644 index 287c7f3..0000000 Binary files a/funvip/src/__pycache__/visualize.cpython-39.pyc and /dev/null differ diff --git a/funvip/src/cluster.py b/funvip/src/cluster.py index 78da7b9..1d8dcd1 100644 --- a/funvip/src/cluster.py +++ b/funvip/src/cluster.py @@ -14,6 +14,7 @@ import logging import gc from funvip.src.ext import mmseqs +from funvip.src.exceptions import ClusterError import sys @@ -45,7 +46,9 @@ def append_query_group(V): for FI in V.list_FI: group_dict[FI.hash] = FI.adjusted_group - V.cSR["query_group"] = V.cSR["qseqid"].apply(lambda x: group_dict.get(x)) + # vectorized map (every qseqid is an FI hash present in group_dict, so this is + # equivalent to the former per-row .apply(lambda x: group_dict.get(x))) + V.cSR["query_group"] = V.cSR["qseqid"].map(group_dict) logging.debug(V.cSR["query_group"]) @@ -80,18 +83,20 @@ def assign_gene(result_dict, V, cutoff=0.99): # get each of the dataframe for each FI current_df = gene_result_grouped.get_group(f"{FI.hash}_{n}") # sort by bitscore - # sorting has peformed after split for better performance + # sorting is performed after split for better performance current_df.sort_values(by=["bitscore"], inplace=True, ascending=False) # reset index to easily get maximum current_df.reset_index(inplace=True, drop=True) - # get result stasifies over cutoff + # get result satisfies over cutoff cutoff_df = current_df[ current_df["bitscore"] > current_df["bitscore"][0] * cutoff ] gene_count = len(set(cutoff_df["gene"])) gene_list = list(set(cutoff_df["gene"])) - except: + except (KeyError, IndexError): + # no search hits for this query seq/gene (get_group raises KeyError) + # or an empty result frame -> treat as "no gene assignable" gene_count = 0 gene_list = [] @@ -109,19 +114,45 @@ def assign_gene(result_dict, V, cutoff=0.99): FI.update_seq(gene_list[0], seq) else: - logging.error("DEVELOPMENTAL ERROR IN GENE ASSIGN") - raise Exception + raise ClusterError( + f"unexpected gene_count={gene_count} while assigning a gene to {FI.id}" + ) return V -# This function assigns group to each FI -def cluster(FI, V_list_group, V_cSR, path, opt): - # Reduce memory by focusing on relevant rows - df_search = V_cSR[V_cSR["qseqid"] == FI.hash] +# Assign a query to a group when its best-hit cutoff set spans >=2 groups. +# Rule: count subject_groups among the hits TIED at the highest bitscore; the +# plurality wins. If that count ties, fold in the next (lower) bitscore level and +# recount, and so on. This respects the best-scoring hits and avoids the whole- +# cutoff-window bias where an over-represented distant group can outvote the real +# best match. cutoff_df must already be sorted by bitscore descending. +def _majority_top_group(cutoff_df): + counts = {} + leaders = [] + for _, level in cutoff_df.groupby("bitscore", sort=False): + for grp in level["subject_group"]: + counts[grp] = counts.get(grp, 0) + 1 + max_count = max(counts.values()) + leaders = [grp for grp, c in counts.items() if c == max_count] + if len(leaders) == 1: + return leaders[0] + # All bitscore levels folded in and still tied: take the highest-bitscore hit + # among the tied leaders (deterministic given the descending sort). + leader_set = set(leaders) + for grp in cutoff_df["subject_group"]: + if grp in leader_set: + return grp + +# This function assigns group to each FI +def cluster(FI, df_search, opt): + # df_search: this FI's pre-sliced rows of the concatenated search table + # (columns qseqid, bitscore, subject_group). Previously each call re-scanned the + # whole cSR (V_cSR[V_cSR["qseqid"] == FI.hash]) -> O(N_FI * N_rows), and the whole + # table was pickled to every mp worker; pipe_cluster now slices once via groupby. if df_search.empty: # If confident is False and FI datatype is "db" - if opt.confident is False and FI.datatype is "db": + if opt.confident is False and FI.datatype == "db": logging.warning(f"No adjusted_group assigned to {FI}") FI.adjusted_group = FI.group return FI @@ -139,10 +170,6 @@ def cluster(FI, V_list_group, V_cSR, path, opt): cutoff = df_search["bitscore"].iloc[0] * opt.cluster.cutoff cutoff_df = df_search[df_search["bitscore"] > cutoff] - # Clear unused data - del df_search - gc.collect() - # Extract group information unique_groups = set(cutoff_df["subject_group"]) group_count = len(unique_groups) @@ -159,12 +186,13 @@ def cluster(FI, V_list_group, V_cSR, path, opt): f"Query seq in {FI.id} has multiple matches to groups within {opt.cluster.cutoff} cutoff: {list(unique_groups)}" ) - # Among the multiple matches, get the best group - FI.adjusted_group = cutoff_df["subject_group"].iloc[0] + # Assign by group plurality among the best (tied-top-bitscore) hits + FI.adjusted_group = _majority_top_group(cutoff_df) else: - logging.error("DEVELOPMENTAL ERROR IN GROUP ASSIGN") - raise Exception + raise ClusterError( + f"unexpected group_count={group_count} while assigning a group to {FI.id}" + ) logging.info(f"{FI.id} has clustered to {FI.adjusted_group}") @@ -173,16 +201,18 @@ def cluster(FI, V_list_group, V_cSR, path, opt): if not (FI.adjusted_group in unique_groups): logging.warning(f"Clustering result collides for {FI.id}") - if unique_groups: - return FI - else: - return FI + return FI ### Append outgroup to given group-gene dataset by search matrix -def append_outgroup(V_list_FI, df_search, gene, group, path, opt): +# V.list_FI shared with outgroup-append pool workers via fork copy-on-write, set +# before the Pool is created, instead of being pickled into every task tuple. +_OUTGROUP_SHARED = {} + + +def append_outgroup(df_search, gene, group, path, opt): logging.info(f"Appending outgroup on group: {group}, Gene: {gene}") - list_FI = V_list_FI + list_FI = _OUTGROUP_SHARED["list_FI"] # In multiprocessing, delete V to reduce memory consumption # del V @@ -208,12 +238,14 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt): bitscore_cutoff = max( 1, min(cutoff_set_df["bitscore"]) - opt.cluster.outgroupoffset ) - except: - bitscore_cutoff = 999999 # use infinite if failed + except ValueError: + # cutoff_set_df empty (no ingroup hits for this group) -> min() raises; + # fall back to an effectively infinite cutoff so all hits stay candidates + bitscore_cutoff = 999999 # print(f"Ingroup cutoff {bitscore_cutoff} selected for group {group} gene {gene}") - ## get result stasifies over cutoff + ## get result satisfies over cutoff # outgroup should be outside of ingroup cutoff_df = df_search[df_search["bitscore"] < bitscore_cutoff] @@ -228,9 +260,15 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt): # For each of the input, should use different cutoff ambiguous_db = set() if opt.suspicious is True: + # Pre-group once instead of rescanning the whole (per-group) search table + # with a boolean mask for every qseqid (was O(n_queries * len(df_search))). + df_search_by_qseqid = df_search.groupby("qseqid") for qseqid, _df in cutoff_set_df.groupby(["qseqid"]): # Select dataframe corresponding to current qseqid - df_qseqid = df_search[df_search["qseqid"] == qseqid[0]] + try: + df_qseqid = df_search_by_qseqid.get_group(qseqid[0]) + except KeyError: + continue # Get the list of subjects, which is closer than furtest ingroup ambiguous_df = df_qseqid[ @@ -310,7 +348,7 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt): ) # Move outgroup within ambiguous db to outgroup - for FI in ambiguous_db: + for FI in list(ambiguous_db): if FI.group == subject_group: ambiguous_db.remove(FI) outgroup_dict[subject_group].append(FI) @@ -340,7 +378,7 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt): ) # Move outgroup within ambiguous db to outgroup - for FI in ambiguous_db: + for FI in list(ambiguous_db): if FI.group == max_group: ambiguous_db.remove(FI) outgroup_dict[max_group].append(FI) @@ -360,10 +398,9 @@ def group_cluster_opt_generator(V, opt, path): # cluster(FO, df_search, V, path, opt) if len(V.list_qr_gene) == 0: - logging.error( - "In group_cluster_option_generator, no available query genes were selected" + raise ClusterError( + "no query genes are available for clustering (V.list_qr_gene is empty)" ) - raise Exception # For concatenated analysis else: @@ -391,7 +428,7 @@ def group_cluster_opt_generator(V, opt, path): def outgroup_append_opt_generator(V, path, opt): opt_append_outgroup = [] - #### Pararellize this part + #### Parallelize this part # Assign different outgroup for each dataset # if concatenated analysis is true @@ -405,10 +442,10 @@ def outgroup_append_opt_generator(V, path, opt): # Generating outgroup opt for multiprocessing for gene in V.dict_dataset[group]: opt_append_outgroup.append( - (V.list_FI, df_group_, gene, group, path, opt) + (df_group_, gene, group, path, opt) ) - except: + except KeyError: logging.warning( f"{group} / concatenated dataset exists, but cannot append outgroup due to no corresponding search result" ) @@ -422,28 +459,20 @@ def pipe_cluster(V, opt, path): if opt.method.search in ("blast", "mmseqs"): logging.info("group clustering") - rslt_cluster = [] - - # cluster opt generation for multiprocessing - # (FI, V, path, opt) - opt_cluster = group_cluster_opt_generator(V, opt, path) - - # print(f"opt_cluster : {sys.getsizeof(opt_cluster)}") - - batch_size = opt.thread * 100 - opt_cluster_batches = batched_generator(opt_cluster, batch_size) - - # run multiprocessing start - if opt.verbose < 3: - for batch in opt_cluster_batches: - batch_list = list(batch) + # Assign each query's group from ONE in-process groupby over the concatenated + # search table. The former path scanned the whole cSR once per FI inside an + # mp.Pool that was handed the entire table per task -> O(N_FI * N_rows) plus + # repeated full-table pickling, which dominated the cluster step at scale. + hash_to_FI = {FI.hash: FI for FI in V.list_FI} + cSR_subset = V.cSR[["qseqid", "bitscore", "subject_group"]] - with mp.Pool(opt.thread) as p: - rslt_cluster.extend(p.starmap(cluster, batch_list)) + rslt_cluster = [] + for qseqid, df_search in cSR_subset.groupby("qseqid", sort=False): + FI = hash_to_FI.get(qseqid) + if FI is None: + continue + rslt_cluster.append(cluster(FI, df_search, opt)) - else: - # non-multithreading mode for debugging - rslt_cluster = [cluster(*o) for o in opt_cluster] # gather cluster result for cluster_result in rslt_cluster: FI = cluster_result @@ -509,6 +538,10 @@ def pipe_cluster(V, opt, path): def pipe_append_outgroup(V, path, opt): opt_append_outgroup = outgroup_append_opt_generator(V, path, opt) + # Share V.list_FI with pool workers via fork copy-on-write (set before the Pool) + # instead of pickling the whole list into every task. + _OUTGROUP_SHARED["list_FI"] = V.list_FI + # run multiprocessing start if opt.verbose < 3: p = mp.Pool(opt.thread) @@ -533,11 +566,12 @@ def pipe_append_outgroup(V, path, opt): # Add outgroup and ambiguous groups to dataset # Ambiguous groups are strains locating between outgroup and ingroups, so cannot be decided - print(f"{group} {gene}") - print(f"outgroup {len(outgroup)}") - print(f"ambiguous_group {len(ambiguous_group)}") - print(f"db: {len(V.dict_dataset[group][gene].list_db_FI)}") - print(f"query: {len(V.dict_dataset[group][gene].list_qr_FI)}") + logging.debug( + f"Outgroup selection for {group} {gene}: outgroup={len(outgroup)}, " + f"ambiguous={len(ambiguous_group)}, " + f"db={len(V.dict_dataset[group][gene].list_db_FI)}, " + f"query={len(V.dict_dataset[group][gene].list_qr_FI)}" + ) if len(outgroup) == 0 and len(ambiguous_group) == 0: logging.critical( @@ -576,11 +610,15 @@ def pipe_append_outgroup(V, path, opt): ) critical_flag = 1 V.dict_dataset.pop(group, None) - except: + except KeyError: + # group already removed above / not present; nothing to do pass # Terminate if terminate option is given, and critical error occurs if critical_flag == 1 and opt.terminate is True: - raise Exception + raise ClusterError( + "stopping: one or more group/gene datasets could not be built " + "(see the CRITICAL messages above); --terminate is set" + ) return V, path, opt diff --git a/funvip/src/command.py b/funvip/src/command.py index 4694843..60dfa60 100644 --- a/funvip/src/command.py +++ b/funvip/src/command.py @@ -54,7 +54,7 @@ def get_args(self) -> argparse.Namespace: ) group_test.add_argument( "--test", - help="Use test dataset, [Penicillium, Terrei, Sanghuangporus]", + help="Run on a bundled test dataset, one of [penicillium, terrei, sanghuangporus]. Requires --email for the bundled GenBank accessions.", type=str, ) @@ -84,7 +84,7 @@ def get_args(self) -> argparse.Namespace: ) group_run.add_argument( "--mode", - help="Mode setup in species identification, see documents for detailed explanations, [validation, identification] default : validation", + help="Analysis mode, one of [identification, validation]. default : identification", type=str, ) group_run.add_argument( @@ -95,12 +95,12 @@ def get_args(self) -> argparse.Namespace: ) group_run.add_argument( "--step", - help="[WIP] Steps to continue from previous run, will be ignored if invalid --continue option [setup, search, cluster, align, trim, concatenate, modeltest, tree, visualize, report]", + help="With --continue, resume from this pipeline step, one of [setup, search, cluster, align, trim, concatenate, modeltest, tree, visualize, report]. Ignored without a valid --continue.", type=str, ) group_run.add_argument( "--level", - help="Taxonomic level for each phylogenetic tree. Should be one of [subseries, series, subsection, section, subtribe, tribe, subfamily, family, suborder, order, subclass, class, subphylum, phylum, subdivision, division, subkingdom, kingdom]", + help="Taxonomic level at which each phylogenetic tree is built, one of [genus, subseries, series, subsection, section, subtribe, tribe, subfamily, family, suborder, order, subclass, class, subphylum, phylum, subdivision, division, subkingdom, kingdom]. default : genus", type=str, ) group_run.add_argument( @@ -119,7 +119,7 @@ def get_args(self) -> argparse.Namespace: ) group_method.add_argument( "--search", - help="Search methods to be used in selecting genes, groups and outgroups, [blast, mmseqs], default : mmseqs", + help="Search method for selecting genes, groups and outgroups, one of [blast, mmseqs]. default : blast", type=str, ) group_method.add_argument( @@ -176,7 +176,7 @@ def get_args(self) -> argparse.Namespace: group_visualize.add_argument( "--backgroundcolor", - help='List of background colors to be shown in tree, default: #f4f4f4, #c6c6c6. Input should be used with quotes, delimit with spaces and recommended to be used as hex codes. To remove background, use --backgroundcolor "#FFFFFF" "#FFFFFF" ', + help='Alternating background band colors in the tree, space-delimited hex codes in quotes. default: "#ffe0e0" "#ffefef". To remove the background use --backgroundcolor "#FFFFFF" "#FFFFFF".', nargs="*", type=str, ) @@ -193,7 +193,7 @@ def get_args(self) -> argparse.Namespace: ) group_visualize.add_argument( "--fsize", - help="Font size to use for phylogenetic tree, default: 10", + help="Font size for phylogenetic tree labels, default: 14", type=float, ) group_visualize.add_argument( @@ -214,7 +214,8 @@ def get_args(self) -> argparse.Namespace: ) group_run.add_argument( "--maxoutgroup", - help="Maximum outgroup numbers to include in phylogenetic analysis, default : 1", + help="Maximum number of outgroup sequences to include in each phylogenetic tree, default : 3", + type=int, ) group_advanced.add_argument( "--collapsedistcutoff", @@ -223,12 +224,12 @@ def get_args(self) -> argparse.Namespace: ) group_advanced.add_argument( "--collapsebscutoff", - help="Minimum bootstrap to be considered as same species, default : 100", + help="Minimum bootstrap support to collapse a clade as one species; the default 101 (above the 100 maximum) disables bootstrap-based collapsing. default : 101", type=float, ) group_advanced.add_argument( "--bootstrap", - help="Boostrap number for tree analysis, will be ignored if fasttree is selected for tree method, default : 1000", + help="Bootstrap replicates for tree analysis (ignored when tree method is fasttree). default : 100 (the accurate preset uses 1000)", type=int, ) group_advanced.add_argument( @@ -252,7 +253,7 @@ def get_args(self) -> argparse.Namespace: group_advanced.add_argument( "--cluster-evalue", dest="cluster_evalue", - help="E-value cutoffs for blast/mmseqs search, default : 0.0000001", + help="E-value cutoff for the blast/mmseqs clustering search; an explicit value overrides the preset. default : 0.0001", type=float, ) group_advanced.add_argument( @@ -271,7 +272,8 @@ def get_args(self) -> argparse.Namespace: group_advanced.add_argument( "--mafft-algorithm", dest="mafft_algorithm", - help="MAFFT algorithm for alignment, see mafft documents, will be ignored if mafft not selected for alignment option, default : auto", + help="MAFFT algorithm for alignment (see MAFFT docs). default : auto", + type=str, ) group_advanced.add_argument( "--mafft-op", @@ -288,10 +290,11 @@ def get_args(self) -> argparse.Namespace: group_advanced.add_argument( "--trimal-algorithm", dest="trimal_algorithm", - help="Trimal algorithm for trimming, see trimal documents, will be ignored if trimal not selected for trimming option, default : gt", + help="trimAl algorithm for trimming (see trimAl docs). default : gt", + type=str, ) group_advanced.add_argument( - "--trimal-gt", dest="trimal_gt", help="gt value for trimal", type=float + "--trimal-gt", dest="trimal_gt", help="trimAl -gt (gap threshold), between 0 and 1. default : 0.2", type=float ) group_advanced.add_argument( "--allow-innertrimming", @@ -301,7 +304,7 @@ def get_args(self) -> argparse.Namespace: ) group_advanced.add_argument( "--criterion", - help="Modeltest criterion to use, either AIC, AICc or BIC", + help="Model-selection criterion for modeltest, one of [AIC, AICc, BIC]. default : BIC", type=str, ) diff --git a/funvip/src/concatenate.py b/funvip/src/concatenate.py index 91d53ce..0213d26 100644 --- a/funvip/src/concatenate.py +++ b/funvip/src/concatenate.py @@ -7,6 +7,7 @@ import pandas as pd from copy import deepcopy from funvip.src import search, hasher +from funvip.src.exceptions import DatasetError from scipy.optimize import minimize import numpy as np import shutil @@ -121,11 +122,10 @@ def combine_alignment(V, opt, path): } else: - logging.error(f"V.dict_dataset {group}: {V.dict_dataset[group]}") - logging.error( - f"[DEVELOPMENTAL ERROR] Failed constructing concatenated alignment for {group}" + raise DatasetError( + f"cannot build concatenated alignment for group {group}: " + f"no gene datasets besides 'concatenated' (got {list(V.dict_dataset[group])})" ) - raise Exception else: logging.warning( @@ -269,23 +269,33 @@ def objective(x): ) def vectorized_prediction(df, gene_list, coeff, grad): - for k, gene in enumerate(gene_list): - linear_constant = (coeff[k] - df[f"{gene}_bitscore"]) / grad[k] - mean_linear_constant = linear_constant.mean() - - # Fill missing bitscores - """ - df[f"{gene}_bitscore"].fillna( - coeff[k] - mean_linear_constant * grad[k], inplace=True - ) - """ - df.fillna( - {f"{gene}_bitscore": coeff[k] - mean_linear_constant * grad[k]}, - inplace=True, - ) + # Fill a row's missing gene bitscore by projecting the row's KNOWN gene + # bitscores onto the fitted regression line (X_i = coeff_i - t*grad_i) and + # reading off the missing coordinate. The former version used each gene's + # column mean, which reduces algebraically to a single constant per gene + # and ignores the row's other genes entirely. t is the least-squares line + # parameter over the row's present dimensions: + # t = -sum_j grad_j (X_j - coeff_j) / sum_j grad_j^2 (j = present genes) + cols = [f"{gene}_bitscore" for gene in gene_list] + coeff_a = np.asarray(coeff, dtype=float) + grad_a = np.asarray(grad, dtype=float) + X = df[cols].to_numpy(dtype=float) + present = ~np.isnan(X) + a = X - coeff_a # NaN where missing + grad_row = np.broadcast_to(grad_a, X.shape) + num = np.sum(np.where(present, grad_row * a, 0.0), axis=1) + den = np.sum(np.where(present, grad_row**2, 0.0), axis=1) + with np.errstate(divide="ignore", invalid="ignore"): + t = np.where(den > 0, -num / den, 0.0) + pred = coeff_a[None, :] - t[:, None] * grad_a[None, :] + # Bitscores are non-negative; guard a line that extrapolates below zero. + pred = np.clip(pred, 0.0, None) + X_filled = np.where(present, X, pred) + for i, col in enumerate(cols): + df[col] = X_filled[:, i] return df - # Change to numpy for faster cazlculation + # Change to numpy for faster calculation np_bitscore = df_multigene_regression[ [f"{gene}_bitscore" for gene in gene_list] ].to_numpy() @@ -324,6 +334,14 @@ def vectorized_prediction(df, gene_list, coeff, grad): ].mean(axis=1) V.cSR = df_multigene_regression.reset_index() + # Dictionary-encode the repeated hash / group string columns of the + # concatenated search table (which persists for the rest of the run) to cut + # its memory footprint (category is ~8-29x smaller than object/str on these + # columns); the stored values are unchanged. + for _col in V.cSR.columns: + if not pd.api.types.is_numeric_dtype(V.cSR[_col]): + V.cSR[_col] = V.cSR[_col].astype("category") + # Save it # decode df is not working well here if opt.nosearchresult is False: @@ -333,4 +351,9 @@ def vectorized_prediction(df, gene_list, coeff, grad): fmt=opt.tableformat, ) + # The per-gene search tables are only needed to build V.cSR above; drop them so + # they do not stay resident (and get re-pickled into every later session + # checkpoint) for the rest of the run. + V.dict_gene_SR = {} + return V diff --git a/funvip/src/dataset.py b/funvip/src/dataset.py index d483749..fc55aca 100644 --- a/funvip/src/dataset.py +++ b/funvip/src/dataset.py @@ -2,6 +2,7 @@ from funvip.src import hasher from funvip.src import ext from funvip.src.opt_generator import opt_generator +from funvip.src.exceptions import ConfigError, DatasetError from Bio import SeqIO import os import sys @@ -154,7 +155,7 @@ def exist_dataset(self, group, gene): try: self.dict_dataset[group][gene] return True - except: + except KeyError: return False # Check if dict_group has been properly generated @@ -199,67 +200,50 @@ def check_dict_group(self, opt): ) else: - logging.error(f"DEVELOPMENTAL ERROR, UNEXPECTED LEVEL {opt.level} selected") - raise Exception + raise ConfigError(f"unexpected taxonomic level {opt.level!r} (check --level)") # generate dataset by group and gene def generate_dataset(self, opt): # Format : dict_funinfo = {group: {gene : [FI]}} dict_funinfo = {} + # Pre-index FIs by adjusted_group in a single pass (preserving list_FI + # order) so the per-group/per-gene loops below filter a small bucket + # instead of rescanning the whole FI universe each time -- the former was + # O(N_groups * N_genes * N_FI) and dominated wall-clock at metabarcoding + # scale. Order is preserved, so datasets are built identically. + group_query = {} + group_db = {} + for FI in self.list_FI: + if FI.datatype == "query": + group_query.setdefault(FI.adjusted_group, []).append(FI) + elif FI.datatype == "db": + group_db.setdefault(FI.adjusted_group, []).append(FI) + for group in self.list_group: logging.info(f"Generating dataset for {group}") - # print(f"opt.queryonly: {opt.queryonly}") - dict_funinfo[group] = {} + group_query_FI = group_query.get(group, []) + group_db_FI = group_db.get(group, []) + # For queryonly case if opt.queryonly is True: - # whether to run this group - group_flag = False - for gene in self.list_db_gene: - logging.debug( - f"Searching dataset {group} {gene} includes query sequences" - ) - list_qr = [ - FI - for FI in self.list_FI - if ( - gene in FI.seq - and FI.datatype == "query" - and FI.adjusted_group == group - ) - ] - - # do not manage db when --queryonly True (--all False) and query does not exists - if len(list_qr) > 0: - group_flag = True + # whether to run this group: any query in this group carries any + # of the target genes + group_flag = any( + gene in FI.seq + for gene in self.list_db_gene + for FI in group_query_FI + ) # if decided to run this group if group_flag is True: logging.info(f"Decided to construct dataset on {group}") for gene in self.list_db_gene: - list_qr = [ - FI - for FI in self.list_FI - if ( - gene in FI.seq - and FI.datatype == "query" - and FI.adjusted_group == group - ) - ] - - list_db = [ - FI - for FI in self.list_FI - if ( - gene in FI.seq - and FI.datatype == "db" - and FI.adjusted_group == group - ) - ] - + list_qr = [FI for FI in group_query_FI if gene in FI.seq] + list_db = [FI for FI in group_db_FI if gene in FI.seq] self.add_dataset(group, gene, list_qr, list_db, []) else: @@ -268,43 +252,26 @@ def generate_dataset(self, opt): ) # for concatenated - list_qr = [ - FI - for FI in self.list_FI - if (FI.datatype == "query" and FI.adjusted_group == group) - ] + list_qr = list(group_query_FI) # do not manage db when query only mode and query does not exists if len(list_qr) > 0: - list_db = [ - FI - for FI in self.list_FI - if (FI.datatype == "db" and FI.adjusted_group == group) - ] + list_db = list(group_db_FI) self.add_dataset(group, "concatenated", list_qr, list_db, []) # For opt.queryonly is False -> run all dataset in database if possible else: for gene in self.list_db_gene: - list_qr = [] - for FI in self.list_FI: - if ( - FI.datatype == "query" - and FI.adjusted_group == group - and gene in FI.seq - ): - if FI.seq[gene] != "": - list_db.append(FI) - - list_db = [] - for FI in self.list_FI: - if ( - FI.datatype == "db" - and FI.adjusted_group == group - and gene in FI.seq - ): - if FI.seq[gene] != "": - list_db.append(FI) + list_qr = [ + FI + for FI in group_query_FI + if gene in FI.seq and FI.seq[gene] != "" + ] + list_db = [ + FI + for FI in group_db_FI + if gene in FI.seq and FI.seq[gene] != "" + ] # If none of the database is possible for this group and gene pair, it should be excluded if len(list_db) > 0: @@ -315,16 +282,8 @@ def generate_dataset(self, opt): ) # for concatenated - list_qr = [ - FI - for FI in self.list_FI - if (FI.datatype == "query" and FI.adjusted_group == group) - ] - list_db = [ - FI - for FI in self.list_FI - if (FI.datatype == "db" and FI.adjusted_group == group) - ] + list_qr = list(group_query_FI) + list_db = list(group_db_FI) self.add_dataset(group, "concatenated", list_qr, list_db, []) self.check_dict_group(opt) @@ -344,10 +303,11 @@ def homogenize_dataset(self): self.dict_hash_FI[h].final_species = FI.final_species # If they collides, it is error else: - logging.error( - f"DEVELOPMNETAL ERROR Both list_FI and dict_hash_FI have conflicting final species, {FI.final_species} and {self.dict_hash_FI[h].final_species} for hash {h}" + raise DatasetError( + f"conflicting final_species for hash {h}: " + f"{FI.final_species!r} (list_FI) vs " + f"{self.dict_hash_FI[h].final_species!r} (dict_hash_FI)" ) - raise Exception # adjusted group if FI.adjusted_group != self.dict_hash_FI[h].adjusted_group: @@ -356,10 +316,11 @@ def homogenize_dataset(self): elif self.dict_hash_FI[h].adjusted_group == "": self.dict_hash_FI[h].adjusted_group = FI.adjusted_group else: - logging.error( - f"DEVELOPMENTAL ERROR Both list_FI and dict_hash_FI have conflicting final group, {FI.adjusted_group} and {self.dict_hash_FI[h].adjusted_group}, {FI}" + raise DatasetError( + f"conflicting adjusted_group for {FI.id}: " + f"{FI.adjusted_group!r} (list_FI) vs " + f"{self.dict_hash_FI[h].adjusted_group!r} (dict_hash_FI)" ) - raise Exception elif ( FI.adjusted_group == "" @@ -384,13 +345,14 @@ def homogenize_dataset(self): pass elif not (FI.bygene_species): FI.bygene_species = self.dict_hash_FI[h].bygene_species - elif self.dict_hash_FI[h].bygene_species: + elif not (self.dict_hash_FI[h].bygene_species): self.dict_hash_FI[h].bygene_species = FI.bygene_species else: - logging.error( - f"DEVELOPMENTAL ERROR Both list_FI and dict_hash_FI have conflicting gene identification results, {FI.bygene_species} and {self.dict_hash_FI[h].bygene}" + raise DatasetError( + f"conflicting bygene_species for hash {h}: " + f"{FI.bygene_species!r} vs " + f"{self.dict_hash_FI[h].bygene_species!r}" ) - raise Exception # Remove invalid dataset to be analyzed def remove_invalid_dataset(self): @@ -448,13 +410,12 @@ def save_dataset(self, path, opt): ) # Remove no seqs - for fasta in fasta_list: - save.save_fasta( - fasta_list, - gene, - f"{path.out_adjusted}/{opt.runname}_Adjusted_{group}_{gene}.fasta", - by="hash", - ) + save.save_fasta( + fasta_list, + gene, + f"{path.out_adjusted}/{opt.runname}_Adjusted_{group}_{gene}.fasta", + by="hash", + ) # Validate if any multiple sequence alignment has no overlapping region def validate_alignments(self, path, opt): @@ -475,11 +436,11 @@ def validate_alignments(self, path, opt): f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta" ) ): - logger.warning( + logging.warning( f"Alignment file {path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta does not exists" ) - fail_list.append(group_gene) + fail_list.append((group, gene)) else: # If alignment exists, check if alignment does have overlapping regions @@ -493,17 +454,16 @@ def validate_alignments(self, path, opt): # Remove sequences that has not been existed during alignment stage seq_id_list = [seq.id for seq in seq_list] - for FI in self.dict_dataset[group][gene].list_db_FI: - if not (FI.hash in seq_id_list): - self.dict_dataset[group][gene].list_db_FI.remove(FI) - - for FI in self.dict_dataset[group][gene].list_qr_FI: - if not (FI.hash in seq_id_list): - self.dict_dataset[group][gene].list_qr_FI.remove(FI) - - for FI in self.dict_dataset[group][gene].list_og_FI: - if not (FI.hash in seq_id_list): - self.dict_dataset[group][gene].list_og_FI.remove(FI) + dataset = self.dict_dataset[group][gene] + dataset.list_db_FI = [ + FI for FI in dataset.list_db_FI if FI.hash in seq_id_list + ] + dataset.list_qr_FI = [ + FI for FI in dataset.list_qr_FI if FI.hash in seq_id_list + ] + dataset.list_og_FI = [ + FI for FI in dataset.list_og_FI if FI.hash in seq_id_list + ] # Remove empty sequences remove_hash = [] @@ -619,7 +579,10 @@ def validate_alignments(self, path, opt): # Terminate if terminate option is given, and critical error occurs if critical_flag == 1 and opt.terminate is True: - raise Exception + raise DatasetError( + "stopping: one or more datasets failed alignment validation " + "(see the CRITICAL messages above); --terminate is set" + ) # Remove bad datasets for fail in fail_list: @@ -647,27 +610,19 @@ def validate_alignments(self, path, opt): if group in self.dict_dataset: if gene in self.dict_dataset[group]: for _hash in remove_dict[group][gene]: - if _hash in self.dict_dataset[group][gene].list_qr_FI: - self.dict_dataset[group][gene].list_qr_FI.remove( - self.dict_hash_FI[_hash] - ) - logging.warning( - f"{self.dict_hash_ID[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" - ) - if _hash in self.dict_dataset[group][gene].list_db_FI: - self.dict_dataset[group][gene].list_db_FI.remove( - self.dict_hash_FI[_hash] - ) - logging.warning( - f"{self.dict_hash_ID[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" - ) - if _hash in self.dict_dataset[group][gene].list_og_FI: - self.dict_dataset[group][gene].list_og_FI.remove( - self.dict_hash_FI[_hash] - ) - logging.warning( - f"{self.dict_hash_ID[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" + for _attr in ("list_qr_FI", "list_db_FI", "list_og_FI"): + _lst = getattr( + self.dict_dataset[group][gene], _attr ) + if any(x.hash == _hash for x in _lst): + setattr( + self.dict_dataset[group][gene], + _attr, + [x for x in _lst if x.hash != _hash], + ) + logging.warning( + f"{self.dict_hash_id[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" + ) # Finally, check again if the datasets meet criteria final_fail_list = [] @@ -745,49 +700,60 @@ def validate_alignments(self, path, opt): if opt.method.tcs is True: bad_cnt = 0 - if opt.verbose < 3: - tcs_opt = opt_generator( - V=self, opt=opt, path=path, step="tcs", thread=1 + # TCS is optional alignment QC; a broken/leaking t-coffee must not abort + # the whole run. Log and skip on failure (the memory cap in ext.TCS keeps + # a leaking t-coffee from consuming the machine). + try: + if opt.verbose < 3: + tcs_opt = opt_generator( + V=self, opt=opt, path=path, step="tcs", thread=1 + ) + with mp.Pool(opt.thread) as p: + p.starmap(ext.TCS, tcs_opt) + + else: + tcs_opt = opt_generator(V=self, opt=opt, path=path, step="tcs") + for option in tcs_opt: + ext.TCS(*option) + except Exception as e: + logging.warning( + f"TCS alignment validation failed and was skipped: {e}" ) - p = mp.Pool(opt.thread) - p.starmap(ext.TCS, tcs_opt) - p.close() - p.join() - else: - tcs_opt = opt_generator(V=self, opt=opt, path=path, step="tcs") - for option in tcs_opt: - ext.TCS(*option) - - # non-multithreading mode for debugging - for group in self.dict_dataset: - for gene in self.dict_dataset[group]: - # Running TCS for concatenated alignment is duplicate - if gene != "concatenated": - tcs_out = f"{path.out_alignment}/alignment/{opt.runname}_{group}_{gene}.tcs" - # Parse tcs result - with open(tcs_out, "r") as f_tcs: - tcs_result_raw = f_tcs.read() - tcs_result = tcs_result_raw.split("*")[2].split("cons")[ - 0 - ] - for line in tcs_result.split("\n")[1:-1]: - _hash = line.split(":")[0].strip() - tcs_score = int(line.split(":")[1].strip()) - if ( - tcs_score < 50 - ): # cutoff 50 comes from TCS documentation - FI_id = self.dict_hash_FI[_hash].id - logging.warning( - f"{FI_id} has poor alignment score in {group} {gene}" - ) - bad_cnt += 1 + # Parse whatever TCS score files were produced. A dataset whose TCS run + # failed leaves no score file, so skip missing/unparseable ones instead + # of crashing the whole run. + for group in self.dict_dataset: + for gene in self.dict_dataset[group]: + # Running TCS for concatenated alignment is duplicate + if gene == "concatenated": + continue + tcs_out = f"{path.out_alignment}/alignment/{opt.runname}_{group}_{gene}.tcs" + if not os.path.exists(tcs_out): + continue + try: + with open(tcs_out, "r") as f_tcs: + tcs_result_raw = f_tcs.read() + tcs_result = tcs_result_raw.split("*")[2].split("cons")[0] + for line in tcs_result.split("\n")[1:-1]: + _hash = line.split(":")[0].strip() + tcs_score = int(line.split(":")[1].strip()) + if tcs_score < 50: # cutoff 50 from TCS documentation + FI_id = self.dict_hash_FI[_hash].id + logging.warning( + f"{FI_id} has poor alignment score in {group} {gene}" + ) + bad_cnt += 1 + except Exception as e: + logging.warning( + f"Could not parse TCS score file {tcs_out}: {e}" + ) if bad_cnt == 0: logging.info(f"All sequences in alignment passed TCS validation") else: logging.warning( - f"{bad_cnt} sequences in alignment failed TCS validation. Please check sequneces" + f"{bad_cnt} sequences in alignment failed TCS validation. Please check sequences" ) # check inconsistency exists along identification result of each genes diff --git a/funvip/src/exceptions.py b/funvip/src/exceptions.py new file mode 100644 index 0000000..d67fc19 --- /dev/null +++ b/funvip/src/exceptions.py @@ -0,0 +1,46 @@ +"""FunVIP exception hierarchy. + +Every error FunVIP raises on purpose derives from :class:`FunVIPError`, so the +top-level handler in ``main`` (and any caller) can tell an *expected, explained* +FunVIP failure apart from an unexpected bug and report it cleanly. + +Guidance: +- Raise the most specific subclass that fits, with a message that names the + offending thing (group, gene, FI id, file path, option) -- never a bare + ``raise Exception``. +- Reserve a plain ``FunVIPError`` for cases that do not fit a subclass. +- Do not use these for internal control flow; they mean "stop and tell the user". +""" + + +class FunVIPError(Exception): + """Base class for all FunVIP errors.""" + + +class ConfigError(FunVIPError): + """Invalid or inconsistent command-line options / configuration.""" + + +class InputError(FunVIPError): + """Malformed or missing query/database input (sequences, tables, metadata).""" + + +class SearchError(FunVIPError): + """Failure in the BLAST/mmseqs search stage or its result matrices.""" + + +class ClusterError(FunVIPError): + """Failure assigning sequences to groups or selecting an outgroup.""" + + +class DatasetError(FunVIPError): + """Failure building a group/gene dataset or its (concatenated) alignment.""" + + +class TreeError(FunVIPError): + """Failure building, rooting, or interpreting a phylogenetic tree.""" + + +class ExternalToolError(FunVIPError): + """An external tool (mafft, trimal, fasttree, iqtree, raxml, mmseqs, gblocks, + modeltest, ...) was missing, failed, or returned an unusable result.""" diff --git a/funvip/src/ext.py b/funvip/src/ext.py index 9c37c7c..b42b7a0 100644 --- a/funvip/src/ext.py +++ b/funvip/src/ext.py @@ -9,6 +9,7 @@ from pathlib import Path from funvip.src.save import save_tree from funvip.src.tool import mkdir +from funvip.src.exceptions import ExternalToolError, SearchError # Search methods @@ -31,6 +32,8 @@ def blast(query, db, out, path, opt): logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise SearchError(f"blastn failed (exit code {Run}) for query {query} vs db {db}") # mmseqs @@ -52,6 +55,8 @@ def mmseqs(query, db, out, tmp, path, opt): logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise SearchError(f"mmseqs easy-search failed (exit code {Run}) for query {query} vs db {db}") # DB building methods @@ -81,7 +86,9 @@ def makeblastdb(fasta, db, path): CMD = f"{path_makeblastdb} -in {fasta_tmp} -blastdb_version 4 -title {db_tmp} -dbtype nucl" logging.info(CMD) # I cannot find any "quiet" options for makeblastdb - Run = subprocess.call(CMD, stdout=open(os.devnull, "wb"), shell=True) + Run = subprocess.call(CMD, stdout=subprocess.DEVNULL, shell=True) + if Run != 0: + raise ExternalToolError(f"makeblastdb failed (exit code {Run}) for {fasta}") # Change db names shutil.move(fasta_tmp + ".nsq", db + ".nsq") shutil.move(fasta_tmp + ".nin", db + ".nin") @@ -95,11 +102,10 @@ def makeblastdb(fasta, db, path): CMD = f"makeblastdb -in '{fasta}' -blastdb_version 4 -title '{db}' -dbtype nucl" logging.info(CMD) # I cannot find any "quiet" options for makeblastdb - return_code = subprocess.call(CMD, stdout=open(os.devnull, "wb"), shell=True) + return_code = subprocess.call(CMD, stdout=subprocess.DEVNULL, shell=True) if return_code != 0: - logging.error(f"Make blast_db failed!!") - install_flag = 1 + raise ExternalToolError(f"makeblastdb failed (exit code {return_code}) for {fasta}") # Change db names shutil.move(fasta + ".nsq", db + ".nsq") @@ -122,6 +128,8 @@ def makemmseqsdb(fasta, db, path): CMD = f"mmseqs createdb '{fasta}' '{db}' --createdb-mode 0 --dbtype 2" logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError(f"mmseqs createdb failed (exit code {Run}) for {fasta}") # Alignments @@ -155,11 +163,9 @@ def MAFFT( CMD = f"mafft --thread {thread} --{algorithm} --maxiterate {maxiterate} --{adjust} --op {op} --ep {ep} --quiet '{fasta}' > '{out}'" logging.info(CMD) - try: - Run = subprocess.call(CMD, shell=True) - except: - logging.error(f"Failed on {CMD}") - raise Exception + Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError(f"MAFFT failed (exit code {Run}) for {fasta}") # Trimming @@ -176,37 +182,40 @@ def Gblocks(fasta, out, path): try: shutil.move(f"{fasta}.gb", out) - except: # when only one sequence and Gblocks failed + except FileNotFoundError: # when only one sequence and Gblocks failed shutil.move(fasta, out) - # Parse and return column statistics - with open(f"{fasta}.gb.txt", "r") as f: - lines = f.readlines() - for line in lines: - if line.startswith("Flanks:"): - flank_log = line - flank_log = ( - flank_log.replace("Flanks:", "") - .replace(" ", " ") - .replace("[", "") - .replace("]", "") - .strip() - ) - flank_log = flank_log.split(" ") - print(flank_log) - try: - flank_log = [int(x) for x in flank_log] - start_pos = flank_log[0] - end_pos = flank_log[-1] - 1 - except: - start_pos = -1 - end_pos = -1 - - print(f"start_pos: {start_pos}, end_pos: {end_pos}") + # Parse and return column statistics. (Gblocks returns a nonzero exit code even + # on success, so its exit code is intentionally not checked.) Default to the + # (-1, -1) failure sentinel so a missing/format-drifted .gb.txt cannot NameError. + start_pos = -1 + end_pos = -1 + if os.path.exists(f"{fasta}.gb.txt"): + with open(f"{fasta}.gb.txt", "r") as f: + for line in f: + if line.startswith("Flanks:"): + flank_log = ( + line.replace("Flanks:", "") + .replace(" ", " ") + .replace("[", "") + .replace("]", "") + .strip() + .split(" ") + ) + logging.debug(f"Gblocks flanks for {fasta}: {flank_log}") + try: + flank_log = [int(x) for x in flank_log] + start_pos = flank_log[0] + end_pos = flank_log[-1] - 1 + except (ValueError, IndexError): + start_pos = -1 + end_pos = -1 + + logging.debug(f"Gblocks start_pos: {start_pos}, end_pos: {end_pos}") try: shutil.move(f"{fasta}.gb.txt", path.extlog) - except: + except FileNotFoundError: pass return (start_pos, end_pos) @@ -229,10 +238,12 @@ def Trimal(fasta, out, path, algorithm="gt", threshold=0.2): CMD = f"{path.sys_path}/external/trimal.v1.4/trimAl/bin/trimal.exe -in {fasta} -out {out_dir} -{algorithm} -terminalonly -colnumbering > {out_colnumbering}" else: - CMD = f"trimal -in {fasta} -out {out} -{algorithm} -terminalonly -colnumbering > {out}.colnumbering" + CMD = f"trimal -in '{fasta}' -out '{out}' -{algorithm} -terminalonly -colnumbering > '{out}.colnumbering'" logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0 or not os.path.exists(out): + raise ExternalToolError(f"trimal failed (exit code {Run}) for {fasta}") # to remove unexpected hash included - maybe not needed after stabilization fasta_list = list(SeqIO.parse(out, "fasta")) @@ -252,13 +263,13 @@ def Trimal(fasta, out, path, algorithm="gt", threshold=0.2): cols = [int(x) for x in cols] start_pos = cols[0] end_pos = cols[-1] - except: + except (ValueError, IndexError): start_pos = -2 end_pos = -2 try: shutil.move(f"{out}.colnumbering", path.extlog) - except: + except FileNotFoundError: pass # Trimal uses 0 based position, return with +1 @@ -280,6 +291,8 @@ def Modeltest_ng(fasta, out, path, models, thread): logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError(f"modeltest-ng failed (exit code {Run}) for {fasta}") # IQTREE ModelFinder @@ -291,10 +304,9 @@ def ModelFinder(fasta, opt, path, thread): elif opt.method.tree == "fasttree": model_term = "-m MF --mset JC,JC+G4,GTR,GTR+G4" else: - logging.error( - f"Modelterm cannot be selected to tree method {opt.method.tree} while running modelfinder" + raise ExternalToolError( + f"cannot select a model term for tree method {opt.method.tree!r} in ModelFinder" ) - raise Exception if platform == "win32": if " " in fasta: @@ -305,6 +317,10 @@ def ModelFinder(fasta, opt, path, thread): CMD = f"iqtree --seqtype DNA -s '{fasta}' {model_term} -merit {opt.criterion} -nt AUTO -ntmax {thread} -mem {opt.memory} --quiet" logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError( + f"IQTREE ModelFinder failed (exit code {Run}) for {fasta}" + ) # Tree building @@ -322,68 +338,66 @@ def RAxML( if model == "skip": model = "" - # Because RAxML does not allows out location, change directory for running + # Because RAxML does not allow an out location, change directory for running. + # Wrap in try/finally so the original cwd is always restored, even if RAxML + # (or CMD selection) fails -- otherwise later relative-path operations break. path_ori = os.getcwd() os.chdir(path.tmp) + try: + if platform == "win32": + if " " in fasta: + fasta = f'"{fasta}"' + if " " in out: + out = f'"{out}"' - if platform == "win32": - if " " in fasta: - fasta = f'"{fasta}"' - if " " in out: - out = f'"{out}"' - - CMD = f"{path.sys_path}/external/RAxML_Windows/raxmlHPC-PTHREADS-AVX2.exe -s {fasta} -n {out} -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" - elif platform == "darwin": - # For Rossetta - - if ( - subprocess.run( - "raxmlHPC-PTHREADS -v", - shell=True, - stdout=open(os.devnull, "wb"), - stderr=subprocess.STDOUT, - ).returncode - == 0 - ): - CMD = f"raxmlHPC-PTHREADS -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}" - # For arm native - elif ( - subprocess.run( - "raxmlHPC -v", - shell=True, - stdout=open(os.devnull, "wb"), - stderr=subprocess.STDOUT, - ).returncode - == 0 - ): - CMD = f"raxmlHPC -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}" - else: - logging.error("Cannot find correct RAxML for apple silicon system!") - raise Exception + CMD = f"{path.sys_path}/external/RAxML_Windows/raxmlHPC-PTHREADS-AVX2.exe -s {fasta} -n {out} -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" + elif platform == "darwin": + # For Rosetta + if ( + subprocess.run( + "raxmlHPC-PTHREADS -v", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ).returncode + == 0 + ): + CMD = f"raxmlHPC-PTHREADS -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}" + # For arm native + elif ( + subprocess.run( + "raxmlHPC -v", + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ).returncode + == 0 + ): + CMD = f"raxmlHPC -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}" + else: + raise ExternalToolError( + "cannot find a working RAxML for this Apple Silicon system" + ) - else: - if version == "old": - CMD = f"raxmlHPC-PTHREADS-AVX -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" - elif version == "new": - CMD = f"raxmlHPC-PTHREADS-AVX2 -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" else: - logging.error( - f"DEVELOPMENTAL ERROR - unexpected RAxML version, {version} in ext.py" - ) - raise Exception + if version == "old": + CMD = f"raxmlHPC-PTHREADS-AVX -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" + elif version == "new": + CMD = f"raxmlHPC-PTHREADS-AVX2 -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model} --silent" + else: + raise ExternalToolError(f"unexpected RAxML version {version!r}") - if not (partition is None): - CMD += f" -q {partition}" + if not (partition is None): + CMD += f" -q {partition}" - logging.info(CMD) - Run = subprocess.call(CMD, shell=True) - - if Run != 0: - logging.error(f"RAxML Failed!") - raise Exception + logging.info(CMD) + Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError(f"RAxML failed (exit code {Run}) for {fasta}") + finally: + # Always restore the original working directory + os.chdir(path_ori) - # Return result to original directory - os.chdir(path_ori) file = out.split("/")[-1] out = f"RAxML_bipartitions.{out}" save_tree( @@ -418,6 +432,9 @@ def FastTree(fasta, out, hash_dict, path, model=""): logging.info(CMD) Run = subprocess.call(CMD, shell=True) + tree_file = f"{path.tmp}/{out}" + if Run != 0 or not os.path.exists(tree_file) or os.path.getsize(tree_file) == 0: + raise ExternalToolError(f"FastTree failed (exit code {Run}) for {fasta}") file = out.split("/")[-1] save_tree( out=f"{path.tmp}/{out}", @@ -470,18 +487,21 @@ def IQTREE( logging.info(CMD) Run = subprocess.call(CMD, shell=True) + if Run != 0: + raise ExternalToolError( + f"IQTREE failed (exit code {Run}) for {fasta}; refusing to use any " + "stale .contree left by a previous run" + ) try: if partition is None: shutil.move(f"{fasta}.contree", f"{path.tmp}/{out}") - print(f"DEBUG Moved {fasta}.contree to {path.tmp}/{out}") else: shutil.move(f"{partition}.contree", f"{path.tmp}/{out}") - print(f"DEBUG Moved {partition}.contree to {path.tmp}/{out}") - - except: - logging.error( - "IQTREE FAILED. Maybe due to memory problem if partitioned analysis included." - ) + except FileNotFoundError as e: + raise ExternalToolError( + f"IQTREE failed (exit code {Run}, no .contree produced) for {fasta}; " + "possibly a memory problem for partitioned analysis" + ) from e file = out.split("/")[-1] save_tree( @@ -495,8 +515,7 @@ def IQTREE( # TCS calculation from T-COFFEE def TCS(fasta, thread, out): if platform == "win32": - logging.error("TCS(From T-COFFEE) is only available in Linux") - raise Exception + raise ExternalToolError("TCS (from T-COFFEE) is only available on Linux") else: # T_COFFEE env variable MAX_N_PID_4_TCOFFEE should be changed for 64bit machine # should be already done in installation check process @@ -504,12 +523,52 @@ def TCS(fasta, thread, out): CMD = f"t_coffee -infile {fasta} -cpu {thread} -method fast_pair -type DNA -evaluate -output score_ascii -outfile {out} -quiet" logging.info(CMD) - # Even though "quiet" option exists, TCS show some blank lines - # Run = subprocess.call(CMD, stdout=open(os.devnull, "wb"), shell=True) - Run = subprocess.run( - CMD, stdout=open(os.devnull, "wb"), stderr=subprocess.STDOUT, shell=True - ).returncode + # Even though "quiet" option exists, TCS shows some blank lines. + # Root cause of the "t-coffee eats all RAM" problem: the bioconda t-coffee + # indexes a static array by the raw OS PID but is compiled with MAX_N_PID=260000. + # On kernels with a large kernel.pid_max (e.g. 4194304) the process PID overflows + # that array -> SIGSEGV -> t-coffee's own signal handler re-runs the faulting + # instruction in an infinite loop, growing the heap until all memory is gone. + # It is a crash-loop, not a real leak, and it fires on ANY invocation (even + # `t_coffee -version`). The documented MAX_N_PID_4_TCOFFEE override is not + # compiled into this binary, so it cannot be raised at runtime. We cannot fix a + # third-party binary here, so bound it: cap the child's address space and wall + # time and kill its whole process group on timeout, so a crash-looping t-coffee + # is reported as failed instead of taking down the machine. + import os + import signal + import resource + + _AS_CAP = 8 * 1024**3 # 8 GB address-space cap for the t-coffee subprocess + + def _limit_tcs_child(): + os.setsid() + try: + resource.setrlimit(resource.RLIMIT_AS, (_AS_CAP, _AS_CAP)) + except (ValueError, OSError): + pass + + proc = subprocess.Popen( + CMD, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + shell=True, + preexec_fn=_limit_tcs_child, + ) + try: + # Bound a hung/slow t-coffee. A leaking one is stopped sooner by the + # address-space cap above; this backstops a t-coffee that hangs at low + # memory. Legit TCS on normal alignments finishes well within this. + Run = proc.wait(timeout=300) + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + raise ExternalToolError( + "TCS (T-COFFEE) timed out (possible t-coffee memory leak/hang)" + ) if Run != 0: - logging.error(f"TCS Failed!") - raise Exception + raise ExternalToolError(f"TCS (T-COFFEE) failed (exit code {Run})") diff --git a/funvip/src/hasher.py b/funvip/src/hasher.py index ce70a54..50d58c7 100644 --- a/funvip/src/hasher.py +++ b/funvip/src/hasher.py @@ -65,25 +65,21 @@ def decode( content = fp.read() if newick and svg: - hash_dict = { - re.escape(k): svg_legal(newick_legal(v)) for k, v in hash_dict.items() - } + lookup = {k: svg_legal(newick_legal(v)) for k, v in hash_dict.items()} elif newick: - hash_dict = {re.escape(k): newick_legal(v) for k, v in hash_dict.items()} + lookup = {k: newick_legal(v) for k, v in hash_dict.items()} elif svg: - hash_dict = {re.escape(k): svg_legal(v) for k, v in hash_dict.items()} + lookup = {k: svg_legal(v) for k, v in hash_dict.items()} else: - hash_dict = {re.escape(k): v for k, v in hash_dict.items()} + lookup = dict(hash_dict) - """ - hash_dict = { - re.escape(k): (newick_legal(v) if newick else v) for k, v in hash_dict.items() - } - """ - pattern = re.compile("|".join(hash_dict.keys())) - - # Perform the substitution - decoded_content = pattern.sub(lambda m: hash_dict[re.escape(m.group(0))], content) + # Every hash has the fixed, self-delimiting form HSHE, so match that + # single token and look each occurrence up, instead of compiling an + # N-alternative regex ("|".join of every hash) and re-escaping each match -- + # the former is O(text * N_hashes) and a hotspot at metabarcoding scale. + # A HSHE token not present in the dict is left unchanged. + pattern = re.compile(r"HS\d+HE") + decoded_content = pattern.sub(lambda m: lookup.get(m.group(0), m.group(0)), content) with open(out, "w") as fw: fw.write(decoded_content) @@ -110,9 +106,9 @@ def decode(hash_dict: dict, file: str, out: str, newick: bool = True) -> None: # Decode given dataframe with given hash_dict def decode_df(hash_dict: dict, df: pd.DataFrame) -> pd.DataFrame: - hash_dict = dict((re.escape(k), v) for k, v in hash_dict.items()) - - df_return = copy.deepcopy(df) + # hashes are HSHE (no regex metacharacters), so no escaping is needed; + # each cell is either exactly a hash to replace or is left as-is. + df_return = df.copy() for column in df.columns: df_return[column] = df_return[column].map(lambda x: hash_dict.get(x, x)) diff --git a/funvip/src/logics.py b/funvip/src/logics.py index 8155909..2ccf8c8 100644 --- a/funvip/src/logics.py +++ b/funvip/src/logics.py @@ -1,5 +1,6 @@ import logging import re +import numpy as np from matplotlib import colors as mcolors # for color debugging # Move all simple logics function deciding True all False here @@ -15,8 +16,8 @@ def isnan(value) -> bool: def isvalidcolor(color: str) -> bool: colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS) - # Check if color is named colors - if color in colors.keys(): + # Check if color is named colors (CSS4 keys are lowercase; normalize input) + if color.lower() in colors: return True # Check if color is available hex color elif re.fullmatch(r"^#(?:[0-9a-fA-F]{3}){1,2}$", color): @@ -25,38 +26,6 @@ def isvalidcolor(color: str) -> bool: return False -def isnewicklegal(string: str) -> bool: - """ - if string is newick legal -> return True - if string is newick illegal -> return False - """ - NEWICK_ILLEGAL = ( - "(", - '"', - "[", - ":", - ";", - "/", - "[", - "]", - "{", - "}", - "(", - ")", - ",", - "]", - "+", - '"', - ")", - " ", - ) - - if any(x in string for x in NEWICK_ILLEGAL): - return True - else: - return False - - def isuniquecolumn( list_column: list, column: tuple, table_name: str, check_none: bool = True ) -> bool: diff --git a/funvip/src/modeltest.py b/funvip/src/modeltest.py index 3019518..ce8ac7b 100644 --- a/funvip/src/modeltest.py +++ b/funvip/src/modeltest.py @@ -340,10 +340,7 @@ def parse_model(modeltest_file, opt, path): ) break - # Currently using BIC model - model_cmd = model_dict["BIC"] - - except: + except Exception: logging.warning(f"Cannot parse modeltest file for {modeltest_file}") else: diff --git a/funvip/src/ncbi.py b/funvip/src/ncbi.py deleted file mode 100644 index f129fd6..0000000 --- a/funvip/src/ncbi.py +++ /dev/null @@ -1,160 +0,0 @@ -from Bio import Entrez -from Bio import SeqIO -from Bio.Blast.Applications import NcbiblastnCommandline -from Bio.Blast import NCBIXML -from Bio.Seq import Seq -import time -from time import sleep -import pickle -import json -import re -import logging -from datetime import datetime -from itertools import repeat -import xmltodict -import os, sys, subprocess -import random -import pickle -import pandas as pd -import shutil - -# from .logger import Mes - - -def Downloadbyacclist(Email, list_ID, out): - if len(list_ID) == 0: - return [] - - path_tmp = "tmp_download" - - # print(list_ID) - - def xml2dict(record): - dict_type = xmltodict.parse(record) - json_type = json.dumps(dict_type, indent=4) - dict2_type = json.loads(json_type) - return dict2_type - - Entrez.email = Email - - logging.info(f"Number of IDs: {len(list_ID)}") - - # Parse - cnt = 0 - cut = 50 # parse by 50 sequences - cnt_all = len(list_ID) - - start_time = time.time() - - record_list = [] - - for i in range(int((len(list_ID) - 1) / cut) + 1): - # last chunk - if i * cut + cut > len(list_ID): - # Mes(i) - ID_string = ",".join(list_ID[i * cut :]) - sleep(0.3) - cnt += len(list_ID[i * cut :]) - logging.info( - f"{cnt}/{cnt_all} {100*cnt/cnt_all}% {time.time()-start_time}s" - ) - - try: - print(ID_string) - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - except: # Retry - logging.info("Requesting again...") - try: - sleep(10) - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - except: - sleep(100) - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - - pre_record = handle.read() - json_record = xml2dict(pre_record) - tmp_record_list = [] - - # If only one result available - if type(json_record["GBSet"]["GBSeq"]) is dict: - json_record["GBSet"]["GBSeq"] = [json_record["GBSet"]["GBSeq"]] - - if len(list_ID[i * cut :]) != 1: - for record in json_record["GBSet"]["GBSeq"]: - print(record) - record_list.append(record) - tmp_record_list.append(record) - else: - record = json_record["GBSet"]["GBSeq"] - record_list.append(record) - tmp_record_list.append(record) - - try: - with open(f"./{path_tmp}/{i}", "wb") as f: - pickle.dump(tmp_record_list, f) - except: - logging.error("Saving Error") - raise Exception - - # non last chunk - else: - # Mes(i) - ID_string = ",".join(list_ID[i * cut : i * cut + cut]) - sleep(0.3) - cnt += cut - logging.info( - f"{cnt}/{cnt_all} {100*cnt/cnt_all}% {time.time()-start_time}s" - ) - - try: - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - except: # Retry - logging.info("Requesting again...") - try: - sleep(10) - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - except: - sleep(100) - handle = Entrez.efetch( - db="nucleotide", id=ID_string, rettype="gb", retmode="xml" - ) - - pre_record = handle.read() - json_record = xml2dict(pre_record) - tmp_record_list = [] - - if len(list_ID[i * cut :]) != 1: - for record in json_record["GBSet"]["GBSeq"]: - record_list.append(record) - tmp_record_list.append(record) - else: - record = json_record["GBSet"]["GBSeq"] - record_list.append(record) - tmp_record_list.append(record) - - try: - with open(f"./{path_tmp}/{i}", "wb") as f: - pickle.dump(tmp_record_list, f) - except: - logging.error("Saving Error") - raise Exception - - with open(out, "w") as fp: - json_term = json.dump(record_list, fp, indent=4) - - # If success, remove tmp files - tmp_file_list = [file for file in os.listdir(f"./{path_tmp}/")] - for file in tmp_file_list: - os.remove(f"./{path_tmp}/{file}") - - return record_list diff --git a/funvip/src/reporter.py b/funvip/src/reporter.py index 6aece83..e48f9ec 100644 --- a/funvip/src/reporter.py +++ b/funvip/src/reporter.py @@ -84,8 +84,7 @@ def __init__(self): "IDENTIFIED": [], "NEW SPECIES CANDIDATE": [], "MISIDENTIFIED": [], - "AMBIGUOUS": [], - "ERROR": [], + "UNDETERMINED": [], "TOTAL": [], } @@ -211,6 +210,7 @@ def update_result(self, V, opt): # Check if data analysis had performed for specific FI, group, gene combination if ( gene in FI.bygene_species + and gene in FI.seq and len(FI.seq[gene]) > 0 and gene in V.dict_dataset[FI.adjusted_group] ): @@ -292,60 +292,45 @@ def update_result(self, V, opt): self.query_result["DATATYPE"] == "query" ] - ### Update statistics by result - - # Groupby group - df_result_group = self.query_result.groupby(["GROUP_ASSIGNED"]) - - # Count groups - for group in sorted(list(set(self.query_result["GROUP_ASSIGNED"]))): - df_group = df_result_group.get_group((group,)) - - # Collect statistics - """ - cnt_correctly_identified = list(df_group["STATUS"]).count( - "correctly identified" - ) - cnt_undetermined = list(df_group["STATUS"]).count("undetermined") - cnt_new_species_candidate = list(df_group["STATUS"]).count( - "new species candidate" - ) - cnt_misidentified = list(df_group["STATUS"]).count("misidentified") - cnt_error = list(df_group["STATUS"]).count("ERROR") - cnt_total = sum( - ( - cnt_correctly_identified, - cnt_undetermined, - cnt_new_species_candidate, - cnt_misidentified, - cnt_error, - ) - ) - """ - - # Write into dictionary - """ + self.update_statistics() + + # Per-group counts of query identification outcomes, from the STATUS column. + def update_statistics(self): + cols = [ + "IDENTIFIED", + "NEW SPECIES CANDIDATE", + "MISIDENTIFIED", + "UNDETERMINED", + ] + self.statistics = {"GROUP": [], **{c: [] for c in cols}, "TOTAL": []} + if self.query_result is None or len(self.query_result) == 0: + return + # STATUS (set in update_result) -> human-facing statistics category + category = { + "match": "IDENTIFIED", # given name confirmed by the tree + "assigned": "IDENTIFIED", # unnamed query assigned a species + "new species": "NEW SPECIES CANDIDATE", + "conflict": "MISIDENTIFIED", # assigned differs from the given name + "failed": "UNDETERMINED", # could not be resolved + } + df = self.query_result + df = df[df["DATATYPE"] == "query"] + for group in sorted(set(df["GROUP_ASSIGNED"])): + sub = df[df["GROUP_ASSIGNED"] == group] + counts = {c: 0 for c in cols} + for status in sub["STATUS"]: + cat = category.get(str(status).strip()) + if cat is not None: + counts[cat] += 1 self.statistics["GROUP"].append(group) - self.statistics["IDENTIFIED"].append(cnt_correctly_identified) - self.statistics["AMBIGUOUS"].append(cnt_undetermined) - self.statistics["NEW SPECIES CANDIDATE"].append(cnt_new_species_candidate) - self.statistics["MISIDENTIFIED"].append(cnt_misidentified) - self.statistics["ERROR"].append(cnt_error) - self.statistics["TOTAL"].append(cnt_total) - """ - - # Add final summations - """ - self.statistics["GROUP"].append("TOTAL") - self.statistics["IDENTIFIED"].append(sum(self.statistics["IDENTIFIED"])) - self.statistics["AMBIGUOUS"].append(sum(self.statistics["AMBIGUOUS"])) - self.statistics["NEW SPECIES CANDIDATE"].append( - sum(self.statistics["NEW SPECIES CANDIDATE"]) - ) - self.statistics["MISIDENTIFIED"].append(sum(self.statistics["MISIDENTIFIED"])) - self.statistics["ERROR"].append(sum(self.statistics["ERROR"])) - self.statistics["TOTAL"].append(sum(self.statistics["TOTAL"])) - """ + for c in cols: + self.statistics[c].append(counts[c]) + self.statistics["TOTAL"].append(len(sub)) + # Grand-total row + if self.statistics["GROUP"]: + self.statistics["GROUP"].append("TOTAL") + for c in cols + ["TOTAL"]: + self.statistics[c].append(sum(self.statistics[c])) ### Main report runner # Update report by pipeline step @@ -377,6 +362,7 @@ def update_report(self, V, path, opt, step, version, GenMine_flag): self.update_result(V, opt) self.report_table(V, path, opt, step) self.report_text(V, path, opt, step, version, GenMine_flag) + self.report_html(V, path, opt, version) else: logging.error( f"DEVELOPMENTAL ERROR : BAD STEP INPUT {step} WHILE UPDATE REPORT" @@ -450,8 +436,8 @@ def report_text(self, V, path, opt, step, version, GenMine_flag): ## Write options used (in concise form) if index_step(step) >= 0: f.write(f"[OPTION]\n") - f.write(f"DB: {opt.query}\n") - f.write(f"QUERY: {opt.db}\n") + f.write(f"DB: {opt.db}\n") + f.write(f"QUERY: {opt.query}\n") f.write(f"GENE: {opt.gene}\n") f.write(f"EMAIL: {opt.email}\n") f.write(f"API: {opt.api}\n") @@ -521,30 +507,23 @@ def report_text(self, V, path, opt, step, version, GenMine_flag): ## Write identification statistics # Should be written after tree_interpretation - ''' if index_step(step) >= 9: f.write(f"[STATISTICS]\n") f.write(tabulate(self.statistics, headers=self.statistics.keys())) f.write("\n\n") f.write( - "IDENTIFIED : Number of well-identified strains without any concerns. \n" + "IDENTIFIED : Query strains assigned a species (a given name confirmed by the tree, or an unnamed query newly assigned)\n" ) - """ - f.write( - "AMBIGUOUS : Multiple clades with same taxon name. Your database may contain misidentified sequences. \n" - ) - """ f.write( - "NEW SPECIES CANDIDATE : New species candidate strains found by topology, phylogenetic distance and bootstrap criteria\n" + "NEW SPECIES CANDIDATE : Query strains flagged as a new species candidate by topology, phylogenetic distance and bootstrap\n" ) f.write( - "MISIDENTIFIED : Strains that shows different identification result from original annotation\n" + "MISIDENTIFIED : Query strains whose assigned species differs from the given name\n" ) f.write( - "ERROR : Strains that cannot be analyzed. Please check if appropriate database sequence / outgroup sequences are given\n" + "UNDETERMINED : Query strains that could not be resolved to a species (check the database and outgroup sequences)\n" ) f.write("\n\n") - ''' ## Write identification result # Should be written after tree_interpretation @@ -563,7 +542,7 @@ def report_text(self, V, path, opt, step, version, GenMine_flag): f.write("\n\n") f.write("ID : Name of the strain\n") f.write( - "HASH : Temporary name of the strain to prevent unexpected error during run. Use this when manually edit intermediate step data and run from middle, or debugging unexpectively terminated run\n" + "HASH : Temporary name of the strain to prevent unexpected error during run. Use this when manually edit intermediate step data and run from middle, or debugging unexpectedly terminated run\n" ) f.write("DATATYPE : query or database\n") f.write("GROUP_ORIGINAL : group name given by user\n") @@ -855,7 +834,7 @@ def report_table(self, V, path, opt, step): if index_step(step) >= 9: list_table.append("identification") - # list_table.append("statistics") + list_table.append("statistics") # Write tables for table in list_table: @@ -884,5 +863,57 @@ def report_table(self, V, path, opt, step): ) raise Exception - def report_html(V, path, opt): - pass + def report_html(self, V, path, opt, version): + import os + import glob + import html as _html + + result_df = ( + self.query_result + if (self.query_result is not None and opt.queryonly is True) + else pd.DataFrame(self.result) + ) + stats_df = pd.DataFrame(self.statistics) + + # Link the collapsed tree figures (relative to the report), skipping the + # uncollapsed *_original.svg companions. + trees = sorted( + t + for t in glob.glob(f"{path.out_tree}/{opt.runname}_*.svg") + if not t.endswith("_original.svg") + ) + tree_items = "".join( + f'
  • ' + f"{_html.escape(os.path.basename(t))}
    • " + for t in trees + ) or "
  • (no tree figures)
    • " + + css = ( + "body{font-family:Arial,Helvetica,sans-serif;margin:2rem;color:#222;" + "line-height:1.4}h1{color:#7a1f1f;margin-bottom:0}h2{color:#7a1f1f;" + "border-bottom:1px solid #e0c0c0;padding-bottom:2px;margin-top:2rem}" + ".meta{color:#555;margin-top:.3rem}table{border-collapse:collapse;" + "margin:.6rem 0;font-size:13px}th,td{border:1px solid #ccc;" + "padding:3px 8px;text-align:left}th{background:#f4f4f4}" + ".tables{overflow-x:auto}" + ) + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + doc = ( + "\n" + f"FunVIP report - {_html.escape(str(opt.runname))}" + f"\n" + "

    FunVIP identification report

    \n" + f"

    Run {_html.escape(str(opt.runname))} " + f"· FunVIP {_html.escape(str(version.FunVIP))} " + f"· {now}

    \n" + "

    Statistics

    \n
    " + f"{stats_df.to_html(index=False, border=0)}
    \n" + "

    Identification result

    \n
    " + f"{result_df.to_html(index=False, border=0)}
    \n" + f"

    Trees

    \n
      {tree_items}
    \n" + "\n" + ) + with open( + f"{path.root}/{opt.runname}.report.html", "w", encoding="UTF8" + ) as f: + f.write(doc) diff --git a/funvip/src/save.py b/funvip/src/save.py index e932b4c..694a5b8 100644 --- a/funvip/src/save.py +++ b/funvip/src/save.py @@ -16,7 +16,6 @@ get_id, manage_unicode, ) -from funvip.src.logics import isnewicklegal from funvip.src.hasher import decode, newick_legal, hash_funinfo_list import shelve @@ -33,9 +32,15 @@ def save_session(opt, path, global_var: dict, var: dict) -> None: for key in global_var: if key in managed_keys: - save[key] = global_var[key] - if opt.verbose >= 3: - logging.debug(f"Saved {key}") + try: + save[key] = global_var[key] + if opt.verbose >= 3: + logging.debug(f"Saved {key}") + except Exception as e: + logging.warning( + f"Could not save session key {key!r} ({e}); " + "resume with --continue may be incomplete" + ) else: if opt.verbose >= 3: logging.debug(f"Did not saved {key}") @@ -171,13 +176,15 @@ def save_df(df, out, fmt="csv"): # For excel size limit elif fmt == "xlsx" or fmt == "excel": # Limit is 1048576 rows with 16384 column, exlcuding one for column - if len(df) <= 1048575 and df.shape[1] <= 16383: + if len(df) <= 1048575 and df.shape[1] <= 16384: df.to_excel(out, index=False) else: + csv_out = out.rsplit(".", 1)[0] + ".csv" logging.warning( - f"Dataframe size exceeds excel limit, 1048575 rows and 16384 columns. Using csv instead" + f"Dataframe exceeds the Excel limit (1048576 rows / 16384 columns); " + f"writing CSV to {csv_out} instead" ) - df.to_csv(out, index=False) + df.to_csv(csv_out, index=False) elif fmt == "parquet": df.to_parquet(out, index=False) elif fmt == "feather" or fmt == "ftr": diff --git a/funvip/src/search.py b/funvip/src/search.py index 83926e7..662563a 100644 --- a/funvip/src/search.py +++ b/funvip/src/search.py @@ -2,6 +2,7 @@ from funvip.src.ext import blast, makeblastdb, mmseqs, makemmseqsdb from funvip.src.save import save_df from funvip.src.tool import mkdir +from funvip.src.exceptions import ConfigError, InputError, SearchError import copy import pandas as pd import numpy as np @@ -45,8 +46,7 @@ def create_search_db(opt, db_fasta, db, path) -> None: ): makemmseqsdb(fasta=db_fasta, db=db, path=path) else: - logging.error("DEVELOPMENTAL ERROR on building search DB!") - raise Exception + raise SearchError("cannot build search DB: method is neither blast nor mmseqs") # Merge fragmented search matches from given blast or mmseqss results @@ -56,26 +56,22 @@ def get_unique(series) -> str: if len(set(series)) == 1: return list(series)[0] elif len(set(series)) == 0: - logging.error(f"Found 0 values in {series} while merging search results") - raise Exception + raise SearchError(f"found 0 values in {series} while merging search results") else: - logging.error( - f"Found {len(set(series))} values in {series} while merging search results" + raise SearchError( + f"found {len(set(series))} conflicting values in {series} while merging search results" ) - raise Exception # Calculate overall percent identity of fragments def calculate_pident(df): pident = df["pident"] length = df["length"] if len(pident) != len(length): - logging.error( - f"During merge blast fragments, found pident {pident} and len {length} are different" + raise SearchError( + f"while merging fragments, pident and length differ in length: {len(pident)} vs {len(length)}" ) - raise Exception elif len(pident) == 0: - logging.error(f"No percent identitiy {pident} found") - raise Exception + raise SearchError(f"no percent identity found while merging fragments: {pident}") else: # Calculate overall percent identity return np.sum(np.array(pident) * np.array(length)) / np.sum(length) @@ -147,8 +143,7 @@ def search(query_fasta, db_fasta, path, opt) -> pd.DataFrame(): if opt.cachedb is True or opt.usecache is True: if _hash is None: - logging.error(f"Database file {db_fasta} missing") - raise Exception + raise InputError(f"database file {db_fasta} is missing") # Try to parse DB if opt.usecache is True: @@ -191,6 +186,16 @@ def search(query_fasta, db_fasta, path, opt) -> pd.DataFrame(): # Create search database create_search_db(opt, db_fasta, db, path) + else: + # usecache=False, cachedb=True: build DB and save to cache + logging.info( + f"--cachedb selected, {opt.method.search.lower()} database will be saved" + ) + mkdir(f"{path.in_db}/{opt.method.search.lower()}/{_hash}") + db = f"{path.in_db}/{opt.method.search.lower()}/{_hash}/{_hash}" + logging.info("The database is in first run, caching database") + create_search_db(opt, db_fasta, db, path) + else: mkdir(f"{path.tmp}/{opt.runname}/{_hash}") db = f"{path.tmp}/{opt.runname}/{_hash}/{_hash}" @@ -229,8 +234,7 @@ def search(query_fasta, db_fasta, path, opt) -> pd.DataFrame(): # remove temporary file # shutil.rmtree(f"{path.tmp}/{opt.runname}") else: - logging.error("DEVELOPMENTAL ERROR on searching!") - raise Exception + raise SearchError("cannot run search: method is neither blast nor mmseqs") # Parse out df = pd.read_csv( @@ -297,10 +301,9 @@ def search_df(V, path, opt): V.list_db_gene.remove(gene) if len(V.list_db_gene) == 0: - logging.error( - f"None of the gene seems to be valid in analysis. Please check your --gene flag" + raise ConfigError( + "no valid gene found in the database; check your --gene flag and the database gene columns" ) - raise Exception # get query fasta from funinfo_list list_qr_FI = tool.select(V.list_FI, datatype="query") @@ -380,7 +383,8 @@ def search_df(V, path, opt): if isinstance(df_search, pd.DataFrame): if not df_search.empty: dict_unclassified[gene] = df_search - except: + except NameError: + # df_search was deleted above (no result exceeded outgroupoffset) pass # assign gene by search result to unassigned sequences @@ -412,10 +416,10 @@ def search_df(V, path, opt): else: # If db column and query column does not matches -> should be moved to validate_input if not (gene in V.list_db_gene): - logging.error( - f"Gene {gene} found in query, but not found in database. Please add {gene} column to database" + raise InputError( + f"gene {gene} is present in the query but not in the database; " + f"add a {gene} column to the database" ) - raise Exception # BLAST or mmseqs search # This part should be changed by using former search result for faster performance diff --git a/funvip/src/tool.py b/funvip/src/tool.py index c3d4b52..3e242a3 100644 --- a/funvip/src/tool.py +++ b/funvip/src/tool.py @@ -76,10 +76,30 @@ def manage_unicode(string, column="", row=""): raise Exception +# Cache of parsed genus lists keyed by file path. genus_file is set once per run +# by initialize_path(), but get_genus_species(genus_list=None) is called +# per-sequence in validate_input, so without a cache the ~3k-line genus DB was +# re-opened and re-read on every call. +_genus_list_cache = {} + + +def _load_genus_list(): + gf = globals().get("genus_file") + if not gf or not os.path.exists(gf): + raise ValueError( + "get_genus_species(genus_list=None) needs the genus database path set " + f"by tool.initialize_path(); it was not initialized (genus_file={gf!r})" + ) + if gf not in _genus_list_cache: + with open(gf, "r") as f: + _genus_list_cache[gf] = f.read().splitlines() + return _genus_list_cache[gf] + + @lru_cache(maxsize=10000) def get_genus_species( string, - endwords=[ + endwords=( "small", "18S", "ribosomal", @@ -91,7 +111,7 @@ def get_genus_species( "strain", "beta", "tubulin", - ], + ), genus_list=None, ): return_genus = "" @@ -99,8 +119,7 @@ def get_genus_species( # en for enumeratable object (splited string) if genus_list is None: - with open(genus_file, "r") as f: - genus_list = f.read().splitlines() + genus_list = _load_genus_list() en = string.replace(" ", "_").split("_") diff --git a/funvip/src/tree.py b/funvip/src/tree.py index bb46977..1c4c374 100644 --- a/funvip/src/tree.py +++ b/funvip/src/tree.py @@ -42,14 +42,17 @@ def pipe_tree(V, path, opt, model_dict): fasttree_opt = [] # for multiprocessing on fasttree - tree_dataset = deepcopy(V.dict_dataset) + # Shallow two-level copy: only `.pop(gene)` on the per-group dict mutates this + # structure below; the Dataset objects are read-only here, so a full deepcopy of + # the entire FI universe (measured ~126 MB at metabarcoding scale) is wasted. + tree_dataset = {group: dict(inner) for group, inner in V.dict_dataset.items()} # Before drawing tree, finalize datasets remove_dataset = [] for group in tree_dataset: for gene in tree_dataset[group]: # draw tree only when outgroup sequence exists - if tree_dataset[group][gene].list_og_FI == 0: + if len(tree_dataset[group][gene].list_og_FI) == 0: logging.warning( f"Passing tree construction of {group} {gene} dataset because no outgroup available" ) @@ -235,39 +238,32 @@ def pipe_tree(V, path, opt, model_dict): ) """ + # Resume-safe decode: a prior interrupted run may already have moved the + # plain-name file into hash/; guard os.rename so --continue does not crash + # with FileNotFoundError. If the plain file exists, move+decode as usual; + # if only the hashed file exists, just (re)decode it; otherwise skip. + def _safe_decode(plain, hashed): + if os.path.exists(plain): + os.rename(plain, hashed) + hasher.decode(tree_hash_dict, hashed, plain) + elif os.path.exists(hashed): + hasher.decode(tree_hash_dict, hashed, plain) + else: + logging.warning(f"Neither {plain} nor {hashed} exist - skipping decode") + if gene != "concatenated": - # Decode adjusted - os.rename( + _safe_decode( f"{path.out_adjusted}/{opt.runname}_Adjusted_{group}_{gene}.fasta", f"{path.out_adjusted}/hash/{opt.runname}_hash_Adjusted_{group}_{gene}.fasta", ) - hasher.decode( - tree_hash_dict, - f"{path.out_adjusted}/hash/{opt.runname}_hash_Adjusted_{group}_{gene}.fasta", - f"{path.out_adjusted}/{opt.runname}_Adjusted_{group}_{gene}.fasta", - ) - - # Decode alignment - os.rename( + _safe_decode( f"{path.out_alignment}/{opt.runname}_MAFFT_{group}_{gene}.fasta", f"{path.out_alignment}/hash/{opt.runname}_hash_MAFFT_{group}_{gene}.fasta", ) - hasher.decode( - tree_hash_dict, - f"{path.out_alignment}/hash/{opt.runname}_hash_MAFFT_{group}_{gene}.fasta", - f"{path.out_alignment}/{opt.runname}_MAFFT_{group}_{gene}.fasta", - ) - # Decode trimmed file - os.rename( + _safe_decode( f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta", f"{path.out_alignment}/hash/{opt.runname}_hash_trimmed_{group}_{gene}.fasta", ) - hasher.decode( - tree_hash_dict, - f"{path.out_alignment}/hash/{opt.runname}_hash_trimmed_{group}_{gene}.fasta", - f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta", - ) - return V, path, opt diff --git a/funvip/src/tree_interpretation.py b/funvip/src/tree_interpretation.py index d495276..8db4172 100644 --- a/funvip/src/tree_interpretation.py +++ b/funvip/src/tree_interpretation.py @@ -1,6 +1,6 @@ # tree interpretation pipeline - collapse and visualize tree -from ete3 import ( - Tree, +from ete4 import Tree +from ete4.treeview import ( TreeStyle, NodeStyle, TextFace, @@ -8,6 +8,66 @@ RectFace, faces, ) + +# Fix ete4 bug: _leaf() uses hasattr(node, "_img_style") but ete4 stores +# _img_style in node.props (dict), so hasattr always returns False, making +# draw_descendants=False a no-op. Patch all modules that have their own copy. +def _ete4_fixed_leaf(node): + collapsed = "_img_style" in node.props and not node.img_style["draw_descendants"] + return collapsed or node.is_leaf + +import ete4.treeview.qt_render as _ete4_qt_render +import ete4.treeview.qt_rect_render as _ete4_qt_rect_render +import ete4.treeview.qt_face_render as _ete4_qt_face_render +import ete4.treeview.qt_gui as _ete4_qt_gui + +_ete4_qt_render._leaf = _ete4_fixed_leaf +_ete4_qt_rect_render._leaf = _ete4_fixed_leaf +_ete4_qt_face_render._leaf = _ete4_fixed_leaf +_ete4_qt_gui._leaf = _ete4_fixed_leaf + + +def _ladderize_ete3_compat(node): + """Replicate ete3's ladderize(direction=1) tie-breaking behavior. + + ete3 sorts ascending by leaf count (stable), then calls reverse(). + ete4 sorts descending (stable) with a secondary key len(children). + When subtrees have equal leaf counts, the two-step ete3 approach reverses + tied items while ete4's one-step approach preserves their original order — + producing different sp. N numbering for balanced clades. + """ + if node.is_leaf: + return 1 + n2s = {} + for child in node.children: + n2s[child] = _ladderize_ete3_compat(child) + node.children.sort(key=lambda x: n2s[x]) # ascending stable (leaf count) + node.children.reverse() # reverse = ete3 direction=1 + return sum(n2s.values()) + + +def _ete4_make_root_consistent(t): + """ete4 compat: ete4's assert_root_consistency (called inside unroot/set_outgroup) + requires root.dist in {0, None}, no 'support' on the root, and equal support on the + two root children. FastTree/RAxML trees read from newick routinely violate the last + (e.g. support 1.0 vs 0.0), raising AssertionError. Coerce the current root to a + consistent state before any reroot operation.""" + try: + if getattr(t, "dist", None) not in (0, None): + t.dist = 0 + if "support" in getattr(t, "props", {}): + t.props.pop("support", None) + ch = t.children + if len(ch) == 2: + s = ch[0].support if ch[0].support is not None else ch[1].support + if s is None: + s = 1.0 + ch[0].support = s + ch[1].support = s + except Exception: + pass + + from Bio import SeqIO from copy import deepcopy from time import sleep @@ -19,12 +79,15 @@ from itertools import combinations import tracemalloc import dendropy +import numpy as np import collections import psutil import os import re import sys import json +import logging +from funvip.src.exceptions import TreeError # Default zero length branch for concatenation CONCAT_ZERO = 0 # for better binding @@ -106,28 +169,58 @@ def concat_clade( root_support=0, ): tmp = Tree() - tmp.dist = root_dist - tmp.support = root_support - tmp.add_child(clade1, dist=dist1, support=support1) - tmp.add_child(clade2, dist=dist2, support=support2) + tmp.dist = root_dist if root_dist is not None else CONCAT_ZERO + tmp.support = root_support if root_support is not None else 0 + tmp.add_child(clade1, dist=dist1 if dist1 is not None else CONCAT_ZERO, + support=support1 if support1 is not None else 1) + tmp.add_child(clade2, dist=dist2 if dist2 is not None else CONCAT_ZERO, + support=support2 if support2 is not None else 1) return tmp +## Combine clades into a BALANCED (log-depth) binary tree for concat_all. +# Bottom-up left-to-right pairwise merge (preserves leaf order). Each input +# clade keeps its OWN dist/support as its edge; every glue node created here is +# (dist=CONCAT_ZERO, support=0), matching the internal nodes of the former +# left-deep comb. Input clades are deep-copied once (as concat_all did), so the +# caller's clades are not mutated. +def _balanced_merge(clades): + nodes = [c.copy("deepcopy") for c in clades] + while len(nodes) > 1: + merged = [] + for i in range(0, len(nodes), 2): + if i + 1 < len(nodes): + a, b = nodes[i], nodes[i + 1] + merged.append( + concat_clade( + clade1=a, + clade2=b, + dist1=a.dist, + dist2=b.dist, + support1=a.support, + support2=b.support, + ) + ) + else: + merged.append(nodes[i]) + nodes = merged + return nodes[0] + + ## concat all given branches for concatenation # clades were given in tuble, and root_dist is given def concat_all(clade_tuple, root_dist, root_support=0): if len(clade_tuple) == 0: - print("No clade input found, abort") - raise Exception + raise TreeError("concat_all called with no clades") # If one clade were input, return self elif len(clade_tuple) == 1: - return clade_tuple[0].copy("newick") + return clade_tuple[0].copy("deepcopy") # If two clades were input, concat it and return elif len(clade_tuple) == 2: - return_clade = clade_tuple[0].copy("newick") + return_clade = clade_tuple[0].copy("deepcopy") return_clade = concat_clade( clade1=return_clade, - clade2=clade_tuple[1].copy("newick"), + clade2=clade_tuple[1].copy("deepcopy"), dist1=return_clade.dist, dist2=clade_tuple[1].dist, support1=return_clade.support, @@ -135,23 +228,23 @@ def concat_all(clade_tuple, root_dist, root_support=0): root_dist=root_dist, root_support=root_support, ) - # If more than 3 clades were input, iteratively concat + # If more than 3 clades were input, concat into a BALANCED (log-depth) tree. # If more than 2 species exists, and sp included, which taxon sp should be included cannot be decided # In that case, move sp clade to last + # Was a left-deep comb (for c in clade_tuple[1:-1]: concat onto accumulator), + # so a k-member same-taxon group became a k-deep nesting -> ete4 copy / + # multiprocessing-pickle RecursionError (5.8S skipped) and O(n^2) build cost. + # Balanced form is byte-identical for k<=4 (balanced==comb there); for k>=5 it + # only rearranges the zero-dist/zero-support glue nodes. Each original clade + # still keeps its own dist/support as its edge, every glue node is + # (dist=CONCAT_ZERO, support=0), the last clade stays a direct child of the + # root, and the root carries root_dist/root_support -- so identification is + # unchanged (leaf sets + per-clade edges + root dist/support are preserved). elif len(clade_tuple) >= 3: - return_clade = clade_tuple[0].copy("newick") - for c in clade_tuple[1:-1]: - return_clade = concat_clade( - clade1=return_clade, - clade2=c.copy("newick"), - dist1=return_clade.dist, - dist2=c.dist, - support1=return_clade.support, - support2=c.support, - ) + return_clade = _balanced_merge(clade_tuple[:-1]) return_clade = concat_clade( clade1=return_clade, - clade2=clade_tuple[-1].copy("newick"), + clade2=clade_tuple[-1].copy("deepcopy"), dist1=return_clade.dist, dist2=clade_tuple[-1].dist, support1=return_clade.support, @@ -160,7 +253,7 @@ def concat_all(clade_tuple, root_dist, root_support=0): root_support=root_support, ) else: - raise Exception + raise TreeError(f"concat_all: unexpected clade_tuple length {len(clade_tuple)}") return return_clade @@ -184,36 +277,93 @@ def __init__(self): self.ts.show_leaf_name = False -@lru_cache(maxsize=10000) -def decide_type(query_list, db_list, outgroup, string, by="hash", priority="query"): - query = False - db = False +# Cache the hash sets derived from (query_list, db_list, outgroup). decide_type is called +# once per leaf inside consist/get_taxon/taxon_count, i.e. O(n * tree-depth) times during +# reconstruct; recomputing three O(n) lists and doing O(n) list-membership on every call made +# it the dominant cost (~O(n^3) on the deep FastTree combs). The three input tuples are set +# once per tree on Tree_information, so key on their object identity and rebuild only when they +# change (a new tree). Single entry: holds one tree's sets, replaced (old GC'd) when the tree +# changes, so RAM stays O(n) per worker; separate worker processes each keep their own. +# (The former @lru_cache was ineffective: its key was these big list-tuples, so every lookup +# re-hashed them in O(n).) +_DECIDE_TYPE_HASHSETS = None - query_hash_list = [FI.hash for FI in query_list] - db_hash_list = [FI.hash for FI in db_list] - outgroup_hash_list = [FI.hash for FI in outgroup] + +def decide_type(query_list, db_list, outgroup, string, by="hash", priority="query"): + global _DECIDE_TYPE_HASHSETS + c = _DECIDE_TYPE_HASHSETS + if c is None or c[0] is not query_list or c[1] is not db_list or c[2] is not outgroup: + c = ( + query_list, + db_list, + outgroup, + frozenset(FI.hash for FI in query_list), + frozenset(FI.hash for FI in db_list), + frozenset(FI.hash for FI in outgroup), + ) + _DECIDE_TYPE_HASHSETS = c if by == "hash": - if string in query_hash_list: + if string in c[3]: return "query" - elif string in db_hash_list: + elif string in c[4]: return "db" - elif string in outgroup_hash_list: + elif string in c[5]: return "outgroup" else: return "none" else: - print( - f"{bold_red}[ERROR] DEVELOPMENTAL ERROR, UNEXPECTED by for decide_type{reset}" + raise TreeError(f"decide_type: unexpected 'by' value {by!r}") + + +# uint8 lookup for encoding aligned sequences: a/t/g/c -> 1..4, everything else (gap '-', +# ambiguity codes, any non-atgc char) -> 0. Matches the old per-char cleaning +# ("'-' if char not in 'atgc-' else char"), where both '-' and non-atgc collapse to gap. +_ATGC_CODE = np.zeros(256, dtype=np.uint8) +for _i, _c in enumerate(b"atgc"): + _ATGC_CODE[_c] = _i + 1 + + +def _encode_alignment(seq_list): + # Encode every aligned sequence once (O(n*L)) into an (n, L) uint8 matrix so calculate_zero + # can compare overlaps with numpy instead of per-pair Python char loops. + n = len(seq_list) + L = len(seq_list[0].seq) + arr = np.zeros((n, L), dtype=np.uint8) + for i, record in enumerate(seq_list): + buf = np.frombuffer( + str(record.seq).lower().encode("ascii", "replace"), dtype=np.uint8 ) - raise Exception + arr[i] = _ATGC_CODE[buf] + return arr # count number of taxons in the clade -def taxon_count( +# Subtree taxon-count memoization, active only during tree_search (topology is static +# there; solve_flat mutations already happened). Keyed by id(node) so it does not pollute +# node.props (ete4 nodes are __slots__ and reject arbitrary attributes) and is not written +# to newick. Cleared per tree by _tc_cache_begin/_tc_cache_end so id() reuse across trees +# cannot cause stale hits. Toggle off with FUNVIP_NO_TC_MEMO for A/B verification. +_TC_CACHE = None + + +def _tc_cache_begin(): + global _TC_CACHE + if not os.environ.get("FUNVIP_NO_TC_MEMO"): + _TC_CACHE = {} + + +def _tc_cache_end(): + global _TC_CACHE + _TC_CACHE = None + + +def _taxon_count_flat( funinfo_dict, query_list, db_list, outgroup, clade, gene, count_query=False ): + # Original non-memoized full-leaf scan. Reference for the FUNVIP_TC_ASSERT self-check + # and used whenever memoization is inactive. taxon_dict = {} for leaf in clade: @@ -221,23 +371,15 @@ def taxon_count( taxon = None if count_query == True: taxon = (FI.genus, FI.bygene_species[gene]) - elif ( - decide_type( - query_list=query_list, - db_list=db_list, - outgroup=outgroup, - string=leaf.name, - ) - == "db" - or decide_type( + else: + _leaf_type = decide_type( query_list=query_list, db_list=db_list, outgroup=outgroup, string=leaf.name, ) - == "outgroup" - ): - taxon = (FI.genus, FI.bygene_species[gene]) + if _leaf_type == "db" or _leaf_type == "outgroup": + taxon = (FI.genus, FI.bygene_species[gene]) if not (taxon is None): if not (taxon in taxon_dict): @@ -248,40 +390,67 @@ def taxon_count( return taxon_dict -def genus_count(funinfo_dict, gene, clade): - taxon_dict = {} - - for leaf in clade.iter_leaves(): - FI = funinfo_dict[leaf.name] - if ( - decide_type( - query_list=self.query_list, - db_list=self.db_list, - outgroup=self.outgroup, - string=leaf.name, - ) - == "db" - or decide_type( - query_list=self.query_list, - db_list=self.db_list, - outgroup=self.outgroup, - string=leaf.name, - ) - == "outgroup" - ): - if not ((FI.genus, FI.bygene_species[gene]) in taxon_dict): - taxon_dict[(FI.genus, FI.bygene_species[gene])[0]] = 1 +def taxon_count( + funinfo_dict, query_list, db_list, outgroup, clade, gene, count_query=False +): + # Memoized recursive path (active during tree_search). Equivalent to the flat scan: + # leaf iteration of a node equals the ordered concatenation of its children's leaves, + # so the merged dict has identical keys, counts, and insertion order (which + # find_majortaxon tie-breaks on). Turns the repeated full-subtree rescans (O(n^2) on + # many-species genera) into one post-order (O(n)). Set FUNVIP_TC_ASSERT to verify the + # memoized result against the flat scan on every call. + if _TC_CACHE is not None: + key = (id(clade), gene, count_query) + hit = _TC_CACHE.get(key) + if hit is not None: + # hand out a copy: callers must never mutate the shared cached dict + result = dict(hit) + else: + taxon_dict = {} + if clade.is_leaf: + FI = funinfo_dict[clade.name] + taxon = None + if count_query == True: + taxon = (FI.genus, FI.bygene_species[gene]) + elif decide_type( + query_list=query_list, + db_list=db_list, + outgroup=outgroup, + string=clade.name, + ) in ("db", "outgroup"): + taxon = (FI.genus, FI.bygene_species[gene]) + if taxon is not None: + taxon_dict[taxon] = 1 else: - taxon_dict[(FI.genus, FI.bygene_species[gene])[0]] += 1 + for child in clade.children: + cd = taxon_count( + funinfo_dict, query_list, db_list, outgroup, child, gene, count_query + ) + for k, v in cd.items(): + taxon_dict[k] = taxon_dict.get(k, 0) + v + _TC_CACHE[key] = taxon_dict + result = dict(taxon_dict) + if os.environ.get("FUNVIP_TC_ASSERT"): + _flat = _taxon_count_flat( + funinfo_dict, query_list, db_list, outgroup, clade, gene, count_query + ) + if list(result.items()) != list(_flat.items()): + sys.stderr.write( + f"[TC_ASSERT] mismatch cq={count_query} " + f"leaves={[l.name for l in clade]}\n memo={result}\n flat={_flat}\n" + ) + return result - return taxon_dict + return _taxon_count_flat( + funinfo_dict, query_list, db_list, outgroup, clade, gene, count_query + ) def designate_genus(funinfo_dict, query_list, db_list, outgroup, gene, clade): genus_dict = {} # Get genus_count - for leaf in clade.iter_leaves(): + for leaf in clade.leaves(): FI = funinfo_dict[leaf.name] if ( decide_type( @@ -433,10 +602,10 @@ def is_monophyletic( if len(taxon_dict.keys()) == 0: for children in clade.children: # if any of the branch length was too long for single clade - if children.dist > opt.collapsedistcutoff: + if children.dist is not None and children.dist > opt.collapsedistcutoff: return False - # or bootstrap is to distinctive - elif children.support > opt.collapsebscutoff: + # or bootstrap is to distinctive (support may be None for leaves in ete4) + elif children.support is not None and children.support > opt.collapsebscutoff: return False return True # if taxon dict.keys() have 1 species: 1 kinds of species @@ -457,13 +626,13 @@ def is_monophyletic( clade=children, gene=gene, )[1].startswith("sp."): - if children.dist > opt.collapsedistcutoff: + if children.dist is not None and children.dist > opt.collapsedistcutoff: return False - elif children.support > opt.collapsebscutoff: + elif children.support is not None and children.support > opt.collapsebscutoff: return False - elif other_children.dist > opt.collapsebscutoff: + elif other_children.dist is not None and other_children.dist > opt.collapsebscutoff: return False - elif other_children.dist > opt.collapsedistcutoff: + elif other_children.dist is not None and other_children.dist > opt.collapsedistcutoff: return False return True else: @@ -522,22 +691,35 @@ class Tree_information: def __init__(self, tree, Tree_style, group, gene, opt): self.tree_name = tree # for debugging self.t = Tree(tree) + # ete4: root node has dist=None when newick has no explicit root branch length + # Normalize to 0.0 so all dist comparisons work correctly + for _n in self.t.traverse(): + if _n.dist is None: + _n.dist = 0.0 self.t_publish = ( None # for publish tree - will substitute tree_original in long_term ) self.dendro_t = dendropy.Tree.get( - path=self.tree_name, schema="newick" + path=self.tree_name, schema="newick", preserve_underscores=True ) # dendropy format for distance calculation + # ete4 compat: dendropy converts unquoted Newick underscores to spaces by default, + # so pdc keys ('Genus species') would not match alignment ids ('Genus_species') and + # calculate_zero() raises KeyError. preserve_underscores=True keeps them aligned. # if support ranges from 0 to 1, change it from 0 to 100 - # b for branch - support_set = set() - for b in self.t.traverse(): - support_set.add(b.support) + # b for branch (leaves have support=None in ete4, skip them) + support_set = set( + b.support for b in self.t.traverse() if b.support is not None + ) - if max(support_set) <= 1: + # Track whether scale conversion happens — used by pipe to normalize + # ete4's None-support nodes for ete3-compatible reconstruct behavior + self.support_scaled = False + if support_set and max(support_set) <= 1: for b in self.t.traverse(): - b.support = int(100 * b.support) + if b.support is not None: + b.support = int(100 * b.support) + self.support_scaled = True self.query_list = [] self.db_list = [] @@ -566,7 +748,7 @@ def __init__(self, tree, Tree_style, group, gene, opt): # to find out already existing new species number to avoid overlapping # e.g. avoid sp 5 if P. sp 5 already exsits in database def reserve_sp(self): # does not seems to be working currently - for leaf in self.t.iter_leaves(): + for leaf in self.t.leaves(): FI = self.funinfo_dict[leaf.name] taxon = (FI.genus, FI.ori_species) sys.stdout.flush() @@ -586,137 +768,139 @@ def calculate_zero(self, alignment_file, gene, partition_dict): if collections.Counter(hash_list_tree) != collections.Counter( hash_list_alignment ): - print( - f"{bold_red}[ERROR] content of tree and alignment is not identical for {self.tree_name}{reset}" + raise TreeError( + f"tree and alignment contents differ for {self.tree_name}" ) - raise Exception - # Find identical or including pairs in alignment - identical_pairs = [] - # different_pairs = [] - - # make phylogenetic distance matrix - pdc = self.dendro_t.phylogenetic_distance_matrix().as_data_table()._data + # Encode each sequence once (uint8; gap and any non-atgc char -> 0, matching the old + # "atgc-" cleaning). The per-partition overlap is applied to every pair (the old code + # reused the loop name `gene` in `for gene in gene_order`, clobbering the `gene` argument). + ids = [str(record.id).strip() for record in seq_list] + arr = _encode_alignment(seq_list) + nongap = arr != 0 + n_seq, n_col = arr.shape + + if gene == "concatenated": + spans = [] + _offset = 0 + for _g in partition_dict["order"]: + spans.append((_offset, _offset + partition_dict["len"][_g])) + _offset += partition_dict["len"][_g] + else: + spans = [(0, n_col)] + + def _overlap_compare(i, j): + # returns (differ?, overlapping_cnt) over the per-partition overlap; None if no overlap + both = nongap[i] & nongap[j] + valid = np.zeros(n_col, dtype=np.bool_) + for a, b in spans: + nz = np.flatnonzero(both[a:b]) + if nz.size: + valid[a + nz[0] : a + nz[-1] + 1] = True + if not valid.any(): + return None + eq = arr[i][valid] == arr[j][valid] + return (not bool(eq.all()), int(eq.sum())) + + # diff_min = min patristic distance among pairs that DIFFER. Any different pair with a + # patristic distance < zero_init must lie inside one <= zero_init-connected cluster (a path + # summing to < zero_init has every branch < zero_init), so only within-cluster pairs can set + # a diff_min below zero_init. And whenever diff_min < zero_init the final self.zero is + # max(diff_min - 1e-8, floor) regardless of max_identical (self.zero is capped at + # diff_min - 1e-8 because diff_min < zero_init <= max(zero_init, max_identical)). So in the + # common (metabarcoding) case we avoid both the O(n^2) phylogenetic distance matrix and the + # O(n^2) all-pairs scan; only clean, well-separated data (no different pair < zero_init) + # falls back to the full scan below. + zero_init = self.zero + idx_of = {name: k for k, name in enumerate(ids)} + + _parent = {} + + def _find(x): + while _parent.get(x, x) != x: + _parent[x] = _parent.get(_parent[x], _parent[x]) + x = _parent[x] + return x + + def _union(a, b): + _parent.setdefault(a, a) + _parent.setdefault(b, b) + _parent[_find(a)] = _find(b) + + for _nd in self.t.traverse(): + _parent.setdefault(id(_nd), id(_nd)) + if _nd.up is not None and (_nd.dist or 0) <= zero_init: + _union(id(_nd), id(_nd.up)) + + _clusters = {} + for _lf in self.t.leaves(): + _clusters.setdefault(_find(id(_lf)), []).append(_lf) diff_min = 999999 - for seq1, seq2 in combinations(seq_list, 2): - if not ( - str(seq1.id).strip() == str(seq2.id).strip() - or (seq1.id, seq2.id) in identical_pairs - or (seq2.id, seq1.id) in identical_pairs - ): - """ - # Chenge unusable chars into gap - seq1_str = str(seq1.seq).lower() - seq2_str = str(seq2.seq).lower() - - for char in set(seq1_str) - {"a", "t", "g", "c", "-"}: - seq1_str = seq1_str.replace(char, "-") - - for char in set(seq2_str) - {"a", "t", "g", "c", "-"}: - seq2_str = seq2_str.replace(char, "-") - """ - - seq1_str, seq2_str = str(seq1.seq).lower(), str(seq2.seq).lower() - seq1_str = "".join( - "-" if char not in "atgc-" else char for char in seq1_str - ) - seq2_str = "".join( - "-" if char not in "atgc-" else char for char in seq2_str - ) - - identical_flag = True - # To prevent distance among different region detected as zero in concatenated analysis - overlapping_cnt = 0 - - # For concatenated sequence alignment, identical sequnece should be checked by each partitions - if gene == "concatenated": - len_dict = partition_dict["len"] - gene_order = partition_dict["order"] - - valid_index = [] - - # calculate valid index to check - previous_index = 0 - - for gene in gene_order: - start = previous_index - end = previous_index + len_dict[gene] - 1 - - # Find the starting point - for n in range(previous_index, len_dict[gene] + previous_index): - if seq1_str[n] != "-" and seq2_str[n] != "-": - start = n - break - - # Compare from the start - for n in range( - len_dict[gene] + previous_index - 1, - previous_index - 1, - -1, - ): - if seq1_str[n] != "-" and seq2_str[n] != "-": - end = n - break - - valid_index.extend(range(start, end + 1)) - - previous_index += len_dict[gene] - - # for valid part - for n in valid_index: - # connected with or to evaluate insertions or deletions - if seq1_str[n] != seq2_str[n]: - identical_flag = False - else: - overlapping_cnt += 1 - - else: - start = 0 - end = len(seq1_str) - 1 - # calculate start and end - for n in range(len(seq1_str)): - if seq1_str[n] != "-" and seq2_str[n] != "-": - start = n - break - - for n in range(len(seq1_str)): - if ( - seq1_str[len(seq1_str) - n - 1] != "-" - and seq2_str[len(seq1_str) - n - 1] != "-" - ): - end = len(seq1_str) - n - break - - # for valid part - for n in range(start, end): - # connected with or to evaluate insertions or deletions - if seq1_str[n] != seq2_str[n]: - identical_flag = False - else: - overlapping_cnt += 1 - - # if identical pairs - id1 = str(seq1.id).strip() - id2 = str(seq2.id).strip() - - if identical_flag is True and overlapping_cnt > 0: - if pdc[id1][id2] > self.zero: - self.zero = pdc[id1][id2] - - # if different pairs - elif identical_flag is False: - if pdc[id1][id2] < diff_min: - diff_min = pdc[id1][id2] - - if diff_min < self.zero: + for _cl in _clusters.values(): + if len(_cl) < 2: + continue + for _a in range(len(_cl)): + _ia = idx_of[_cl[_a].name] + for _b in range(_a + 1, len(_cl)): + _cmp = _overlap_compare(_ia, idx_of[_cl[_b].name]) + if _cmp is not None and _cmp[0]: + _pat = self.t.get_distance(_cl[_a], _cl[_b]) + if _pat < diff_min: + diff_min = _pat + + if diff_min < zero_init: + # short-circuit: no distance matrix and no max-identical scan needed self.zero = diff_min - 0.00000001 + else: + # clean, well-separated data: build the distance matrix and do the full pairwise scan + # for the exact max-identical and the (>= zero_init) diff_min + pdc = self.dendro_t.phylogenetic_distance_matrix().as_data_table()._data + diff_min = 999999 + for i in range(n_seq): + pdc_i = pdc[ids[i]] + for j in range(i + 1, n_seq): + if ids[i] == ids[j]: + continue + _cmp = _overlap_compare(i, j) + if _cmp is None: + continue + _differ, _overlap = _cmp + if (not _differ) and _overlap > 0: + d = pdc_i[ids[j]] + if d > self.zero: + self.zero = d + elif _differ: + d = pdc_i[ids[j]] + if d < diff_min: + diff_min = d + if diff_min < self.zero: + self.zero = diff_min - 0.00000001 + + # Engine-aware floor on self.zero. A branch that is "effectively zero" is not a single + # value but the band [min_branch_length, min_branch_length + optimizer_tolerance]: the + # engine clamps a near-zero branch to its minimum, then Brent's branch optimizer stops + # within its absolute tolerance of that minimum, so identical sequences get branch lengths + # spread across that band (e.g. FastTree emits both 5e-9 AND 6e-9 = xmin + atol). self.zero + # must be the TOP of the band, otherwise the just-above-minimum values are treated as real + # branches and split otherwise-identical flat clusters. Verified on Lactarius: raising + # 5e-9 -> 6e-9 merges the noise-split clusters and does NOT over-merge divergent sequences + # (every query whose call changed was 0-1 base different from its nearest DB neighbour over + # the overlap). Values read from each engine's source: + # FastTree (double): MLMinBranchLength 5e-9 + MLMinBranchLengthTolerance 1e-9 = 6e-9 + # IQ-TREE : -blmin 1e-6 + TOL_BRANCH_LEN 1e-6 = 2e-6 + # RAxML (classic) : -log(zmax) = -log(1-1e-6) = 1.0000005e-6, rounded up = 2e-6 + _engine_min_branch = {"fasttree": 6e-9, "iqtree": 2e-6, "raxml": 2e-6} + self.zero = max( + self.zero, + _engine_min_branch.get(str(self.opt.method.tree).lower(), 6e-9), + ) if self.opt.verbose >= 3: - print(f"[DEBUG] End of calculate zero") + logging.debug("End of calculate zero") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") # I think also finding minimal distance between non-identical sequences are also needed return self.zero @@ -727,10 +911,17 @@ def reroot_outgroup(self, out): outgroup_leaves = [] # Resolve polytomy before rerooting + # ete4 compat: resolve_polytomy zeros all support values; save them to restore after rerooting + _support_backup = {id(n): n.support for n in self.t.traverse()} self.t.resolve_polytomy() + # ete4 compat: new internal nodes created by resolve_polytomy get support=0; + # set to 1.0 to match ete3 DEFAULT_SUPPORT so NWK output writes 1 not 0 + for _n in self.t.traverse(): + if id(_n) not in _support_backup and not _n.is_leaf: + _n.support = 1.0 # Check if outgroup sequences exists - print(f"[INFO] Rerooting {self.outgroup} in {self.tree_name}") + logging.info(f"Rerooting {self.outgroup} in {self.tree_name}") for leaf in self.t: if any(outgroup.hash in leaf.name for outgroup in self.outgroup): outgroup_leaves.append(leaf) @@ -743,65 +934,71 @@ def reroot_outgroup(self, out): # find smallest monophyletic clade that contains all leaves in outgroup_leaves # reroot with outgroup_clade try: - # For more than one outgroups, after rerooting, get_common_ancestor of outgroup again + # For more than one outgroups, after rerooting, common_ancestor of outgroup again # Before rerooting, unroot the tree to work properly if len(outgroup_leaves) >= 2: + _ete4_make_root_consistent(self.t) self.t.unroot() - self.outgroup_clade = self.t.get_common_ancestor(outgroup_leaves) + self.outgroup_clade = self.t.common_ancestor(outgroup_leaves) self.t.set_outgroup(self.outgroup_clade) - self.t.ladderize(direction=1) - self.outgroup_clade = self.t.get_common_ancestor(outgroup_leaves) + _ladderize_ete3_compat(self.t) + self.outgroup_clade = self.t.common_ancestor(outgroup_leaves) elif len(outgroup_leaves) == 1: + _ete4_make_root_consistent(self.t) self.t.unroot() self.outgroup_clade = outgroup_leaves[0] self.t.set_outgroup(self.outgroup_clade) - self.t.ladderize(direction=1) + _ladderize_ete3_compat(self.t) self.outgroup_clade = outgroup_leaves[0] else: - print( - f"{bold_red}[ERROR] no outgroup selected in {self.tree_name}{reset}" - ) - raise Exception + # control flow: no outgroup in the primary path -> trigger the + # flexible-reroot fallback in the except below (TreeError is an + # Exception, so it is caught there). + raise TreeError(f"no outgroup selected in {self.tree_name}") # If number of outgroup leaves and outgroup clade does not matches, paraphyletic if len(outgroup_leaves) != len(self.outgroup_clade): - print( - f"{yellow}[WARNING] outgroup seems to be paraphyletic in {self.tree_name}{reset}" + logging.warning( + f"outgroup seems to be paraphyletic in {self.tree_name}" ) - except: - print(f"{yellow}[WARNING] no outgroup selected in {self.tree_name}{reset}") + except Exception: + logging.warning( + f"no outgroup selected in {self.tree_name}, trying flexible reroot" + ) outgroup_flag = False # if outgroup_clade is on the root side, reroot with other leaf temporarily and reroot again for leaf in self.t: if not (leaf in outgroup_leaves): + _ete4_make_root_consistent(self.t) self.t.set_outgroup(leaf) # Rerooting again while outgrouping gets possible try: - self.outgroup_clade = self.t.get_common_ancestor( + self.outgroup_clade = self.t.common_ancestor( outgroup_leaves ) # print(f"Ancestor: {self.outgroup_clade}") + _ete4_make_root_consistent(self.t) self.t.set_outgroup(self.outgroup_clade) outgroup_flag = True break - except: + except Exception: + # this temp-root attempt failed; try the next leaf pass if outgroup_flag is False: - # never erase this for debugging - print( - f"{bold_red}[ERROR] Outgroup not selected in {self.tree_name}{reset}" - ) - print( - f"{bold_red}[ERROR] local variable outgroup_leaves : {outgroup_leaves}{reset}" + raise TreeError( + f"outgroup could not be selected for {self.tree_name} " + f"(outgroup_leaves={outgroup_leaves}, outgroup={self.outgroup}, " + f"outgroup_clade={self.outgroup_clade})" ) - print(f"{bold_red}[ERROR] tree_info.outgroup : {self.outgroup}{reset}") - print( - f"{bold_red}[ERROR] tree_info.outgroup_clade : {self.outgroup_clade}{reset}" - ) - raise Exception + + # ete4 compat: restore support values destroyed by resolve_polytomy + # (set_outgroup creates new nodes not in backup; those stay None for None→100 fix) + for _n in self.t.traverse(): + if id(_n) in _support_backup: + _n.support = _support_backup[id(_n)] self.Tree_style.ts.show_leaf_name = True @@ -811,7 +1008,7 @@ def reroot_outgroup(self, out): for node in copied_tree.traverse(): node.img_style["size"] = 0 # removing circles whien size is 0 - if len(node) > 1: # Prevent bootstrap on single branch + if len(node) > 1 and node.support is not None: # Prevent bootstrap on single/leaf branch node.add_face( TextFace( f"{int(node.support)}", @@ -826,10 +1023,10 @@ def reroot_outgroup(self, out): self.Tree_style.ts.show_leaf_name = False if self.opt.verbose >= 3: - print(f"[DEBUG] End of reroot outgroup") + logging.debug("End of reroot outgroup") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") def collapse(self, collapse_info, clade, taxon): collapse_info.clade = clade @@ -840,7 +1037,7 @@ def collapse(self, collapse_info, clade, taxon): elif len(clade) >= 2: collapse_info.collapse_type = "triangle" else: - raise Exception + raise TreeError(f"collapse: clade has {len(clade)} leaves (expected >=1)") if ( any( @@ -851,7 +1048,7 @@ def collapse(self, collapse_info, clade, taxon): string=leaf.name, ) == "query" - for leaf in clade.iter_leaves() + for leaf in clade.leaves() ) == True ): @@ -866,7 +1063,7 @@ def collapse(self, collapse_info, clade, taxon): collapse_info.height = len(clade) * self.opt.visualize.heightmultiplier # count query, db, others - for leaf in clade.iter_leaves(): + for leaf in clade.leaves(): if ( decide_type( query_list=self.query_list, @@ -899,16 +1096,11 @@ def collapse(self, collapse_info, clade, taxon): ) collapse_info.n_query += 1 else: - print( - f"{bold_red}[ERROR] DEVELOPMENTAL ERROR : UNEXPECTED LEAF TYPE FOR {leaf.name}{reset}" + raise TreeError( + f"unexpected leaf type for {leaf.name} in {self.tree_name}" ) - print(self.tree_name) - print(f"Query: {sorted([FI.hash for FI in self.query_list])}") - print(f"DB: {sorted([FI.hash for FI in self.db_list])}") - print(f"Outgroup: {sorted([FI.hash for FI in self.outgroup])}") - raise Exception - # Species level delimitaion on tree + # Species level delimitation on tree def tree_search(self, clade, gene, opt=None): def local_generate_collapse_information(self, clade, opt=None): collapse_info = Collapse_information() @@ -932,7 +1124,7 @@ def local_generate_collapse_information(self, clade, opt=None): while 1: self.sp_cnt += 1 if str(self.sp_cnt) in self.reserved_sp: - print(f"Skipping {self.sp_cnt} to avoid overlap in database") + logging.debug(f"Skipping {self.sp_cnt} to avoid overlap in database") continue else: break @@ -951,13 +1143,13 @@ def local_generate_collapse_information(self, clade, opt=None): ## start of tree_search if len(clade.children) == 1: - local_generate_collapse_information(clade, opt=opt) + local_generate_collapse_information(self, clade, opt=opt) if self.opt.verbose >= 3: - print(f"[DEBUG] End of Tree search with monophyletic branches") + logging.debug("End of Tree search with monophyletic branches") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") return @@ -966,7 +1158,7 @@ def local_generate_collapse_information(self, clade, opt=None): for child_clade in clade.children: # Calculate root distance between two childs to check flat self.flat = ( - True if child_clade.dist <= self.opt.collapsedistcutoff else False + True if (child_clade.dist is None or child_clade.dist <= self.opt.collapsedistcutoff) else False ) # Check if child clades are monophyletic @@ -989,19 +1181,18 @@ def local_generate_collapse_information(self, clade, opt=None): self.tree_search(child_clade, gene, opt=opt) if self.opt.verbose >= 3: - print(f"[DEBUG] End of Tree search with bifurcated branches") + logging.debug("End of Tree search with bifurcated branches") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") return # if error (more than two branches or no branches) else: - print( - f"{bold_red}[ERROR] DEVELOPMENTAL ERROR : FAILED TREE SEARCH ON LEAF {clade.children}{reset}" + raise TreeError( + f"tree_search: clade has {len(clade.children)} children (expected 1 or 2)" ) - raise Exception # end of tree_search # Reconstruct tree tree to solve flat branches @@ -1025,10 +1216,7 @@ def consist(c): query += 1 if db == 0 and query == 0: - print( - f"{bold_red}[ERROR] DEVELOPMENTAL ON CONSIST, {c} {db} {query}{reset}" - ) - raise Exception + raise TreeError(f"consist: clade has neither db nor query leaves: {c}") elif db == 0 and query != 0: return "query" elif db != 0 and query == 0: @@ -1044,11 +1232,10 @@ def t(leaf): try: FI = self.funinfo_dict[leaf.name] return (FI.genus, FI.bygene_species[gene]) - except: - print( - f"{bold_red}[DEVELOPMENTAL ERROR] in leaf.name tree_interpretation.py line 869 {resety}" - ) - raise Exception + except KeyError as e: + raise TreeError( + f"leaf {leaf.name} has no funinfo / bygene_species[{gene}] entry" + ) from e taxon_dict = {} @@ -1061,10 +1248,7 @@ def t(leaf): taxon_dict[t(leaf)] = 1 if len(taxon_dict) == 0: - print( - f"{bold_red}[DEVELOPMENTAL ERROR] in tree_interpretation.py line 884 {taxon_dict}\n {c}{reset}" - ) - raise Exception + raise TreeError(f"get_taxon(db): empty taxon_dict for clade {c}") # If only one species in the clade elif len(taxon_dict) == 1: # If only one taxon here, return the taxon @@ -1082,10 +1266,7 @@ def t(leaf): taxon_dict[("", "")] = 1 if len(taxon_dict) == 0: - print( - f"{bold_red}[DEVELOPMENTAL ERROR] Error in tree_interpretation.py line 912 {taxon_dict}\n {c}{reset}" - ) - raise Exception + raise TreeError(f"get_taxon(query): empty taxon_dict for clade {c}") elif len(taxon_dict) == 1: return list(taxon_dict.keys())[0] else: @@ -1123,8 +1304,7 @@ def t(leaf): taxon_dict[t(leaf)] += 1 if len(taxon_dict) == 0: - print(f"{taxon_dict}\n {c}") - raise Exception + raise TreeError(f"get_taxon(both): empty taxon_dict for clade {c}") elif len(taxon_dict) == 1: return list(taxon_dict.keys())[0] else: @@ -1132,14 +1312,22 @@ def t(leaf): def seperate_clade(clade, gene, clade_list): for c in clade.children: - c_tmp = c.copy() + # Only deep-copy what is KEPT: a single tip appended to + # clade_list, or the input handed to reconstruct. The former + # upfront copy of every child re-copied the whole resolve_polytomy + # comb (~D deep for conserved genes like 5.8S) at each recursion + # level -> O(D^2). seperate_clade/reconstruct never mutate their + # input, so the zero-length comb is descended in place instead. + # (deepcopy, not the default cpickle copy, which RecursionErrors on + # the deep comb; dist/len read identically on c and a copy of c.) # zero clades - if c_tmp.dist <= self.zero: + if c.dist is None or c.dist <= self.zero: # Original version was == instead of >= . Revert if error occurs # What does the "len" means here? -> len means number of tips # If only one tip - if len(c_tmp) <= 1: + if len(c) <= 1: # In the zero branch tip, the query with zero length should move to sp., because they cannot be fully determined + c_tmp = c.copy("deepcopy") clade_list.append( ( get_taxon(c=c_tmp, gene=gene, mode=consist(c_tmp)), @@ -1152,11 +1340,12 @@ def seperate_clade(clade, gene, clade_list): # I'm not sure if any of the recursion enters here, but just in case else: clade_list = seperate_clade( - clade=c_tmp, gene=gene, clade_list=clade_list + clade=c, gene=gene, clade_list=clade_list ) # non-zero clades else: + c_tmp = c.copy("deepcopy") c2 = self.reconstruct(c_tmp, gene, opt) clade_list.append( ( @@ -1232,7 +1421,7 @@ def seperate_clade(clade, gene, clade_list): for taxon in taxon_to_merge: l = clade_dict[taxon] # l for list of results r_list = [r[1] for r in l] # result clade list - r_list.sort(key=lambda r: r.dist, reverse=True) + r_list.sort(key=lambda r: r.dist if r.dist is not None else 0.0, reverse=True) r_tuple = tuple(r_list) # concatenate within taxon clades @@ -1241,7 +1430,7 @@ def seperate_clade(clade, gene, clade_list): ) tmp_final_clade.append(concatenated_clade) - if taxon != ("", "") and concatenated_clade.dist <= self.zero: + if taxon != ("", "") and (concatenated_clade.dist is None or concatenated_clade.dist <= self.zero): flat_issue_cnt += 1 final_clade = tmp_final_clade + final_clade @@ -1260,12 +1449,12 @@ def seperate_clade(clade, gene, clade_list): # print(f"Status flat: {self.flat_clades}") - return final.copy("newick") + return final.copy("deepcopy") ## end of solve flat ## Start of function reconstruct if len(clade.children) in (0, 1): - return clade.copy("newick") + return clade.copy("deepcopy") elif len(clade.children) == 2: clade1 = clade.children[0] @@ -1273,9 +1462,9 @@ def seperate_clade(clade, gene, clade_list): # Solve flat if clade.dist <= self.zero: - return solve_flat(clade).copy("newick") + return solve_flat(clade).copy("deepcopy") elif clade1.dist <= self.zero or clade2.dist <= self.zero: - return solve_flat(clade).copy("newick") + return solve_flat(clade).copy("deepcopy") else: r_clade1 = self.reconstruct(clade1, gene, opt) r_clade2 = self.reconstruct(clade2, gene, opt) @@ -1289,19 +1478,20 @@ def seperate_clade(clade, gene, clade_list): support2=clade2.support, root_dist=clade.dist, root_support=clade.support, - ).copy("newick") + ).copy("deepcopy") if self.opt.verbose >= 3: - print(f"[DEBUG] End of reconstruct") + logging.debug("End of reconstruct") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") return concatanated_clade else: - print(f"[ERROR] {clade} {clade.children} {len(clade.children)}") - raise Exception + raise TreeError( + f"reconstruct: clade has {len(clade.children)} children (expected 0-2)" + ) ## end of reconstruct def get_bgcolor(self): @@ -1424,7 +1614,7 @@ def collapse_tree(self): # change this part when debugging flat trees node.img_style["size"] = 0 # removing circles whien size is 0 - if node.support >= self.opt.visualize.bscutoff: + if node.support is not None and node.support >= self.opt.visualize.bscutoff: # node.add_face without generating extra line # add_face_to_node node.add_face( @@ -1480,7 +1670,7 @@ def polish_image(self, out, taxon_string_dict): try: int(text.text) text_type = "bootstrap" - except: + except (ValueError, TypeError): if text.text == "0.05": text_type = "scale" elif any( @@ -1513,7 +1703,7 @@ def polish_image(self, out, taxon_string_dict): int(species) tspan = ET.SubElement(text, "{http://www.w3.org/2000/svg}tspan") tspan.text = species - except: + except ValueError: if "sp." in species: tspan = ET.SubElement( text, "{http://www.w3.org/2000/svg}tspan" @@ -1573,9 +1763,11 @@ def polish_image(self, out, taxon_string_dict): if self.funinfo_dict[word.strip()].color is not None: try: tspan.set("fill", self.funinfo_dict[word.strip()].color) - except: - print("DEVELOPMENTAL ERROR: Failed coloring tree") - raise Exception + except Exception: + logging.debug( + f"failed to set tree tip color for {word.strip()}" + ) + raise elif ( decide_type( @@ -1588,7 +1780,8 @@ def polish_image(self, out, taxon_string_dict): == "query" ): tspan.set("fill", self.opt.visualize.highlight) - except: + except Exception: + # word is not a known FI hash / coloring failed -> leave uncolored pass # fit size of tree_xml to svg # find svg from tree_xml @@ -1602,7 +1795,7 @@ def polish_image(self, out, taxon_string_dict): ) if self.opt.verbose >= 3: - print(f"[DEBUG] End of Tree visualization") + logging.debug("End of Tree visualization") process = psutil.Process(os.getpid()) memory_info = process.memory_info() - print(f"[DEBUG] RAM usage: {memory_info.rss / 1000 / 1000} MB") + logging.debug(f"RAM usage: {memory_info.rss / 1000 / 1000} MB") diff --git a/funvip/src/tree_interpretation_pipe.py b/funvip/src/tree_interpretation_pipe.py index a6aeaa3..b86dbb5 100644 --- a/funvip/src/tree_interpretation_pipe.py +++ b/funvip/src/tree_interpretation_pipe.py @@ -1,10 +1,12 @@ # Performing multiple tree interpretation -from ete3 import Tree +from ete4 import Tree from funvip.src import tree_interpretation from funvip.src.tool import initialize_path, get_genus_species from funvip.src.tool import sizeof_fmt from funvip.src.hasher import encode, decode from funvip.src.reporter import Singlereport +from funvip.src.exceptions import TreeError +import traceback from copy import deepcopy import pandas as pd import re @@ -19,13 +21,17 @@ ### For single dataset # Input : out, group, gene, V, path, opt +# Whole-run FI collections (V.dict_hash_FI / V.list_FI) shared with +# interpretation-pool workers via fork copy-on-write, set before the Pool is +# created, instead of being pickled into every (group, gene) task tuple. +_INTERP_SHARED = {} + + def pipe_module_tree_interpretation( out, group, gene, V_tup_genus, - funinfo_dict, - funinfo_list, hash_dict, query_list, outgroup, @@ -35,7 +41,12 @@ def pipe_module_tree_interpretation( ): # time_start = time() - # for unexpectively included sequence during clustering + # Read the whole-run FI collections from the fork-inherited shared store rather + # than receiving a freshly pickled copy of the entire universe per task. + funinfo_dict = _INTERP_SHARED["funinfo_dict"] + funinfo_list = _INTERP_SHARED["funinfo_list"] + + # for unexpectedly included sequence during clustering db_list = list( set([FI for FI in funinfo_list if FI.datatype == "db"]) - set(outgroup) @@ -54,16 +65,12 @@ def pipe_module_tree_interpretation( if os.path.isfile(tree_name): try: # If iqtree, missing supports are not shown - if opt.method.tree.lower() == "iqtree": - Tree(tree_name, format=0) - else: - Tree(tree_name, format=2) - except: - logging.error(f"[DEVELOPMENTAL ERROR] Failed on importing tree {tree_name}") - raise Exception + # ete4 autodetects newick format regardless of software + Tree(tree_name) + except Exception as e: + raise TreeError(f"failed to parse tree file {tree_name}: {e}") from e else: - logging.error(f"Cannot find {tree_name}") - raise Exception + raise TreeError(f"cannot find tree file {tree_name}") # initialize before analysis Tree_style = tree_interpretation.Tree_style() @@ -122,14 +129,39 @@ def pipe_module_tree_interpretation( # Reconstruct flat branches if option given if opt.solveflat is True: + # ete4: copy("newick") uses parser=1 which drops support values; + # use deepcopy to preserve branch support for visualization. + _clade = tree_info.t.copy("deepcopy") + for _n in _clade.traverse(): + if _n.dist is None: + _n.dist = 0.0 + # ete4 compat: ete3 DEFAULT_SUPPORT=1.0 became 100 after scale + # conversion; ete4 leaves/root get support=None. Normalize only + # INTERNAL nodes None→100 (when scale conversion occurred) so + # intermediate nodes created by reconstruct/solve_flat carry + # support=100 like ete3. Leaves must stay None: concat_clade maps + # None→1 for them, which is < bscutoff (correct — ete3 also sets + # leaf DEFAULT_SUPPORT=1.0 → 1 after newick round-trip → not shown). + if not _n.is_leaf and _n.support is None and tree_info.support_scaled: + _n.support = 100 tree_info.t = tree_info.reconstruct( - clade=tree_info.t.copy("newick"), gene=gene, opt=opt + clade=_clade, gene=gene, opt=opt ) + # reconstruct() replaced tree_info.t with a shallow rebuild, but + # tree_info.outgroup_clade still points at a node of the PRE-reconstruct + # tree. reroot_outgroup's resolve_polytomy() combs a conserved gene's giant + # star (5.8S: ~3670-leaf polytomy) into a ~3681-deep chain, and pickling any + # ete4 node drags in its whole tree via .up/.children -- so this stale ref + # re-inflates that deep tree when the worker pickles tree_info back to the + # parent (multiprocessing), raising RecursionError -> MaybeEncodingError. + # outgroup_clade is unused after rerooting, so drop it. (Third part of the + # 5.8S fix, with seperate_clade deepcopy + balanced concat_all.) + tree_info.outgroup_clade = None # print(f"Reconstruct {time() - time_start}") - # reorder tree for pretty look - tree_info.t.ladderize(direction=1) + # reorder tree for pretty look (ete3-compatible tie-breaking) + tree_interpretation._ladderize_ete3_compat(tree_info.t) # print(f"Ladderize {time() - time_start}") @@ -137,8 +169,13 @@ def pipe_module_tree_interpretation( # Is not currently used # tree_info.t_publish = deepcopy(tree_info.t) - # Search tree and delimitate species - tree_info.tree_search(tree_info.t, gene) + # Search tree and delimitate species. + # Enable subtree taxon-count memoization for this (static, post-solve_flat) tree only. + tree_interpretation._tc_cache_begin() + try: + tree_info.tree_search(tree_info.t, gene) + finally: + tree_interpretation._tc_cache_end() # print(f"Tree search {time() - time_start}") @@ -148,7 +185,7 @@ def pipe_module_tree_interpretation( f"{path.out_tree}/{opt.runname}_{group}_{gene}_original.nwk", ) tree_info.t.write( - format=0, outfile=f"{path.out_tree}/{opt.runname}_{group}_{gene}.nwk" + outfile=f"{path.out_tree}/{opt.runname}_{group}_{gene}.nwk" ) decode( tree_hash_dict, @@ -164,7 +201,7 @@ def pipe_module_tree_interpretation( ### synchronize sp. numbers from multiple dataset # to use continuous sp numbers over trees -# Seperated from multithreading, because this step should traverse over multiple trees, therefore cannot be done simultaniously +# Seperated from multithreading, because this step should traverse over multiple trees, therefore cannot be done simultaneously def synchronize(V, path, tree_info_list): # Gets hash dict, and returns taxon name of hash_dict # Generate final taxon name for synchronizing @@ -211,7 +248,8 @@ def get_new_taxon(hash_list, hash_taxon_dict): dict_species[" ".join(splited_species[:-1])].append( int(splited_species[-1]) ) - except: + except (ValueError, IndexError): + # species label does not end in an integer -> treat as unnumbered dict_species[s] = [0] species = "" @@ -255,8 +293,9 @@ def get_new_taxon(hash_list, hash_taxon_dict): elif not (tree_info.gene in tree_info_dict[tree_info.group]): tree_info_dict[tree_info.group][tree_info.gene] = tree_info else: - logging.error("DEVELOPMENTAL ERROR, DUPLICATED TREE_INFO") - raise Exception + raise TreeError( + f"duplicated tree_info for group {tree_info.group} gene {tree_info.gene}" + ) # Memoize iterative calling # For each group list @@ -365,6 +404,11 @@ def get_new_taxon(hash_list, hash_taxon_dict): # Then, non-concatenated for group in tree_info_dict: + # A group whose concatenated tree failed interpretation (dropped by the + # per-item guard) can still have gene entries here; skip it rather than + # KeyError on the missing "concatenated" key and abort the whole run. + if "concatenated" not in tree_info_dict[group]: + continue for gene in tree_info_dict[group]: if gene != "concatenated": tree_info = tree_info_dict[group]["concatenated"] @@ -572,6 +616,37 @@ def pipe_module_tree_visualization( ### For all datasets, multiprocessing part +def _safe_pipe_module_tree_interpretation(*args): + # Per-item guard: a crash in one (group, gene) must not kill the whole batch + # (starmap re-raises the first worker exception -> no result.csv for the 900+ + # genera that DID succeed). Log and skip the bad one instead. + try: + return pipe_module_tree_interpretation(*args) + except Exception as e: + group = args[1] if len(args) > 1 else "?" + gene = args[2] if len(args) > 2 else "?" + logging.error( + f"[TREE INTERPRETATION FAILED] {group} {gene}: {e!r}\n{traceback.format_exc()}" + ) + return None + + +def _safe_pipe_module_tree_visualization(*args): + try: + return pipe_module_tree_visualization(*args) + except Exception as e: + ti = args[0] if args else None + gg = ( + f"{getattr(ti, 'group', '?')} {getattr(ti, 'gene', '?')}" + if ti is not None + else "?" + ) + logging.error( + f"[TREE VISUALIZATION FAILED] {gg}: {e!r}\n{traceback.format_exc()}" + ) + return None + + def pipe_tree_interpretation(V, path, opt): # Generate tree_interpretation opt to run # tree_interpretation_opt = [] @@ -586,6 +661,12 @@ def pipe_tree_interpretation(V, path, opt): funinfo_list = V.list_FI hash_dict = V.dict_hash_name + # Share the whole-run FI collections with pool workers via fork copy-on-write + # (must be set before the Pool below is created) instead of pickling them into + # every task; workers read them from _INTERP_SHARED. + _INTERP_SHARED["funinfo_dict"] = funinfo_dict + _INTERP_SHARED["funinfo_list"] = funinfo_list + # Generate options using generator def generate_interpretation_opt(): # make option variables @@ -625,8 +706,6 @@ def generate_interpretation_opt(): group, gene, V.tup_genus, - funinfo_dict, - funinfo_list, hash_dict, query_list, outgroup, @@ -649,7 +728,7 @@ def generate_interpretation_opt(): if opt.verbose < 3: with mp.Pool(opt.thread) as p: tree_info_list.extend( - p.starmap(pipe_module_tree_interpretation, tree_interpretation_opt) + p.starmap(_safe_pipe_module_tree_interpretation, tree_interpretation_opt) ) else: @@ -659,6 +738,13 @@ def generate_interpretation_opt(): for option in tree_interpretation_opt ] + # Drop genera that failed interpretation (guarded above) so one bad tree + # does not sink the whole run; failures are logged as [TREE INTERPRETATION FAILED]. + _n_failed = sum(1 for ti in tree_info_list if ti is None) + if _n_failed: + logging.warning(f"{_n_failed} (group, gene) trees failed interpretation and were skipped") + tree_info_list = [ti for ti in tree_info_list if ti is not None] + # Gather flat branch issues for tree_info in tree_info_list: for flat_hash in tree_info.flat_clades: @@ -685,7 +771,7 @@ def generate_visualization_opt(): if opt.verbose < 3: with mp.Pool(opt.thread) as p: tree_visualization_result = p.starmap( - pipe_module_tree_visualization, tree_visualization_opt + _safe_pipe_module_tree_visualization, tree_visualization_opt ) else: @@ -694,6 +780,9 @@ def generate_visualization_opt(): pipe_module_tree_visualization(*option) for option in tree_visualization_opt ] + # Drop genera that failed visualization (guarded) before flattening + tree_visualization_result = [r for r in tree_visualization_result if r is not None] + ### Collect identifiation result to V for reporting # Merge report list all_report_list = [ diff --git a/funvip/src/trim.py b/funvip/src/trim.py index 27ba97a..9dfaa91 100644 --- a/funvip/src/trim.py +++ b/funvip/src/trim.py @@ -40,9 +40,10 @@ def trimming(alignment, out, path, opt): "trimal", ) ): - # Check if trimming successed - if trimming_result is not ((-1, -1)): - # Repair trimmend alignment by analysis flanking region + # Check if trimming succeeded (both Gblocks and Trimal return (-1, -1) on + # failure; compare by value, not identity) + if trimming_result != (-1, -1): + # Repair trimmed alignment by analysis flanking region trimmed_msa = AlignIO.read(out, "fasta") logging.debug(f"Trimming region for {alignment} : {trimming_result}") @@ -51,7 +52,7 @@ def trimming(alignment, out, path, opt): AlignIO.write(revived_msa, out, "fasta") else: logging.warning( - f"Trimming {alignment} failed. Check if the alignment includes invalid sequneces" + f"Trimming {alignment} failed. Check if the alignment includes invalid sequences" ) return trimming_result @@ -93,7 +94,7 @@ def pipe_trimming(V, path, opt): "fasta", ) ) - if len(seq_list[0].seq) == 0: + if not seq_list or len(seq_list[0].seq) == 0: trim_fail.append((group, gene)) else: pass @@ -111,7 +112,7 @@ def pipe_trimming(V, path, opt): # If non of the genes left for group except for concatenate, remove group if len(V.dict_dataset[group]) == 1 and "concatenated" in V.dict_dataset[group]: logging.warning( - f"Dataset {group} has removed because non of the genes are available" + f"Dataset {group} has removed because none of the genes are available" ) V.dict_dataset.pop(group) diff --git a/funvip/src/validate_input.py b/funvip/src/validate_input.py index e77b76d..1e2f149 100644 --- a/funvip/src/validate_input.py +++ b/funvip/src/validate_input.py @@ -24,7 +24,7 @@ from time import sleep from funvip.src import save -from funvip.src.logics import isnewicklegal, isuniquecolumn, isvalidcolor +from funvip.src.logics import isuniquecolumn, isvalidcolor from funvip.src.hasher import decode, newick_legal, hash_funinfo_list # funinfo_dict: {"ID" : FI} @@ -77,7 +77,8 @@ def update_seqrecord(self, seq, gene=None): else: self.seq[gene] = str(seq.seq).replace("-", "") - self.bygene_species[gene] = self.ori_species + if gene is not None: + self.bygene_species[gene] = self.ori_species return error @@ -208,7 +209,7 @@ def update_datatype(self, datatype): return error def update_id(self, id_, regexs=None): - if not regexs == None: + if regexs is not None: id_ = get_id(id_, tuple(regexs)) # if cannot find id by regex @@ -365,7 +366,7 @@ def input_fasta(path, opt, fasta_list, funinfo_dict, datatype): try: seq_list = list(SeqIO.parse(file, "fasta")) for seq in seq_list: - if not opt.regex == None: + if opt.regex is not None: id_ = get_id(seq.description, tuple(opt.regex)) else: id_ = seq.description @@ -430,6 +431,27 @@ def input_fasta(path, opt, fasta_list, funinfo_dict, datatype): # getting datafile from excel or tabular file +def _resolve_genmine(): + """Resolve the GenMine executable next to the running interpreter rather than + via a bare shell PATH lookup. A different, incompatible GenMine version + installed elsewhere on PATH (another venv, a stray global pip install) would + otherwise silently shadow the one FunVIP was installed with, and GenMine's + output format has changed between versions -- validate_input's parsing of + its results would then quietly fail (raw accessions left unreplaced) with no + error, instead of a normal, cleanly-diagnosable version mismatch. + """ + import sysconfig + + exe_name = "GenMine.exe" if sys.platform == "win32" else "GenMine" + # Console scripts live in the interpreter's script dir: bin/ on POSIX, + # Scripts\ on Windows -- NOT next to python.exe, which on Windows is the env + # root, so dirname(sys.executable) would miss GenMine.exe there. + candidate = os.path.join(sysconfig.get_path("scripts"), exe_name) + if os.path.isfile(candidate): + return candidate + return shutil.which("GenMine") or "GenMine" + + def input_table(funinfo_dict, path, opt, table_list, datatype): # Whether to check if GenMine has run GenMine_flag = 0 @@ -578,7 +600,7 @@ def input_table(funinfo_dict, path, opt, table_list, datatype): regex_genbank = r"(([A-Z]{1}[0-9]{5})(\.[0-9]{1}){0,1})|(([A-Z]{2}[\_]{0,1}[0-9]{6}){1}([\.][0-9]){0,1})|(([A-Z]{4}[0-9]{8})(\.[0-9]{1}){0,1})|(([A-Z]{6}[0-9]{9,})(\.[0-9]{1}){0,1})" # if gene name were not designated by user, use seq - opt.gene = list(set([gene.lower().strip() for gene in opt.gene])) + opt.gene = list(dict.fromkeys([gene.lower().strip() for gene in opt.gene])) # find all NCBI accessions in seq for gene in opt.gene: @@ -625,7 +647,11 @@ def input_table(funinfo_dict, path, opt, table_list, datatype): else: GenMine_path = path.GenMine - cmd = f"GenMine -c {accession_path} -o {GenMine_path} -e {opt.email}" + genmine_exe = _resolve_genmine() + if " " in genmine_exe: + genmine_exe = f'"{genmine_exe}"' + + cmd = f"{genmine_exe} -c {accession_path} -o {GenMine_path} -e {opt.email}" logging.info(cmd) sleep(5) # To run GenMine safetly between run and run @@ -655,8 +681,11 @@ def input_table(funinfo_dict, path, opt, table_list, datatype): download_df = pd.read_excel(GenMine_df_list[0]) # Generate download_dict (I think this can be done with pandas operation, but a bit tricky. Will be done later) + # Key on the version-less accession: GenMine returns some accessions + # with a ".N" version suffix and some without depending on which NCBI + # fetch path served them, but the lookup below always strips the version. for n, acc in enumerate(download_df["acc"]): - download_dict[acc.strip()] = download_df["seq"][n] + download_dict[acc.strip().split(".")[0]] = download_df["seq"][n] # replace accession to sequence downloaded def update_from_GenMine(string): @@ -785,10 +814,18 @@ def update_from_GenMine(string): seq_error_list = [] # seq_string = manage_unicode(seq_string) - for x in seq_string: # x is every character of sequence - if not x.lower() in "acgtryswkmbdhvn-.": - seq_error_cnt += 1 - seq_error_list.append(x) + # Fast path: check the unique characters once; only build the + # per-character error list when an illegal character is present + # (the common case is a clean sequence, so this avoids a + # Python-level scan of every base of every sequence). + _illegal_chars = set(seq_string.lower()) - set( + "acgtryswkmbdhvn-." + ) + if _illegal_chars: + seq_error_list = [ + x for x in seq_string if x.lower() in _illegal_chars + ] + seq_error_cnt = len(seq_error_list) if seq_error_cnt > 0: warnings.append( @@ -819,7 +856,7 @@ def update_from_GenMine(string): # After successfully parsed this table, save it save.save_df( df, - f"{path.out_db}/Saved_{'.'.join(table.split('/')[-1].split('.')[:-1])}.{opt.tableformat}", + f"{path.out_query if datatype == 'query' else path.out_db}/Saved_{'.'.join(table.split('/')[-1].split('.')[:-1])}.{opt.tableformat}", fmt=opt.tableformat, ) @@ -935,7 +972,7 @@ def query_input(funinfo_dict, opt, path): shutil.copy(f"{file}", f"{path.out_query}") logging.info( - f"Total {len([funinfo_dict[key].datatype =='query' for key in funinfo_dict.keys()])} sequences parsed from query" + f"Total {sum(1 for key in funinfo_dict if funinfo_dict[key].datatype == 'query')} sequences parsed from query" ) return funinfo_dict, GenMine_flag diff --git a/funvip/src/validate_option.py b/funvip/src/validate_option.py index d41c7c3..2fef272 100644 --- a/funvip/src/validate_option.py +++ b/funvip/src/validate_option.py @@ -41,7 +41,7 @@ def __init__(self): class Cluster_Option: def __init__(self): self.cutoff = 0.95 - self.evalue = 10 + self.evalue = 0.0001 self.wordsize = 7 self.outgroupoffset = 20 self.max_target_seqs = 100 @@ -123,117 +123,121 @@ def update_from_preset(self, preset_file): # Update loaded preset for key in parser_dict: # Basic options - if key.lower() in ("query"): + if key.lower() in ("query",): self.query = parser_dict[key] - elif key.lower() in ("db"): + elif key.lower() in ("db",): self.db = parser_dict[key] - elif key.lower() in ("gene"): + elif key.lower() in ("gene",): self.gene = parser_dict[key] - elif key.lower() in ("email"): + elif key.lower() in ("email",): self.email = parser_dict[key] - elif key.lower() in ("api"): + elif key.lower() in ("api",): self.api = parser_dict[key] - elif key.lower() in ("thread"): + elif key.lower() in ("thread",): self.thread = parser_dict[key] - elif key.lower() in ("memory"): + elif key.lower() in ("memory",): self.memory = parser_dict[key] - elif key.lower() in ("outdir"): + elif key.lower() in ("outdir",): self.outdir = parser_dict[key] - elif key.lower() in ("runname"): + elif key.lower() in ("runname",): self.runname = parser_dict[key] - elif key.lower() in ("mode"): + elif key.lower() in ("mode",): self.mode = parser_dict[key] - elif key.lower() in ("continue"): + elif key.lower() in ("continue",): self.continue_from_previous = parser_dict[key] - elif key.lower() in ("step"): + elif key.lower() in ("step",): self.step = parser_dict[key] - elif key.lower() in ("level"): + elif key.lower() in ("level",): self.level = parser_dict[key] elif key.lower() in ("queryonly", "all"): if key.lower() == "queryonly": self.queryonly = parser_dict[key] elif key.lower() == "all": - self.queryonly = ~parser_dicy[key] - elif key.lower() in ("verbose"): + self.queryonly = not parser_dict[key] + elif key.lower() in ("verbose",): self.verbose = parser_dict[key] - elif key.lower() in ("maxoutgroup"): + elif key.lower() in ("maxoutgroup",): self.maxoutgroup = parser_dict[key] - elif key.lower() in ("collapsedistcutoff"): + elif key.lower() in ("collapsedistcutoff",): self.collapsedistcutoff = parser_dict[key] - elif key.lower() in ("collapsebscutoff"): + elif key.lower() in ("collapsebscutoff",): self.collapsebscutoff = parser_dict[key] - elif key.lower() in ("bootstrap"): + elif key.lower() in ("bootstrap",): self.bootstrap = parser_dict[key] - elif key.lower() in ("nosolveflat"): - self.solveflat = ~parser_dict[key] - elif key.lower() in ("regex"): + elif key.lower() in ("nosolveflat",): + self.solveflat = not parser_dict[key] + elif key.lower() in ("solveflat",): + self.solveflat = parser_dict[key] + elif key.lower() in ("regex",): self.regex = parser_dict[key] - elif key.lower() in ("avx"): + elif key.lower() in ("avx",): self.avx = parser_dict[key] - elif key.lower() in ("suspicious"): + elif key.lower() in ("suspicious",): self.suspicious = parser_dict[key] - elif key.lower() in ("allow-innertrimming"): + elif key.lower() in ("allow-innertrimming",): self.allow_innertrimming = parser_dict[key] - elif key.lower() in ("criterion"): + elif key.lower() in ("criterion",): self.criterion = parser_dict[key] - elif key.lower() in ("nocachedb"): - self.cachedb = ~parser_dict[key] - elif key.lower() in ("usecache"): + elif key.lower() in ("nocachedb",): + self.cachedb = not parser_dict[key] + elif key.lower() in ("cachedb",): + self.cachedb = parser_dict[key] + elif key.lower() in ("usecache",): self.usecache = parser_dict[key] elif key.lower() in ("tableformat", "matrixformat"): self.tableformat = parser_dict[key] - elif key.lower() in ("nosearchresult"): + elif key.lower() in ("nosearchresult",): self.nosearchresult = parser_dict[key] elif key.lower() in ("confident", "confident_db"): self.confident = parser_dict[key] - elif key.lower() in ("terminate"): - self.terminate = parse_dict[key] + elif key.lower() in ("terminate",): + self.terminate = parser_dict[key] # Method options - elif key.lower() in ("search"): + elif key.lower() in ("search",): self.method.search = parser_dict[key] - elif key.lower() in ("alignment"): + elif key.lower() in ("alignment",): self.method.alignment = parser_dict[key] - elif key.lower() in ("notcs"): - self.method.tcs = ~parser_dict[key] - elif key.lower() in ("trim"): + elif key.lower() in ("notcs",): + self.method.tcs = not parser_dict[key] + elif key.lower() in ("trim",): self.method.trim = parser_dict[key] - elif key.lower() in ("modeltest"): + elif key.lower() in ("modeltest",): self.method.modeltest = parser_dict[key] - elif key.lower() in ("tree"): + elif key.lower() in ("tree",): self.method.tree = parser_dict[key] # Visualize options - elif key.lower() in ("bscutoff"): + elif key.lower() in ("bscutoff",): self.visualize.bscutoff = parser_dict[key] - elif key.lower() in ("bootstrapcutoff"): + elif key.lower() in ("bootstrapcutoff",): self.visualize.bscutoff = parser_dict[key] - elif key.lower() in ("highlight"): + elif key.lower() in ("highlight",): self.visualize.highlight = parser_dict[key] - elif key.lower() in ("heightmultiplier"): + elif key.lower() in ("heightmultiplier",): self.visualize.heightmultiplier = parser_dict[key] - elif key.lower() in ("maxwordlength"): + elif key.lower() in ("maxwordlength",): self.visualize.maxwordlength = parser_dict[key] - elif key.lower() in ("backgroundcolor"): + elif key.lower() in ("backgroundcolor",): self.visualize.backgroundcolor = parser_dict[key] - elif key.lower() in ("outgroupcolor"): + elif key.lower() in ("outgroupcolor",): self.visualize.outgroupcolor = parser_dict[key] - elif key.lower() in ("ftype"): + elif key.lower() in ("ftype",): self.visualize.ftype = parser_dict[key] - elif key.lower() in ("fsize"): + elif key.lower() in ("fsize",): self.visualize.fsize = parser_dict[key] - elif key.lower() in ("fsize_bootstrap"): + elif key.lower() in ("fsize_bootstrap",): self.visualize.fsize_bootstrap = parser_dict[key] # Cluster options elif key.lower() in ("cluster-cutoff", "clustering_cutoff"): - self.cluster.evalue = parser_dict[key] + self.cluster.cutoff = parser_dict[key] elif key.lower() in ("evalue", "cluster-evalue"): self.cluster.evalue = parser_dict[key] - elif key.lower() in ("wordsize"): - self.cluster.wordsize = parser_dict[key] - elif key.lower() in ("outgroupoffset"): + elif key.lower() in ("wordsize",): self.cluster.wordsize = parser_dict[key] + elif key.lower() in ("outgroupoffset",): + self.cluster.outgroupoffset = parser_dict[key] elif key.lower() in ( "max_target_seqs", "max-target-seqs", @@ -247,17 +251,17 @@ def update_from_preset(self, preset_file): self.cluster.max_target_seqs = parser_dict[key] # MAFFT options - elif key.lower() in ("mafft-algorithm"): + elif key.lower() in ("mafft-algorithm",): self.mafft.algorithm = parser_dict[key] - elif key.lower() in ("mafft-op"): + elif key.lower() in ("mafft-op",): self.mafft.op = parser_dict[key] - elif key.lower() in ("mafft-ep"): + elif key.lower() in ("mafft-ep",): self.mafft.ep = parser_dict[key] # TrimAl options - elif key.lower() in ("trimal-algorithm"): + elif key.lower() in ("trimal-algorithm",): self.trimal.algorithm = parser_dict[key] - elif key.lower() in ("trimal-gt"): + elif key.lower() in ("trimal-gt",): self.trimal.gt = parser_dict[key] else: print( @@ -331,8 +335,8 @@ def update_from_parser(self, parser): pass try: - if not parser.continue_from_previous is None: - self.continue_from_previous = parser.continue_from_previous + if parser.continue_from_previous is True: + self.continue_from_previous = True except: pass @@ -350,7 +354,7 @@ def update_from_parser(self, parser): try: if parser.all is True: - self.queryonly = ~parser.all + self.queryonly = False except: pass @@ -379,7 +383,7 @@ def update_from_parser(self, parser): pass try: - if not parser.notcs is None: + if parser.notcs is True: self.method.tcs = False except: pass @@ -511,8 +515,8 @@ def update_from_parser(self, parser): pass try: - if not parser.cluster_outgroupoffset is None: - self.cluster.outgroupoffset = parser.cluster_outgroupoffset + if not parser.outgroupoffset is None: + self.cluster.outgroupoffset = parser.outgroupoffset except: pass @@ -571,8 +575,8 @@ def update_from_parser(self, parser): pass try: - if not parser.allow_innertrimming is None: - self.allow_innertrimming = parser.allow_innertrimming + if parser.allow_innertrimming is True: + self.allow_innertrimming = True except: pass @@ -972,7 +976,7 @@ def validate(self): if not (type(self.method.search) is str): list_error.append(f"search method should be string") else: - self.method.search = search_adjust[self.method.search.lower()] + self.method.search = search_adjust.get(self.method.search.lower(), self.method.search.lower()) if not (self.method.search in search): list_error.append(f"search method should be one of {str(search)}") @@ -989,7 +993,7 @@ def validate(self): if not (type(self.method.alignment) is str): list_error.append(f"align method should be string") else: - self.method.alignment = alignment_adjust[self.method.alignment.lower()] + self.method.alignment = alignment_adjust.get(self.method.alignment.lower(), self.method.alignment.lower()) if not (self.method.alignment in alignment): list_error.append(f"align method should be one of {str(alignment)}") @@ -1002,17 +1006,20 @@ def validate(self): ) self.method.tcs = False else: - # Check if tcs available - cmd = "export MAX_N_PID_4_TCOFFEE=4194304 | t_coffee -help" - return_code = subprocess.run( - cmd, - shell=True, - stdout=open(os.devnull, "wb"), - stderr=subprocess.STDOUT, - ).returncode - if return_code != 0: + # Only check that t-coffee is present on PATH; do NOT execute it here. + # The bioconda t-coffee SIGSEGV-crash-loops and consumes all memory on + # ANY invocation -- even `t_coffee -help` -- when the OS PID exceeds its + # compiled MAX_N_PID=260000 (i.e. on kernels with a large pid_max). Its + # PID indexes an undersized static array; the overflow faults and its + # signal handler re-faults forever. Running it just to detect it would + # hang option validation and eat RAM, so we never execute it. + import shutil + + if shutil.which("t_coffee") is None: print( - f"[WARNING] t-coffee (TCS) not installed! Excluding from analysis" + "[WARNING] t-coffee (TCS) not found on PATH; skipping TCS. " + "To enable TCS, build a working t-coffee with " + "tools/tcoffee/build_tcoffee_for_tcs.sh (see tools/tcoffee/README.md)." ) self.method.tcs = False @@ -1037,7 +1044,7 @@ def validate(self): if not (type(self.method.trim) is str): list_error.append(f"trim method should be string") else: - self.method.trim = trim_adjust[self.method.trim.lower()] + self.method.trim = trim_adjust.get(self.method.trim.lower(), self.method.trim.lower()) if not (self.method.trim in trim): list_error.append( f"trim method should be one of {str(trim_adjust.keys())}" @@ -1114,7 +1121,7 @@ def validate(self): list_warning.append( f"option jmodeltest will be substituted to modeltest-ng" ) - self.method.modeltest = modeltest_adjust[self.method.modeltest.lower()] + self.method.modeltest = modeltest_adjust.get(self.method.modeltest.lower(), self.method.modeltest.lower()) if not (self.method.modeltest in modeltest): list_error.append(f"modeltest method should be one of {str(modeltest)}") @@ -1141,7 +1148,7 @@ def validate(self): if not (type(self.method.tree) is str): list_error.append(f"tree method should be string") else: - self.method.tree = tree_adjust[self.method.tree.lower()] + self.method.tree = tree_adjust.get(self.method.tree.lower(), self.method.tree.lower()) if not (self.method.tree in tree): list_error.append(f"tree method should be one of {str(tree)}") @@ -1181,7 +1188,7 @@ def validate(self): else: if not (isvalidcolor(self.visualize.highlight)): list_error.append( - f"in --highlight, color {color} does not seems to be valid svg color nor hex code" + f"in --highlight, color {self.visualize.highlight} does not seem to be a valid svg color nor hex code" ) else: self.visualize.highlight = self.visualize.highlight.lower() @@ -1244,7 +1251,7 @@ def validate(self): else: if not (isvalidcolor(self.visualize.outgroupcolor)): list_error.append( - f"in --outgroupcolor, color {color} does not seems to be valid svg color nor hex code" + f"in --outgroupcolor, color {self.visualize.outgroupcolor} does not seem to be a valid svg color nor hex code" ) else: self.visualize.outgroupcolor = self.visualize.outgroupcolor.lower() @@ -1299,6 +1306,7 @@ def validate(self): self.maxoutgroup = int(self.maxoutgroup) if self.maxoutgroup < 1: list_warning.append(f"invalid maxoutgroup, automatically selecting 1") + self.maxoutgroup = 1 except: list_warning.append(f"invalid maxoutgroup, automatically selecting 1") self.maxoutgroup = 1 @@ -1431,7 +1439,7 @@ def validate(self): list_warning.append( "max_target_seqs should be positive, setting to default 100" ) - self.cluster.outgroupoffset = 100 + self.cluster.max_target_seqs = 100 except: list_error.append("max_target_seqs should be positive integer") @@ -1500,8 +1508,8 @@ def validate(self): try: self.trimal.algorithm = str(self.trimal.algorithm) if not (self.trimal.algorithm.lower() in ("auto", "gt")): - list_warning( - f"Invalid trimal algorithm {self.trimal.algorithm}. Chaniging to gt" + list_warning.append( + f"Invalid trimal algorithm {self.trimal.algorithm}. Changing to gt" ) self.trimal.algorithm = "gt" except: @@ -1516,7 +1524,7 @@ def validate(self): if str(self.method.trim).lower() == "trimal": try: self.trimal.gt = float(self.trimal.gt) - if self.trimal.gt < 0 and self.trimal.gt >= 1: + if self.trimal.gt < 0 or self.trimal.gt >= 1: list_warning.append( f"trimal gt value should be between 1 and 0, setting to 0.2" ) diff --git a/funvip/src/validation.py b/funvip/src/validation.py deleted file mode 100644 index 67b1df2..0000000 --- a/funvip/src/validation.py +++ /dev/null @@ -1,38 +0,0 @@ -from Bio import SeqIO -from Bio.Seq import Seq -from Bio.SeqRecord import SeqRecord - -from io import StringIO - -# from .logger import Mes - - -# validate if string is good sequence -def validate_seq(seqstring): - if seqstring.startswith(">"): - tmp = StringIO(seqstring) - - try: - seqlist = list(SeqIO.parse(tmp, fasta)) - return "seqrecord", seqlist - except: - logging.warning(f"Invalid seqrecord {seqstring}") - return "invalid", None - - else: - seqstring = seqstring.replace(" ", "").replace("\n", "") - - try: - seq = Seq(seqstring) - seqlist = [ - SeqRecord( - seq, - id="input", - description="tmp", - annotations={"molecule_type": "DNA"}, - ) - ] - return "seqrecord", seqlist - except: - logging.warning(f"Invalid sequence {seqstring}") - return "invalid", None diff --git a/funvip/src/version.py b/funvip/src/version.py index a6e0d06..30df29e 100644 --- a/funvip/src/version.py +++ b/funvip/src/version.py @@ -1,346 +1,207 @@ -# Version management module +# Version management + external-tool preflight import sys +import shutil +import logging import subprocess -from importlib.metadata import version +from importlib.metadata import version as _pkg_version -# import +from funvip.src.exceptions import ConfigError -""" -version = { - "FunVIP": "0.3.19.0.1.3", - "BLASTn": "", - "MMseqs2": "", - "MAFFT": "", - "TrimAl": "", - "Gblocks": "0.91b", - "FastTree": "", - "IQTREE2": "", - "RAxML": "", -} -""" +def _probe(candidates, args, parse): + """Return a version string for the first available command among `candidates`, + or '' if none is found or the probe/parse fails. Never raises.""" + for cmd in candidates: + if shutil.which(cmd) is None: + continue + try: + result = subprocess.run( + [cmd, *args], capture_output=True, text=True, timeout=30 + ) + except Exception: + continue + try: + parsed = parse(result.stdout or "", result.stderr or "") + except Exception: + parsed = "" + if parsed: + return parsed.strip() + return "" class Version: + """Best-effort version stamp for the report. Probes only succeed for tools + that are installed; anything missing or unparseable is left as '' rather than + crashing the run (a run only needs the tools for its selected methods, which + the preflight verifies separately).""" + def __init__(self, opt, path): - self.FunVIP = "" - self.GenMine = "" - self.BLASTn = "" - self.MMseqs2 = "" - self.MAFFT = "" - self.trimAl = "" + win = sys.platform == "win32" + ext = f"{path.sys_path}/external" + + def cand(linux, win_path): + return [win_path] if win else linux + + try: + self.FunVIP = _pkg_version("FunVIP") + except Exception: + self.FunVIP = "" + try: + self.GenMine = _pkg_version("GenMine") + except Exception: + self.GenMine = "" + + self.BLASTn = _probe( + cand(["blastn"], f"{ext}/BLAST_Windows/bin/blastn.exe"), + ["-version"], + lambda o, e: o.split("\n")[0].split(" ")[1] if o.strip() else "", + ) + self.MMseqs2 = _probe( + cand(["mmseqs"], f"{ext}/mmseqs_Windows/mmseqs.bat"), + ["-h"], + lambda o, e: o.split("Version: ")[1].split("\n")[0] if "Version: " in o else "", + ) + self.MAFFT = _probe( + cand(["mafft"], f"{ext}/MAFFT_Windows/mafft-win/mafft.bat"), + ["--version"], + lambda o, e: e.split("\n")[-2].split(" ")[0] if e.strip() else "", + ) + self.trimAl = _probe( + cand(["trimal"], f"{ext}/trimal.v1.4/trimAl/bin/trimal.exe"), + ["--version"], + lambda o, e: o.split("\n")[1].split(" ")[1] if o.count("\n") >= 1 else "", + ) self.Gblocks = "0.91b" - self.Modeltest_NG = "" - self.FastTree = "" - self.IQTREE2 = "" - self.RAxML = "" - - # For windows platform - if sys.platform == "win32": - ### FunVIP - self.FunVIP = version("FunVIP") - - ### GenMine - self.GenMine = version("GenMine") - - ### BLASTn - CMD = [f"{path.sys_path}/external/BLAST_Windows/bin/blastn.exe", "-version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - ## tableformat - # blastn: blastn: 2.12.0+ - # Package: blast 2.12.0, build Jun 4 2021 03:25:07 - self.BLASTn = stdout_str.split("\n")[0].split(" ")[1].strip() - # print("BLASTn", self.BLASTn) - - ### MMSeqs2 - CMD = [ - f"{path.sys_path}/external/mmseqs_Windows/mmseqs.bat", - "-h", - ] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.MMseqs2 = stdout_str.split("Version: ")[1].split("\n")[0].strip() - # print("MMseqs2", self.MMseqs2) - - ### MAFFT - CMD = [ - f"{path.sys_path}/external/MAFFT_Windows/mafft-win/mafft.bat", - "--version", - ] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - # MAFFT, output is on the stderr - stderr_str = stderr.decode("utf-8") - self.MAFFT = stderr_str.split("\n")[-2].split(" ")[0].strip() - # print("MAFFT", self.MAFFT) - - ### TrimAl - CMD = [ - f"{path.sys_path}/external/trimal.v1.4/trimAl/bin/trimal.exe", - "--version", - ] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.trimAl = stdout_str.split("\n")[1].split(" ")[1] - # print("trimAl", self.trimAl) - - ### Modeltest-ng - ## Not supported in Windows - self.Modeltest_NG = "not supported" - - ### FastTree - ## Also use stderr of FastTree - CMD = [ - f"{path.sys_path}/external/FastTree_Windows/FastTree.exe", - "-expert", + self.Modeltest_NG = ( + "not supported" + if win + else _probe( + ["modeltest-ng"], + ["--version"], + lambda o, e: o.split("ModelTest-NG ")[1].split(" ")[0] + if "ModelTest-NG " in o + else "", + ) + ) + self.FastTree = _probe( + cand(["FastTree", "fasttree"], f"{ext}/FastTree_Windows/FastTree.exe"), + ["-expert"], + lambda o, e: e.split(" ")[4] if len(e.split(" ")) > 4 else "", + ) + self.IQTREE2 = _probe( + cand(["iqtree", "iqtree2"], f"{ext}/iqtree/bin/iqtree2.exe"), + ["--version"], + lambda o, e: o.split(" ")[3] if len(o.split(" ")) > 3 else "", + ) + if win: + raxml_cand = [ + f"{ext}/RAxML_Windows/raxmlHPC-PTHREADS-AVX2.exe" + if opt.avx + else f"{ext}/RAxML_Windows/raxmlHPC-PTHREADS-SSE3.exe" ] - - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - stderr_str = stderr.decode("utf-8") - self.FastTree = stderr_str.split(" ")[4] - # print("FastTree", self.FastTree) - - ### IQTREE - CMD = [ - f"{path.sys_path}/external/iqtree/bin/iqtree2.exe", - "--version", + elif opt.avx: + raxml_cand = [ + "raxmlHPC-PTHREADS-AVX2", + "raxmlHPC-PTHREADS-SSE3", + "raxmlHPC", ] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.IQTREE2 = stdout_str.split(" ")[3] - # print("IQTREE2", self.IQTREE2) - - ### RAxML - if opt.avx is True: - CMD = [ - f"{path.sys_path}/external/RAxML_Windows/raxmlHPC-PTHREADS-AVX2.exe", - "-v", - ] - else: - CMD = [ - f"{path.sys_path}/external/RAxML_Windows/raxmlHPC-PTHREADS-SSE3.exe", - "-v", - ] - - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.RAxML = stdout_str.split("\n")[2].split(" ")[4] - # print("RAxML", self.RAxML) - - # For apple silicon platform - elif sys.platform == "darwin": - ### FunVIP - self.FunVIP = version("FunVIP") - - ### GenMine - self.GenMine = version("GenMine") - - ### BLASTn - CMD = ["blastn", "-version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - ## Format - # blastn: blastn: 2.12.0+ - # Package: blast 2.12.0, build Jun 4 2021 03:25:07 - self.BLASTn = stdout_str.split("\n")[0].split(" ")[1].strip() - # print("BLASTn", self.BLASTn) - - ### MMSeqs2 - CMD = ["mmseqs", "-h"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.MMseqs2 = stdout_str.split("Version: ")[1].split("\n")[0].strip() - # print("MMseqs2", self.MMseqs2) - - ### MAFFT - CMD = ["mafft", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - # MAFFT, output is on the stderr - stderr_str = stderr.decode("utf-8") - self.MAFFT = stderr_str.split("\n")[-2].split(" ")[0].strip() - # print("MAFFT", self.MAFFT) - - ### TrimAl - CMD = ["trimal", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.trimAl = stdout_str.split("\n")[1].split(" ")[1] - # print("trimAl", self.trimAl) - - ### Modeltest-ng - ## Not supported in apple silicon - self.Modeltest_NG = "not supported" - - ### FastTree - ## Also use stderr of FastTree - CMD = ["FastTree", "-expert"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stderr_str = stderr.decode("utf-8") - self.FastTree = stderr_str.split(" ")[4] - # print("FastTree", self.FastTree) - - ### IQTREE - CMD = ["iqtree", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.IQTREE2 = stdout_str.split(" ")[3] - # print("IQTREE2", self.IQTREE2) - - ### RAxML - if opt.avx is True: - CMD = ["raxmlHPC-PTHREADS-AVX2", "-v"] - else: - CMD = ["raxmlHPC-PTHREADS-SSE3", "-v"] - - try: - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - # For arm native raxml - except: - CMD = ["raxmlHPC", "-v"] - - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.RAxML = stdout_str.split("\n")[2].split(" ")[4] - # print("RAxML", self.RAxML) - - # For linux platform else: - ### FunVIP - self.FunVIP = version("FunVIP") - - ### GenMine - self.GenMine = version("GenMine") - - ### BLASTn - CMD = ["blastn", "-version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - ## Format - # blastn: blastn: 2.12.0+ - # Package: blast 2.12.0, build Jun 4 2021 03:25:07 - self.BLASTn = stdout_str.split("\n")[0].split(" ")[1].strip() - # print("BLASTn", self.BLASTn) - - ### MMSeqs2 - CMD = ["mmseqs", "-h"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.MMseqs2 = stdout_str.split("Version: ")[1].split("\n")[0].strip() - # print("MMseqs2", self.MMseqs2) - - ### MAFFT - CMD = ["mafft", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - # MAFFT, output is on the stderr - stderr_str = stderr.decode("utf-8") - self.MAFFT = stderr_str.split("\n")[-2].split(" ")[0].strip() - # print("MAFFT", self.MAFFT) - - ### TrimAl - CMD = ["trimal", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.trimAl = stdout_str.split("\n")[1].split(" ")[1] - # print("trimAl", self.trimAl) - - ### Modeltest-ng - ## Not supported in Windows - CMD = ["modeltest-ng", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.Modeltest_NG = stdout_str.split("ModelTest-NG ")[1].split(" ")[0] - - ### FastTree - ## Also use stderr of FastTree - CMD = ["FastTree", "-expert"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stderr_str = stderr.decode("utf-8") - self.FastTree = stderr_str.split(" ")[4] - # print("FastTree", self.FastTree) - - ### IQTREE - CMD = ["iqtree", "--version"] - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.IQTREE2 = stdout_str.split(" ")[3] - # print("IQTREE2", self.IQTREE2) - - ### RAxML - if opt.avx is True: - CMD = ["raxmlHPC-PTHREADS-AVX2", "-v"] - else: - CMD = ["raxmlHPC-PTHREADS-SSE3", "-v"] - - result = subprocess.Popen( - CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = result.communicate() - stdout_str = stdout.decode("utf-8") - self.RAxML = stdout_str.split("\n")[2].split(" ")[4] + raxml_cand = ["raxmlHPC-PTHREADS-SSE3", "raxmlHPC"] + self.RAxML = _probe( + raxml_cand, + ["-v"], + lambda o, e: o.split("\n")[2].split(" ")[4] + if o.count("\n") >= 2 and len(o.split("\n")[2].split(" ")) > 4 + else "", + ) + + +def _needed_tools(opt): + """(label, [command candidates], required) for the tools the selected methods + will actually invoke.""" + tools = [] + + search = str(opt.method.search).lower() + if search == "blast": + tools += [ + ("BLAST+ (blastn)", ["blastn"], True), + ("BLAST+ (makeblastdb)", ["makeblastdb"], True), + ] + elif search == "mmseqs": + tools += [("MMseqs2", ["mmseqs"], True)] + + tools += [("MAFFT", ["mafft"], True)] + + if opt.method.tcs is True: + tools += [("T-COFFEE (TCS)", ["t_coffee"], False)] + + trim = str(opt.method.trim).lower() + if trim == "trimal": + tools += [("trimAl", ["trimal"], True)] + elif trim == "gblocks": + tools += [("Gblocks", ["Gblocks"], True)] + + modeltest = str(opt.method.modeltest).lower() + if modeltest in ("modeltest-ng", "modeltestng"): + tools += [("modeltest-ng", ["modeltest-ng"], True)] + elif modeltest == "iqtree": + tools += [("IQ-TREE (ModelFinder)", ["iqtree", "iqtree2"], True)] + + tree = str(opt.method.tree).lower() + if tree == "fasttree": + tools += [("FastTree", ["FastTree", "fasttree"], True)] + elif tree == "iqtree": + tools += [("IQ-TREE", ["iqtree", "iqtree2"], True)] + elif tree == "raxml": + cands = ( + ["raxmlHPC-PTHREADS-AVX2", "raxmlHPC-PTHREADS-SSE3", "raxmlHPC"] + if opt.avx + else ["raxmlHPC-PTHREADS-SSE3", "raxmlHPC"] + ) + tools += [("RAxML", cands, True)] + + # de-duplicate by label (e.g. IQ-TREE can be both modeltest and tree) + seen, out = set(), [] + for label, cands, req in tools: + if label not in seen: + seen.add(label) + out.append((label, cands, req)) + return out + + +def preflight(opt): + """Verify the external tools the selected methods need are available; raise a + clear ConfigError listing what is missing. Optional tools (TCS) only warn. + On Windows the tools are bundled, so this is a no-op.""" + if sys.platform == "win32": + logging.info("Windows platform: using the bundled external tools") + return + + present, missing = [], [] + for label, candidates, required in _needed_tools(opt): + found = next((c for c in candidates if shutil.which(c)), None) + if found: + present.append(f"{label} [{found}]") + elif required: + missing.append((label, candidates)) + else: + logging.warning( + f"Optional tool {label} not found on PATH " + f"({', '.join(candidates)}); that step will be skipped" + ) + + if present: + logging.info("External tools found: " + ", ".join(present)) + + if missing: + detail = "\n".join( + f" - {label}: none of [{', '.join(cands)}] found on PATH" + for label, cands in missing + ) + raise ConfigError( + "Required external tools for the selected methods are not on PATH:\n" + f"{detail}\n" + "Install them (e.g. `conda install -c bioconda blast mmseqs2 mafft " + "trimal fasttree iqtree raxml modeltest-ng t-coffee`) or choose " + "different --search / --trim / --modeltest / --tree methods." + ) diff --git a/funvip/src/visualize.py b/funvip/src/visualize.py deleted file mode 100644 index c46ef17..0000000 --- a/funvip/src/visualize.py +++ /dev/null @@ -1,579 +0,0 @@ -# For final visualization of the FunVIP tree -from ete3 import ( - Tree, - TreeStyle, - NodeStyle, - TextFace, - CircleFace, - RectFace, - faces, -) -from funvip.src.tool import get_genus_species -import pandas as pd -import lxml.etree as ET -import re - -# Genus abbreviation -abbreviate = True -# True - "*", False - "T", "NT" -asterisk_type = False - -# Tree style designation -ts = TreeStyle() -ts.scale = 5000 -ts.branch_vertical_margin = 3 -ts.allow_face_overlap = True -ts.children_faces_on_top = True -ts.complete_branch_lines_when_necessary = False -ts.extra_branch_line_color = "black" -ts.margin_left = 200 -ts.margin_right = 200 -ts.margin_top = 200 -ts.margin_bottom = 200 - -name_offset = -1000 - - -def visualize( - outgroup, - out, - FunVIP_result, - type_file, - ml_tree, - tmp_tree, - bayesian_tree=None, -): - -# Visualize with Lim's FEP style -def visualize( - V=V, - path=path, - opt=opt -): - - # To make name in excel sheet meets to FunVIP format - def excel_sheetname_legal(string): - newick_illegal = ["'", "[", "]", ":", "*", "?", "/", "\\", ".", ",", "(", ")"] - for i in newick_illegal: - string = string.replace(i, "") - - string = string.replace(" ", " ") - string = string.replace(" ", "_") - # string = string.replace("-", "_") - - return string - - # get index from dataframe of leaf - def get_index(leaf): - print(leaf.name) - - dup_check = 0 - leaf_id_candidate = [] - for key in dict_indicated_name: - if key in leaf.name: - dup_check += 1 - leaf_id_candidate.append(key) - - if dup_check == 0: - print(dup_check) - # print(dict_indicated_name) - print(leaf.name) - raise Exception - elif dup_check > 1: - remove_set = set() - for _id1 in leaf_id_candidate: - for _id2 in leaf_id_candidate: - if _id1 != _id2 and _id1 in _id2: - remove_set.add(_id1) - - for r in remove_set: - leaf_id_candidate.remove(r) - - if len(leaf_id_candidate) != 1: - print(leaf_id_candidate) - raise Exception - - leaf_id = leaf_id_candidate[0] - for n, i in enumerate(df["ID"]): - if df["ID"][n] == dict_indicated_name[leaf_id]: - return n - - print(leaf.name) - print(leaf_id) - raise Exception - - ## Read result - if FunVIP_result.endswith(".csv"): - encodings = ["UTF-8", "latin-1", "cp1252"] - for enc in encodings: - try: - df = pd.read_csv(FunVIP_result, encoding=enc, quoting=1) - break - except UnicodeDecodeError: - continue - elif FunVIP_result.endswith(".xlsx"): - df = pd.read_excel(FunVIP_result) - else: - raise Exception - - # Make excel pair - # {"legalized_ID" : "ID"} - dict_indicated_name = {} - for n, i in enumerate(df["ID"]): - dict_indicated_name[excel_sheetname_legal(i)] = i - - - def is_outgroup(leaf): - if outgroup in leaf.name: - print(leaf.name) - return True - else: - return False - - # get id of from leaf - def get_id(leaf): - if is_outgroup(leaf): - return leaf.name - else: - n = get_index(leaf) - return df["ID"][n] - - # monophyletic check function - def monophyletic(clade): - set_taxon = set() - for leaf in clade.iter_leaves(): - set_taxon.add(get_taxon(leaf)) - - if len(set_taxon) == 1: - return True - elif len(set_taxon) == 0: - raise Exception - else: - return False - - # check if clade includes query - def query_found(clade): - for leaf in clade.iter_leaves(): - if not (is_outgroup(leaf)): - n = get_index(leaf) - if df["DATATYPE"][n] == "query": - return True - return False - - # check if clade is new_speces - def new_species(clade): - set_taxon = set() - for leaf in clade.iter_leaves(): - set_taxon.add(get_taxon(leaf)) - - if len(set_taxon) == 1: - if "sp." in list(set_taxon)[0][1]: - return True - else: - return False - else: - raise Exception - - # get taxon from leaf - def get_taxon(leaf): - if is_outgroup(leaf): - return get_genus_species(leaf.name) - else: - n = get_index(leaf) - print(leaf.name, df["SPECIES_ASSIGNED"][n]) - return get_genus_species(df["SPECIES_ASSIGNED"][n]) - - # Update background color for monophyletic clade - # input tree - def find_and_color_monophyletic_clade(t): - for clade in t.children: - # Check if clade is monophyletic - if is_monophyletic(funinfo_dict, query_list, db_list, outgroup, opt, sp_cnt, clade, gene, taxon): - # If query is in it - if query_found(clade): - if new_species(clade): - # Color with new species color - child.img_style["bgcolor"] = "paleturquoise" - # Change abbreviate to False if you want to show full name of new species - - ''' - for leaf in child.iter_leaves(): - print(leaf.name) - - print(f"DEBUG point 1: {list(child.iter_leaves())[0]}") - print( - f"DEBUG point 2: {get_taxon(list(child.iter_leaves())[0])}" - ) - ''' - - taxon = get_taxon(list(child.iter_leaves())[0]) - print(f"Taxon: {taxon}") - # raise Exception - child.add_face( - TextFace( - taxon[0] + " " + taxon[1], - ftype="Arial", - fsize=24, - fstyle="italic", - ), - column=0, - position="float-behind", - ) - - else: - # Color with recorded species color - child.img_style["bgcolor"] = "lightgrey" - taxon = get_genus_species(list(child.iter_leaves())[0].name) - child.add_face( - TextFace( - taxon[0] + " " + taxon[1], - ftype="Arial", - fsize=24, - fstyle="italic", - ), - column=0, - position="float-behind", - ) - - else: - find_and_color_monophyletic_clade(child) - - - ## Start of the function - # Read hashed tree - t_ml = Tree(f"{path.out_tree}/hash/hash_{opt.runname}_{group}_{gene}.nwk") - - ''' - t_ml_leaves = set(str(l.name).replace("-", "_").replace("/", "") for l in t_ml) - t_ml_leaves = set( - name.replace("__", "") if name.endswith("__") else name for name in t_ml_leaves - ) - t_ml_sub = t_ml.get_descendants() - ''' - - find_and_color_monophyletic_clade(t_ml) - - # patch.py file - #from patch import patch - #patch() - - # t_ml.write(format=2, outfile="reallytemporarytreetocheck.nwk") - # raise Exception - - # Visualization - # Edit writings - for leaf in t_ml.iter_leaves(): - leaf.img_style["draw_descendants"] = False - space_text = TextFace( - " ", - fsize=8, - ftype="Arial", - fgcolor="black", - ) - - # Else each of the text should be inspected - if not ("sp." in get_taxon(leaf)[1]): - try: - n = get_index(leaf) - ifquery = df["DATATYPE"][n] - except: - ifquery = "outgroup" - # If query - if ifquery == "query": - id_text = id_text = TextFace( - get_id(leaf), - fsize=16, - ftype="Arial", - bold=True, - fgcolor="black", - ) - - leaf.add_face(space_text, 1, position="branch-right") - leaf.add_face(id_text, 2, position="branch-right") - - else: - genus_text = TextFace( - get_taxon(leaf)[0], - fsize=16, - ftype="Arial", - fstyle="italic", - fgcolor="black", - ) - species_text = TextFace( - get_taxon(leaf)[1], - fsize=16, - ftype="Arial", - fstyle="italic", - fgcolor="black", - ) - - id_text = id_text = TextFace( - get_id(leaf), - fsize=16, - ftype="Arial", - fgcolor="black", - ) - leaf.add_face(space_text, 1, position="branch-right") - leaf.add_face(genus_text, 2, position="branch-right") - leaf.add_face(space_text, 3, position="branch-right") - leaf.add_face(species_text, 4, position="branch-right") - leaf.add_face(space_text, 5, position="branch-right") - leaf.add_face(space_text, 6, position="branch-right") - leaf.add_face(id_text, 7, position="branch-right") - - # Add stars for type - """ - if excel_sheetname_legal(get_id(leaf)) in dict_type: - if dict_type[excel_sheetname_legal(get_id(leaf))]: - if asterisk_type is True: - type_text = TextFace( - "*", - fsize=16, - ftype="Arial", - fgcolor="black", - ) - else: - type_text = TextFace( - str( - dict_type[excel_sheetname_legal(get_id(leaf))] - ).upper(), - fsize=12, - ftype="Arial", - fgcolor="black", - ) - - # leaf.add_face(space_text, 8, position="branch-right") - leaf.add_face(type_text, 9, position="branch-right") - else: - print(f"{leaf.name} not found in dict_type") - """ - - # For new species, all of them are our species, so do not have to draw species - else: - id_text = id_text = TextFace( - get_id(leaf), - fsize=16, - ftype="Arial", - bold=True, - fgcolor="black", - ) - - leaf.add_face(space_text, 1, position="branch-right") - leaf.add_face(id_text, 2, position="branch-right") - - leaf.name = "" - - - # show branch support above 70% - for node in t_ml.traverse(): - # change this part when debugging flat trees - node.img_style["size"] = 0 # removing circles whien size is 0 - node.img_style["vt_line_width"] = 2 - node.img_style["hz_line_width"] = 2 - - if node.support >= 70 and node.support < 100.1: - # node.add_face without generating extra line - # add_face_to_node - if bayesian_tree is None: - node.add_face( - TextFace( - f"{int(node.support)}", - fsize=12, - fstyle="Arial", - ), - column=0, - position="float", - ) - - else: - node.add_face( - TextFace( - f"{float(node.support)}", - fsize=12, - fstyle="Arial", - ), - column=0, - position="float", - ) - - # If bootstrap and bayesian pp both 100 - if node.support == 100.1: - # node.img_style["vt_line_width"] = 4 - node.img_style["hz_line_width"] = 8 - - # Imaging - for node in t_ml.traverse(): - node.img_style["size"] = 0 - - t_ml.render(out, tree_style=ts) - - # Polishing - tree_xml = ET.parse(f"{out}") - # in tree_xml, find all group - _group = list(tree_xml.iter("{http://www.w3.org/2000/svg}g")) - group_list = list(_group[0].findall("{http://www.w3.org/2000/svg}g")) - - # Find the width attribute in the root element - # Hard-coding for Penicillium project, should be fixed later - svg_width = 5000 - - rect_y_coords = [] - - # shorten height of background rectangle - for group in group_list: - if len(list(group.findall("{http://www.w3.org/2000/svg}rect"))) == 1: - if group.get("fill") in ( - "#d3d3d3", - "#afeeee", - ): - rect = list(group.findall("{http://www.w3.org/2000/svg}rect"))[0] - rect.set("width", f'{int(float(rect.get("width"))+2000)}') - rect.set("height", f'{int(float(rect.get("height"))-5)}') - rect.set("y", f"{int(rect.get('y'))+2}") - rect_y_coords.append(int(rect.get("y"))) - - # for taxons, gather all texts - text_list = list(tree_xml.iter("{http://www.w3.org/2000/svg}text")) - - # Change this module to be worked with FI hash - for text in text_list: - # Decide if string of the tree is bootstrap, scale, taxon or id - # taxon_list = [" ".join(x) for x in self.collapse_dict.keys()] - try: - float(text.text) - if float(text.text) == 0.05: - text_type = "scale" - else: - text_type = "bootstrap" - except: - try: - int(text.text) - text_type = "bootstrap" - except: - if text.text == "0.05": - text_type = "scale" - elif str(text.text) == "*" or str(text.text) in ("T", "NT", "ET", "HT"): - text_type = "type" - elif text.get("font-size") == "24pt": - text_type = "species_group" - print(f"DEBUG species_group {text.text}") - - else: - text_type = "" - - # relocate text position little bit for better visualization - text.set("y", f'{int(float(text.get("y")))-2}') - - if text_type == "bootstrap": - if bayesian_tree is None: - # float(text.text) - # print(f"bootstrap: {text.text}") - original_text = str(int(float(text.text))) - # move text a little bit higher position - text.set("y", f'{int(text.get("y"))+8}') - text.set("x", f'{int(text.get("x"))-5}') - # When pp value is under 1 - - """ - text.text = text.text.replace(".0", "/0.") - text.text = text.text.replace(".1", "/*") - - if text.text.endswith("."): - text.text = text.text + "00" - """ - - else: - float(text.text) - # print(f"bootstrap: {text.text}") - original_text = text.text - # move text a little bit higher position - text.set("y", f'{int(text.get("y"))+8}') - text.set("x", f'{int(text.get("x"))-5}') - # When pp value is under 1 - text.text = text.text.replace(".0", "/0.") - - try: - if float(text.text.split("/")[1]) < 0.95: - text.text = text.text.split("/")[0] + "/-" - except: - # When pp value does not exists - if not (".") in text.text: - text.text = text.text + "/-" - - pass - - text.text = text.text.replace(".1", "/*") - - if text.text.endswith("."): - text.text = text.text + "00" - - print(f"Original text {original_text} has been changed to {text.text}") - - if text_type == "species_group": - parent = text.getparent() - - raw_transform = parent.get("transform") - # print(raw_transform) - transform_values = [ - float(x.strip()) - for x in raw_transform.split("(")[1].split(")")[0].split(",") - ] - # X value - # print(transform_values[4]) - transform_values[4] = svg_width - len(text.text) * 16 - name_offset - transform_values = [str(x) for x in transform_values] - - # print(transform_values) - parent.set("transform", f'matrix({", ".join(transform_values)})') - - # Y value - - # parent.set("x", ) - # print(f'matrix({", ".join(transform_values)})') - # text.set("x", f"{int(text.get('x')) + 800}") - - if text_type == "type": - print("TYPE") - print(text.text) - print(text.get("y"), text.get("x")) - text.set("y", f'{int(text.get("y"))-4}') - text.set("x", f'{int(text.get("x"))+2}') - - print(svg_width) - - # fit size of tree_xml to svg - # find svg from tree_xml - svg = list(tree_xml.iter("{http://www.w3.org/2000/svg}svg"))[0] - - # write to svg file - tree_xml.write( - "polished_" + out, - encoding="utf-8", - xml_declaration=True, - ) - - -# This should be edited in FunVIP implemented version -# outgroup = "Coleosporium" -outgroup = "Tylospora" - -# Output file name -# out = f"Delastria.svg" -out = f"Rhizopogon.svg" - -# FunVIP result -FunVIP_result = f"./hypogeous_final.result.csv" - -# Type file name - DB file that includes type (holotype etc) information -type_file = f"./hypogeous_final.result.csv" -ml_tree = f"./hypogeous_final_Rhizopogon_concatenated.nwk" -tmp_tree = "tmp.nwk" - -FunVIP_visualizer( - outgroup=outgroup, - out=out, - FunVIP_result=FunVIP_result, - type_file=type_file, - ml_tree=ml_tree, - tmp_tree=tmp_tree, -) diff --git a/funvip/test_dataset/sanghuangporus/preset.yaml b/funvip/test_dataset/sanghuangporus/preset.yaml index f8fb130..b4aa777 100644 --- a/funvip/test_dataset/sanghuangporus/preset.yaml +++ b/funvip/test_dataset/sanghuangporus/preset.yaml @@ -4,7 +4,7 @@ GENE: - ITS CONCATENATE: true DB: -- FunVIP_Sanghaungporus_db.xlsx +- FunVIP_Sanghuangporus_db.xlsx QUERY: - FunVIP_Sanghuangporus_query.xlsx LEVEL: genus diff --git a/pyproject.toml b/pyproject.toml index e95dc62..fade60b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,34 +1,43 @@ [project] name = "FunVIP" -version = "0.5.9" +version = "1.0.0" description = "Fungal Validation & Identification Pipeline" authors = [{name = "Changwan Seo", email = "wan101010@snu.ac.kr"}] urls = { "Homepage" = "https://github.com/Changwanseo/FunVIP" } -requires-python = ">=3.9, <3.13" +requires-python = ">=3.9, <3.14" license = {text = "GPL-3.0"} dependencies = [ - "biopython==1.84", - "ete3==3.1.3", + "biopython>=1.84", + # ete4 has no Windows wheel on PyPI and cannot compile there, so it is a + # dependency only off Windows. On Windows, FunVIP installs a prebuilt ete4 + # wheel bundled under funvip/_vendor/ete4_wheels on first run (see + # _ensure_ete4 in main.py); the wheel is installed with --no-deps, so ete4's + # own runtime deps are listed as Windows-only dependencies just below. + "ete4>=4.4.0,<4.5.0; platform_system != 'Windows'", + "bottle; platform_system == 'Windows'", + "cheroot; platform_system == 'Windows'", + "brotli; platform_system == 'Windows'", + "requests; platform_system == 'Windows'", "Cython", - "contourpy<1.3", + "contourpy>=1.0", "dendropy>=4.0.0, <5.0.0", "GenMine>=1.3.0, <1.5.0", "lxml", "matplotlib", - "numpy<2.0.0", - "openpyxl==3.1.0", - "pandas==2.2.2", + "numpy>=1.20.0,<2.0.0", + "openpyxl>=3.1.0", + "pandas>=2.2.2,<4.0.0", "psutil", "pyyaml", "sip>=4.19.4", "scikit-learn", "scipy", "tabulate", - "unidecode==1.2.0", - "xlrd==2.0.1", + "unidecode>=1.2.0", + "xlrd>=2.0.1", "xlsxwriter", - "xmltodict==0.12.0", - "PyQt5>=5.15.0; sys_platform!='darwin'", + "xmltodict>=0.12.0,<1.0.0", + "PyQt6>=6.0.0; sys_platform!='darwin'", ] [project.readme] @@ -39,15 +48,19 @@ content-type = "text/markdown" FunID = "funvip.main:main" FunVIP = "funvip.main:main" +[project.optional-dependencies] +test = ["pytest>=7.0"] + [tool.setuptools] include-package-data = true [tool.setuptools.packages.find] where = ["."] -include = ["funvip", "funvip.src", "funvip.external", "funvip.data", "funvip.preset", "funvip.test_dataset" ] +include = ["funvip", "funvip.src", "funvip.external", "funvip.data", "funvip.preset", "funvip.test_dataset", "funvip._vendor" ] [tool.setuptools.package-data] funvip = ["data/*", "external/*", "preset/*", "src/*", "test_dataset/*"] +"funvip._vendor" = ["ete4_wheels/*.whl", "ete4_wheels/*.txt", "ete4_wheels/LICENSE*", "ete4_wheels/README*"] [build-system] @@ -57,4 +70,11 @@ build-backend = "setuptools_ext" [metadata] obsoletes-dist = "FunID" +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: end-to-end tests that shell out to external tools (deselected by default)", +] +addopts = "-m 'not integration'" + diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..cebc423 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,16 @@ +# FunVIP tests + +Fast, dependency-light unit tests for the pure/near-pure functions, plus (marked) +slower integration checks. + +```bash +pip install -e ".[test]" # or: conda run -n FunVIP_dev pip install pytest +pytest # unit tests only (no external tools needed) +pytest -m integration # end-to-end (needs the bundled external tools + --email) +``` + +The unit tests (`test_hasher`, `test_logics`, `test_tool`, `test_validate_option`, +`test_cluster`) require none of the external aligners/tree tools and run in +seconds, so they are safe for CI. Integration tests that shell out to +BLAST/mmseqs/MAFFT/FastTree are marked `@pytest.mark.integration` and skipped by +default. diff --git a/tests/test_cluster.py b/tests/test_cluster.py new file mode 100644 index 0000000..5a9b33e --- /dev/null +++ b/tests/test_cluster.py @@ -0,0 +1,33 @@ +"""Tests for funvip.src.cluster._majority_top_group (the group tie-break).""" +import pandas as pd + +from funvip.src.cluster import _majority_top_group + + +def _df(rows): + return pd.DataFrame(rows, columns=["bitscore", "subject_group"]).sort_values( + "bitscore", ascending=False + ) + + +def test_unique_top_bitscore_group_wins(): + # A alone holds the top bitscore -> A, regardless of lower tiers. + assert _majority_top_group(_df([(250, "A"), (240, "B"), (240, "B")])) == "A" + + +def test_plurality_at_top_level_wins(): + assert _majority_top_group(_df([(250, "A"), (250, "A"), (250, "B")])) == "A" + + +def test_tie_folds_in_next_bitscore_level(): + # A and B tie at 250; fold in 240 where A has more -> A. + d = _df([(250, "A"), (250, "B"), (240, "A"), (240, "A")]) + assert _majority_top_group(d) == "A" + + +def test_full_tie_returns_highest_bitscore_hit(): + # Perfect tie at every level -> the highest-bitscore hit (deterministic). + d = _df([(250, "A"), (250, "B"), (240, "A"), (240, "B")]) + assert _majority_top_group(d) in {"A", "B"} + # first row after the descending sort is the tiebreak + assert _majority_top_group(d) == d.iloc[0]["subject_group"] diff --git a/tests/test_hasher.py b/tests/test_hasher.py new file mode 100644 index 0000000..128899a --- /dev/null +++ b/tests/test_hasher.py @@ -0,0 +1,76 @@ +"""Tests for funvip.src.hasher — hash encode/decode round-trips. + +Guards the single-pass HSHE decode against the previous N-alternative-regex +implementation across every newick/svg flag combination and the edge cases that +would break a naive rewrite (adjacent hashes, prefix-overlapping HS1HE/HS12HE, +tokens not in the dict, unicode/special-char values). +""" +import re + +import pandas as pd +import pytest + +from funvip.src import hasher +from funvip.src.hasher import newick_legal, svg_legal + + +def _reference_decode(hash_dict, content, newick=True, svg=False): + """The original N-alternative-regex decode, kept as an oracle.""" + if newick and svg: + hd = {re.escape(k): svg_legal(newick_legal(v)) for k, v in hash_dict.items()} + elif newick: + hd = {re.escape(k): newick_legal(v) for k, v in hash_dict.items()} + elif svg: + hd = {re.escape(k): svg_legal(v) for k, v in hash_dict.items()} + else: + hd = {re.escape(k): v for k, v in hash_dict.items()} + pattern = re.compile("|".join(hd.keys())) + return pattern.sub(lambda m: hd[re.escape(m.group(0))], content) + + +HASH_DICT = { + "HS0HE": "AB123 Aspergillus terreus (strain A):var. B; 'note' & ", + "HS1HE": "CD456 Penicillium citrinum", + "HS12HE": "EF789 Talaromyces_marneffei, sp. 3", + "HS2HE": "GH000 Fungus ésp. 中文", + "HS10HE": "IJ111 Genus species", +} +CONTENT = ( + "(HS0HE:0.1,(HS1HE:0.2,HS12HE:0.3):0.05,HS2HE:0.4);\n" + "adjacent:HS1HE HS0HEHS1HE end\n" + "overlap:HS12HE vs HS1HE vs HS10HE\n" + "not-in-dict:HS999HE and HS1HEZ trailing\n" + "plain line with no hashes\n" +) + + +@pytest.mark.parametrize("newick,svg", [(True, True), (True, False), (False, True), (False, False)]) +def test_decode_matches_reference(tmp_path, newick, svg): + infile = tmp_path / "in.txt" + infile.write_text(CONTENT) + out = tmp_path / "out.txt" + hasher.decode(HASH_DICT, str(infile), str(out), newick=newick, svg=svg) + assert out.read_text() == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg) + + +def test_decode_leaves_unknown_hash(tmp_path): + infile = tmp_path / "in.txt" + infile.write_text("keep HS999HE unchanged\n") + out = tmp_path / "out.txt" + hasher.decode({"HS0HE": "x"}, str(infile), str(out), newick=False) + assert out.read_text() == "keep HS999HE unchanged\n" + + +def test_decode_df_exact_cells_only(): + df = pd.DataFrame( + {"qseqid": ["HS0HE", "HS12HE", "notahash"], "sseqid": ["HS1HE", "HS999HE", "HS2HE"]} + ) + out = hasher.decode_df(HASH_DICT, df) + assert list(out["qseqid"]) == [HASH_DICT["HS0HE"], HASH_DICT["HS12HE"], "notahash"] + assert list(out["sseqid"]) == [HASH_DICT["HS1HE"], "HS999HE", HASH_DICT["HS2HE"]] + + +def test_decode_df_does_not_mutate_input(): + df = pd.DataFrame({"qseqid": ["HS0HE"]}) + hasher.decode_df(HASH_DICT, df) + assert list(df["qseqid"]) == ["HS0HE"] diff --git a/tests/test_integration_terrei.py b/tests/test_integration_terrei.py new file mode 100644 index 0000000..0b1dc31 --- /dev/null +++ b/tests/test_integration_terrei.py @@ -0,0 +1,49 @@ +"""End-to-end integration test on the bundled `terrei` dataset. + +Marked `integration` (deselected by default; run with `pytest -m integration`). +Requires the external tools (mmseqs/mafft/fasttree) on PATH and an e-mail for the +bundled GenBank accessions via FUNVIP_TEST_EMAIL. + +Asserts the pipeline's STABLE invariants rather than an exact byte match, because +mmseqs/mafft/fasttree are nondeterministic across runs (a few SPECIES_ASSIGNED +cells vary); species assignment is checked for presence, not exact value. +""" +import csv +import os +import shutil +import subprocess + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.mark.skipif( + shutil.which("mmseqs") is None or shutil.which("mafft") is None, + reason="external tools (mmseqs/mafft) not on PATH", +) +def test_terrei_end_to_end(tmp_path): + email = os.environ.get("FUNVIP_TEST_EMAIL") + if not email: + pytest.skip("set FUNVIP_TEST_EMAIL to run the terrei integration test") + + outdir = tmp_path / "out" + env = {**os.environ, "QT_QPA_PLATFORM": "offscreen"} + proc = subprocess.run( + [ + "FunVIP", "--test", "terrei", "--email", email, + "--thread", "4", "--memory", "8G", + "--outdir", str(outdir), "--runname", "terrei", + ], + env=env, capture_output=True, text=True, timeout=1200, + ) + assert proc.returncode == 0, proc.stderr[-3000:] + + result = outdir / "terrei" / "terrei.result.csv" + assert result.exists(), "no result.csv produced" + + rows = list(csv.DictReader(open(result))) + queries = [r for r in rows if r["DATATYPE"].strip().lower() == "query"] + assert len(queries) == 79 + assert all(r["GROUP_ASSIGNED"] == "Aspergillus" for r in queries) + assert all((r["SPECIES_ASSIGNED"] or "").strip() for r in queries) diff --git a/tests/test_logics.py b/tests/test_logics.py new file mode 100644 index 0000000..eee5eff --- /dev/null +++ b/tests/test_logics.py @@ -0,0 +1,30 @@ +"""Tests for funvip.src.logics — colour and NaN helpers.""" +import numpy as np + +from funvip.src.logics import isnan, isvalidcolor + + +def test_isvalidcolor_is_case_insensitive(): + # Regression: capitalized CSS names crashed option validation before the fix. + assert isvalidcolor("Red") + assert isvalidcolor("red") + assert isvalidcolor("DarkGreen") + + +def test_isvalidcolor_hex(): + assert isvalidcolor("#aabbcc") + assert isvalidcolor("#AABBCC") + assert isvalidcolor("#abc") + + +def test_isvalidcolor_rejects_garbage(): + assert not isvalidcolor("notacolor") + assert not isvalidcolor("#xyz123") + + +def test_isnan_no_longer_nameerrors(): + # Regression: isnan referenced np without importing numpy. + assert isnan(float("nan")) is True + assert isnan(np.float64("nan")) is True + assert isnan(1.0) is False + assert isnan("not a number") is False diff --git a/tests/test_reporter.py b/tests/test_reporter.py new file mode 100644 index 0000000..b5ac871 --- /dev/null +++ b/tests/test_reporter.py @@ -0,0 +1,39 @@ +"""Tests for funvip.src.reporter.Report.update_statistics (per-group summary).""" +import pandas as pd + +from funvip.src.reporter import Report + + +def test_update_statistics_maps_status_per_group(): + r = Report() + r.query_result = pd.DataFrame( + { + "DATATYPE": ["query"] * 6, + "GROUP_ASSIGNED": ["A", "A", "A", "B", "B", "A"], + "STATUS": ["match", "assigned", "conflict", "new species", "failed", "match"], + } + ) + r.update_statistics() + s = r.statistics + + a = s["GROUP"].index("A") + assert s["IDENTIFIED"][a] == 3 # match + assigned + match + assert s["MISIDENTIFIED"][a] == 1 # conflict + assert s["TOTAL"][a] == 4 + + b = s["GROUP"].index("B") + assert s["NEW SPECIES CANDIDATE"][b] == 1 + assert s["UNDETERMINED"][b] == 1 + assert s["TOTAL"][b] == 2 + + t = s["GROUP"].index("TOTAL") + assert s["TOTAL"][t] == 6 + assert s["IDENTIFIED"][t] == 3 + assert s["MISIDENTIFIED"][t] == 1 + + +def test_update_statistics_empty_is_safe(): + r = Report() + r.query_result = None + r.update_statistics() # must not raise + assert r.statistics["GROUP"] == [] diff --git a/tests/test_tool.py b/tests/test_tool.py new file mode 100644 index 0000000..b13d92a --- /dev/null +++ b/tests/test_tool.py @@ -0,0 +1,48 @@ +"""Tests for funvip.src.tool.get_genus_species and its genus-list handling.""" +import os + +import pytest + +from funvip.src import tool +from funvip.src.tool import get_genus_species + +GENUS_FILE = os.path.join(os.path.dirname(tool.__file__), "..", "data", "genus_line.txt") + + +@pytest.fixture() +def genus_list(): + return tuple(open(GENUS_FILE).read().splitlines()) + + +def test_uninitialized_raises_valueerror(): + # Regression: an uninitialized genus_file gave a cryptic NameError, not this. + tool._genus_list_cache.clear() + tool.__dict__.pop("genus_file", None) + with pytest.raises(ValueError): + get_genus_species("Aspergillus terreus strain") + + +@pytest.mark.parametrize( + "text,expected", + [ + ("Aspergillus terreus strain ABC", ("Aspergillus", "terreus")), + ("Penicillium citrinum internal transcribed spacer", ("Penicillium", "citrinum")), + ("Talaromyces marneffei sp. 3", ("Talaromyces", "marneffei")), + ("Fusarium sp.", ("Fusarium", "sp.")), + ("sequence with no known genus here", ("", "")), + ], +) +def test_get_genus_species_with_explicit_list(genus_list, text, expected): + # genus_list is a tuple (get_genus_species is @lru_cache'd -> args must hash). + assert get_genus_species(text, genus_list=genus_list) == expected + + +def test_cached_global_matches_explicit_list(genus_list): + class _Path: + genusdb = GENUS_FILE + + tool._genus_list_cache.clear() + tool.initialize_path(_Path()) + text = "Cladosporium cladosporioides voucher X" + assert get_genus_species(text) == get_genus_species(text, genus_list=genus_list) + assert GENUS_FILE in tool._genus_list_cache # read once, cached diff --git a/tests/test_validate_option.py b/tests/test_validate_option.py new file mode 100644 index 0000000..d49e21c --- /dev/null +++ b/tests/test_validate_option.py @@ -0,0 +1,54 @@ +"""Tests for funvip.src.validate_option preset resolution. + +Guards the preset-handler fixes: the substring->tuple membership conversion, the +CLUSTER-CUTOFF -> cluster.cutoff fix, the ~-on-bool fixes, and the e-value +default/preset consolidation. +""" +import os + +import pytest + +import funvip +from funvip.src.validate_option import Option + +PRESET_DIR = os.path.join(os.path.dirname(funvip.__file__), "preset") + + +def _load(preset_name): + opt = Option() + opt.update_from_preset(os.path.join(PRESET_DIR, f"{preset_name}.yaml")) + return opt + + +def test_option_defaults(): + opt = Option() + assert opt.cluster.cutoff == 0.95 + assert opt.cluster.evalue == 0.0001 # default e-value is 1e-4 + assert opt.solveflat is True + assert opt.method.tcs is True + + +@pytest.mark.parametrize("preset", ["fast", "accurate"]) +def test_preset_resolves_cleanly(preset): + opt = _load(preset) + # CLUSTER-CUTOFF now lands on cluster.cutoff (was silently discarded to evalue) + assert opt.cluster.cutoff == 0.97 + # single clean e-value key resolves to 1e-4 + assert opt.cluster.evalue == 0.0001 + # ~-on-bool fixes: these resolve to real booleans, not -2 + assert opt.solveflat is True + assert opt.cachedb is True + # substring-collision fixes: these keep their proper defaults, not a collided value + assert opt.allow_innertrimming is False + assert opt.nosearchresult is False + assert opt.method.search == "blast" + assert opt.method.trim == "trimal" + + +def test_fast_and_accurate_differ_on_tree_and_confident(): + fast = _load("fast") + accurate = _load("accurate") + assert fast.method.tree == "fasttree" + assert accurate.method.tree == "raxml" + assert fast.confident is True + assert accurate.confident is False # was the 'fale' typo diff --git a/tools/ete4-windows/README.md b/tools/ete4-windows/README.md new file mode 100644 index 0000000..c9da271 --- /dev/null +++ b/tools/ete4-windows/README.md @@ -0,0 +1,82 @@ +# Windows wheels for ete4 + +FunVIP depends on **ete4**, which upstream does **not** ship for Windows: + +- PyPI has only an **sdist** (source) for ete4, so `pip install` tries to compile, + and that compile fails on Windows. +- conda-forge builds ete4 only for **linux-64 / osx-64 / osx-arm64** (no win-64); + bioconda has no ete4 at all. + +Because ete4 is a hard FunVIP dependency, this means FunVIP currently **cannot be +installed with pip or conda on Windows**; only WSL/Docker work. This directory +produces a prebuilt Windows wheel so ordinary Windows users can install FunVIP +without a compiler or a terminal-heavy workflow. + +## The fix + +The only change needed is the small path-separator fix from +[etetoolkit/ete PR #783](https://github.com/etetoolkit/ete/pull/783), captured here +as [`ete4-windows.patch`](ete4-windows.patch) (three lines): + +- `setup.py`: derive Cython module names with `os.path.sep` instead of a hardcoded + `'/'` (Windows uses `\`), so the extensions are named/placed correctly. +- `ete4/config.py`: use `os.path.expanduser('~')` instead of `os.environ['HOME']` + (Windows has no `HOME`). + +The patch is a no-op on Linux/macOS (verified: the patched source still builds a +normal Linux wheel), so it is safe to apply unconditionally. + +## Build the wheels (GitHub Actions) + +1. Push this repo to GitHub (the workflow lives in + `.github/workflows/build-ete4-windows-wheels.yml`). +2. Actions tab → **build-ete4-windows-wheels** → **Run workflow** (optionally set the + ete4 version; default `4.4.0`). +3. When it finishes, download the **`ete4--windows-wheels`** artifact. It + contains `ete4--cp3XX-cp3XX-win_amd64.whl` for CPython 3.10–3.13. + +The workflow downloads the pinned ete4 sdist, applies the patch, builds with +`cibuildwheel` on `windows-latest`, and runs an import test that loads the compiled +extension (so a green run means the wheel actually *works* on Windows, not just that +it compiled). To make a first "does it even compile?" run faster, narrow +`CIBW_BUILD` in the workflow to a single version, e.g. `cp312-win_amd64`. + +## Using the wheels: bundle them into FunVIP + +FunVIP bundles these wheels and installs the matching one on the **first Windows +run** (see `_ensure_ete4` in `funvip/main.py`), so Windows users run the exact same +`pip install FunVIP` as Linux/macOS. To wire that up: + +1. Download the workflow's `ete4--windows-wheels` artifact. +2. Drop the `.whl` files into **`funvip/_vendor/ete4_wheels/`** and commit them + (they are declared as package data, so they ship inside the FunVIP wheel/sdist). +3. Release FunVIP as usual. On a user's first Windows run, FunVIP `pip install`s the + wheel for their Python version from that folder (offline, `--no-deps`; ete4's + runtime deps are already pulled in as Windows-only FunVIP dependencies). + +The other external tools are already bundled for Windows under `funvip/external/`, +so ete4 is the last missing piece. Keep the bundled wheel version within FunVIP's +ete4 pin (`>=4.4.0,<4.5.0`). + +If no bundled wheel matches the user's Python, `_ensure_ete4` falls back to +**building ete4 from source** (applying the same patch, via pip) on first run. That +needs a C/C++ compiler on the user's machine, so it is only a developer/edge-case +fallback: bundling the wheels is what lets ordinary Windows users install with no +compiler. + +## Even cleaner: upstream the patch + +The best outcome is to get PR #783 merged and ask the ete4 maintainers to publish +official `win_amd64` wheels. Then you redistribute nothing and Windows users install +ete4 straight from PyPI. Hosting your own wheels (below) is a fine stopgap until then. + +--- + +ETE4 LICENSE NOTICE + ete4 is distributed under the GNU General Public License v3 or later + (GPL-3.0-or-later) -- a standard GPL with no extra restrictions. You may build and + redistribute these wheels, provided you (1) offer the corresponding source for the + exact version built, (2) ship this patch and note that the source is modified, and + (3) keep ete4's LICENSE and copyright notices (the wheel already carries them). No + additional restrictions may be added. See: https://github.com/etetoolkit/ete +END NOTICE diff --git a/tools/ete4-windows/ete4-windows.patch b/tools/ete4-windows/ete4-windows.patch new file mode 100644 index 0000000..beb550c --- /dev/null +++ b/tools/ete4-windows/ete4-windows.patch @@ -0,0 +1,33 @@ +diff --git a/ete4/config.py b/ete4/config.py +index 4de6cb9..5a23000 100644 +--- a/ete4/config.py ++++ b/ete4/config.py +@@ -11,7 +11,7 @@ import requests + + # Helper function to define global ETE_* variables. + def ete_path(xdg_var, default): +- return os.environ.get(xdg_var, os.environ['HOME'] + default) + '/ete' ++ return os.environ.get(xdg_var, os.path.expanduser('~') + default) + '/ete' + + ETE_DATA_HOME = ete_path('XDG_DATA_HOME', '/.local/share') + ETE_CONFIG_HOME = ete_path('XDG_CONFIG_HOME', '/.config') +diff --git a/setup.py b/setup.py +index 20a4a25..fb5d715 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,13 +1,13 @@ + from setuptools import setup, Extension + + from glob import glob +-from os.path import isfile ++from os.path import isfile, sep + + from Cython.Build import cythonize + + + def make_extension(path): # to create cython extensions the way we want +- name = path.replace('/', '.')[:-len('.pyx')] # / -> . and remove .pyx ++ name = path.replace(sep, '.')[:-len('.pyx')] # / -> . and remove .pyx + return Extension(name, [path], extra_compile_args=['-O3']) + + setup( diff --git a/tools/tcoffee/README.md b/tools/tcoffee/README.md new file mode 100644 index 0000000..23522ed --- /dev/null +++ b/tools/tcoffee/README.md @@ -0,0 +1,71 @@ +# Building T-Coffee for FunVIP's optional TCS step + +FunVIP can validate its alignments with **TCS** (Transitive Consistency Score, +from T-Coffee). TCS is **optional** and off unless a `t_coffee` binary is on your +PATH. FunVIP itself works fully without it; only enable TCS if you specifically +want that alignment-quality metric (for example to reproduce a published TCS +analysis). + +FunVIP does **not** bundle a `t_coffee` binary. The prebuilt (bioconda) t-coffee +is broken on modern Linux, and T-Coffee's license discourages redistribution +(see the notice below). This directory instead ships a **build recipe** so you can +compile a working `t_coffee` yourself. + +## Why a prebuilt t-coffee breaks (and eats all your RAM) + +The stock t-coffee indexes an internal table by the **raw OS process id**, but the +table is sized by a **compile-time constant `MAX_N_PID = 260000`**. Modern Linux +kernels default `kernel.pid_max = 4194304`, so t-coffee's own PID overflows the +table on essentially every run: + +``` +out-of-bounds table access -> SIGSEGV -> t-coffee's signal handler re-runs the +faulting instruction forever -> heap grows without bound -> all RAM consumed +``` + +It is a **crash-loop, not a memory leak**, and it triggers on *any* invocation, +even `t_coffee -version`. The `MAX_N_PID_4_TCOFFEE` environment variable does not +fully fix it: even when the runtime check honours it, the table is still allocated +with the compile-time `MAX_N_PID` and overflows anyway. The only correct fix is to +**recompile with `MAX_N_PID >= kernel.pid_max`**, which is exactly what +[`max_n_pid.patch`](max_n_pid.patch) does (a one-line change to `coffee_defines.h`). + +## Usage + +```bash +bash tools/tcoffee/build_tcoffee_for_tcs.sh # installs into $CONDA_PREFIX/bin +# or +bash tools/tcoffee/build_tcoffee_for_tcs.sh --prefix ~/tcoffee # -> ~/tcoffee/bin +``` + +Requirements: a C++ compiler (`g++`), `make`, and `curl` or `wget`. In a conda +env: `conda install -c conda-forge gxx make`. + +The script downloads the pinned T-Coffee source (checksum-verified), applies the +`MAX_N_PID` patch, builds only the `t_coffee` binary, **self-tests it** (runs TCS +on a tiny alignment and fails if it still crashes), and installs it. Make sure the +install `bin` directory is on your PATH; FunVIP then enables TCS automatically +(disable with `--notcs`). + +If your kernel's `pid_max` is unusually large, pass `--max-n-pid N` with a value +above it. + +## Alternatives (no compiling) + +- Run FunVIP **without TCS** (the default when t-coffee is absent); nothing to do. +- On a host where you have root, lowering `kernel.pid_max` below 260000 also avoids + the crash, but it is system-wide and affects everything else, so it is not + recommended on shared machines. + +--- + +T-COFFEE LICENSE NOTICE + T-Coffee is developed by Cedric Notredame (CNRS) and distributed under the GNU + General Public License, with the authors' added condition that it may be + incorporated only into NON-COMMERCIAL pipelines; for commercial use, contact the + T-Coffee authors (https://tcoffee.org). By building and using t_coffee via this + script you obtain and accept T-Coffee directly under its own license -- FunVIP + neither relicenses nor redistributes the T-Coffee binary. See: + https://tcoffee.readthedocs.io/en/latest/tcoffee_license.html + Please also cite the T-Coffee / TCS papers when you use this functionality. +END NOTICE diff --git a/tools/tcoffee/build_tcoffee_for_tcs.sh b/tools/tcoffee/build_tcoffee_for_tcs.sh new file mode 100755 index 0000000..9fad2c7 --- /dev/null +++ b/tools/tcoffee/build_tcoffee_for_tcs.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# build_tcoffee_for_tcs.sh +# ----------------------------------------------------------------------------- +# Build a T-Coffee `t_coffee` binary that works with FunVIP's optional TCS +# (Transitive Consistency Score) alignment-validation step. +# +# Why this script exists +# ---------------------- +# The prebuilt (e.g. bioconda) t-coffee indexes an internal table by the raw OS +# process id, but the table is sized by a COMPILE-TIME constant MAX_N_PID=260000. +# Modern Linux kernels default kernel.pid_max=4194304, so t-coffee's own PID +# overflows that table on essentially every run -> out-of-bounds access -> +# SIGSEGV -> its signal handler re-runs the faulting instruction forever, growing +# the heap until all system memory is gone. It is a crash-loop (not a real leak) +# and it fires on ANY invocation, even `t_coffee -version`. The MAX_N_PID_4_TCOFFEE +# environment variable does NOT fully fix it: even where the runtime check honours +# it, the table is still allocated with the compile-time MAX_N_PID and overflows. +# +# The only correct fix is to recompile t-coffee with MAX_N_PID >= kernel.pid_max. +# This script downloads the T-Coffee source, applies exactly that one-line change +# (tools/tcoffee/max_n_pid.patch), builds only the `t_coffee` binary, verifies it, +# and installs it. +# +# FunVIP does NOT ship a t-coffee binary: t-coffee is GPL but its authors add a +# "non-commercial pipelines only" caveat (see README.md / the printed notice), so +# redistributing the binary is avoided. You build it yourself with this script and +# thereby accept T-Coffee's license directly. +# +# Usage +# ----- +# bash build_tcoffee_for_tcs.sh [--prefix DIR] [--max-n-pid N] [--jobs N] [--keep] +# +# --prefix DIR install t_coffee into DIR/bin (default: $CONDA_PREFIX, else +# ~/.local). DIR/bin must be on your PATH for FunVIP to find it. +# --max-n-pid N override the compiled limit (default: kernel.pid_max + margin, +# floored at 4194305). Increase only if your kernel.pid_max is +# unusually large. +# --jobs N parallel compile jobs (default: all cores). +# --keep keep the build directory (default: removed on success). +# ----------------------------------------------------------------------------- +set -euo pipefail + +# --- pinned upstream source (matches bioconda t-coffee 13.46.2) --------------- +TC_VERSION="13.46.2.7c9e712d" +TC_URL="https://s3.eu-central-1.amazonaws.com/tcoffee-packages/Archives/T-COFFEE_distribution_Version_${TC_VERSION}.tar.gz" +TC_SHA256="84f9b4076767d39dec6619c5eb91c9538a7c58c68a3731a92ebbf2e1f914296f" + +# --- defaults ----------------------------------------------------------------- +PREFIX="${CONDA_PREFIX:-$HOME/.local}" +JOBS="$( (command -v nproc >/dev/null && nproc) || echo 4)" +KEEP=0 +# kernel.pid_max + margin, floored at PID_MAX_LIMIT+1 (4194304+1) so the table can +# hold any 64-bit-Linux PID even if this build host's pid_max is lower than a +# future run host's. +_PIDMAX="$(cat /proc/sys/kernel/pid_max 2>/dev/null || echo 4194304)" +MAX_N_PID=$(( _PIDMAX + 16 )); [ "$MAX_N_PID" -lt 4194305 ] && MAX_N_PID=4194305 + +while [ $# -gt 0 ]; do + case "$1" in + --prefix) PREFIX="$2"; shift 2 ;; + --max-n-pid) MAX_N_PID="$2"; shift 2 ;; + --jobs) JOBS="$2"; shift 2 ;; + --keep) KEEP=1; shift ;; + -h|--help) sed -n '2,45p' "$0"; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac +done + +say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } +die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; } + +# --- prerequisites ------------------------------------------------------------ +say "Checking build tools" +MISSING="" +for t in make g++ sed awk tar; do command -v "$t" >/dev/null || MISSING="$MISSING $t"; done +command -v curl >/dev/null || command -v wget >/dev/null || MISSING="$MISSING curl-or-wget" +[ -n "$MISSING" ] && die "missing required tools:$MISSING (install e.g. 'conda install -c conda-forge gxx make', or a build-essential package)" +echo " ok (g++ $(g++ -dumpversion), make, $( command -v curl >/dev/null && echo curl || echo wget ))" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/tcoffee_build.XXXXXX")" +cleanup() { [ "$KEEP" -eq 1 ] || rm -rf "$WORK"; } +trap cleanup EXIT + +# --- download + verify -------------------------------------------------------- +say "Downloading T-Coffee ${TC_VERSION} source" +TARBALL="$WORK/tcoffee_src.tar.gz" +if command -v curl >/dev/null; then curl -fSL -o "$TARBALL" "$TC_URL" +else wget -O "$TARBALL" "$TC_URL"; fi + +say "Verifying checksum" +if command -v sha256sum >/dev/null; then + echo "${TC_SHA256} ${TARBALL}" | sha256sum -c - || die "checksum mismatch (download corrupt or upstream changed)" +elif command -v shasum >/dev/null; then + echo "${TC_SHA256} ${TARBALL}" | shasum -a 256 -c - || die "checksum mismatch" +else + echo " (no sha256 tool found; skipping verification)" +fi + +tar -xzf "$TARBALL" -C "$WORK" +SRCROOT="$(find "$WORK" -maxdepth 1 -type d -name 'T-COFFEE_distribution_*' | head -1)" +[ -d "$SRCROOT/t_coffee_source" ] || die "unexpected source layout under $SRCROOT" + +# --- patch MAX_N_PID ---------------------------------------------------------- +say "Patching MAX_N_PID 260000 -> ${MAX_N_PID} (kernel.pid_max=${_PIDMAX})" +DEFS="$SRCROOT/t_coffee_source/coffee_defines.h" +grep -qE '^#define MAX_N_PID[[:space:]]+260000' "$DEFS" \ + || die "MAX_N_PID definition not found/changed upstream in $DEFS; update this script" +sed -i.orig -E "s/^#define MAX_N_PID[[:space:]]+260000/#define MAX_N_PID ${MAX_N_PID}/" "$DEFS" +grep -E '^#define MAX_N_PID' "$DEFS" | sed 's/^/ /' + +# --- build only the t_coffee binary (modern-g++ compatible flags) ------------- +say "Building t_coffee (-j${JOBS}); this takes ~1 minute" +( cd "$SRCROOT/t_coffee_source" + make t_coffee -j"$JOBS" CC=g++ \ + CFLAGS="-O3 -fpermissive -fsigned-char -Wno-write-strings -Wno-register -Wno-return-type -Dregister=" ) +BIN="$SRCROOT/t_coffee_source/t_coffee" +[ -x "$BIN" ] || die "build did not produce a t_coffee binary" + +# --- self-test: must NOT crash-loop and MUST produce a score ------------------ +say "Self-test: running TCS on a tiny alignment" +T="$WORK/selftest"; mkdir -p "$T" +cat > "$T/aln.fasta" <<'FASTA' +>s1 +ACGTACGTACGTACGTACGTACGTACGTACGTACGTACGT +>s2 +ACGTACGTACGTACGTTCGTACGTACGTACGTACGTACGT +>s3 +ACGTACGTACGTACGTACGTACGTACGAACGTACGTACGT +FASTA +( cd "$T" + # 8 GB address-space guard so a still-broken build cannot take down the host + ( ulimit -v 8388608 2>/dev/null || true + timeout 120 "$BIN" -infile aln.fasta -method fast_pair -type DNA -evaluate \ + -output score_ascii -outfile out.tcs -quiet tc.log 2>&1 ) || true + if grep -q "MAX_N_PID exceded" tc.log; then + die "self-test still hit MAX_N_PID -- increase --max-n-pid above your kernel.pid_max" + fi + [ -s out.tcs ] || { echo "----- t_coffee output -----"; tail -20 tc.log; die "self-test produced no .tcs score file"; } + echo " ok: produced a valid TCS score file" +) + +# --- install ------------------------------------------------------------------ +say "Installing to ${PREFIX}/bin/t_coffee" +mkdir -p "$PREFIX/bin" +install -m 0755 "$BIN" "$PREFIX/bin/t_coffee" + +cat </dev/null || true) +EOF diff --git a/tools/tcoffee/max_n_pid.patch b/tools/tcoffee/max_n_pid.patch new file mode 100644 index 0000000..ed3ab0a --- /dev/null +++ b/tools/tcoffee/max_n_pid.patch @@ -0,0 +1,11 @@ +--- a/t_coffee_source/coffee_defines.h ++++ b/t_coffee_source/coffee_defines.h +@@ -40,7 +40,7 @@ + #define TMP_MAX_KEEP 10 + #define CACHE_MAX_SIZE 2000 + #define CACHE_MAX_KEEP 180 +-#define MAX_N_PID 260000 ++#define MAX_N_PID 4194305 + //Importnat Values Affecting the Program Behavior + #define O2A_BYTE 50 + #define SCORE_K 10 diff --git a/tutorial/advanced.md b/tutorial/advanced.md index 2ccc4a5..9315b02 100644 --- a/tutorial/advanced.md +++ b/tutorial/advanced.md @@ -21,7 +21,7 @@ Utilize validation ability of FunVIP as much as possible. Try to put all putative database and queries that you are going to use for the analysis at the initial run. -Run initial run with fast mode (because there will be a lot of sequneces). +Run the initial run with the fast preset (because there will be a lot of sequences). Filter out invalid databases according to the result of the first run. diff --git a/tutorial/development.md b/tutorial/development.md index a3035db..cdcf3f4 100644 --- a/tutorial/development.md +++ b/tutorial/development.md @@ -24,7 +24,6 @@ class tree_information def calculate_zero def reroot_outgroup def taxon_count - def genus_count def designate_genus def find_major_taxon def collapse @@ -135,4 +134,3 @@ def is_monophyletic(self, clade, gene, taxon): -genus_count(funinfo_dict, gene, clade) \ No newline at end of file diff --git a/tutorial/installation.md b/tutorial/installation.md index 9f2e768..46b1d8c 100644 --- a/tutorial/installation.md +++ b/tutorial/installation.md @@ -1,49 +1,116 @@ ## Installation -### Windows -1. ```conda create -n FunVIP python=3.12``` -2. ```conda activate FunVIP``` -3. ```pip install FunVIP``` -4. run ```FunVIP --test Terrei --email [your email] ``` to check installation + +FunVIP needs two things: the Python package (installed with `pip`, which pulls in +ete4 and the other Python dependencies) and a set of external command-line tools +(installed with `conda` from the bioconda / conda-forge channels). + +The bundled test run `FunVIP --test terrei --email ` at the end of +each recipe checks the installation. `--test` is case-insensitive, so `terrei` +and `Terrei` both work. +

    -## Linux -1. ```conda create -n FunVIP python=3.11``` + +### Recommended: one conda environment file +From the repository root (or after downloading `environment.yml`): +``` +conda env create -f environment.yml # or: mamba env create -f environment.yml +conda activate funvip +FunVIP --test terrei --email +``` +This installs the external tools and FunVIP (from PyPI) together. To use the +development version, install from source (below) instead. + +

    + +### Linux +1. ```conda create -n FunVIP python=3.12``` 2. ```conda activate FunVIP``` -3. ```pip install FunVIP``` -4. ```conda config --add channels conda-forge``` -5. ```conda install -c bioconda raxml iqtree "modeltest-ng==0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree "t-coffee>=13"``` -6. run ```FunVIP --test Terrei --email [your email] ``` to check installation -* For intel mac system, this method may work, but we couldn't test it because we don't have any intel mac device. We're looking for feedbacks in intel mac -* If you have memory leakage problems, install without t-coffee +3. ```conda config --add channels conda-forge``` +4. ```conda install -c bioconda raxml iqtree "modeltest-ng==0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree``` +5. ```pip install FunVIP``` +6. run ```FunVIP --test terrei --email ``` to check installation + +* TCS (optional alignment validation) is left out on purpose: the bioconda + t-coffee can cause memory problems, and FunVIP skips TCS automatically when it is + absent. To use TCS, build t-coffee with ```tools/tcoffee/build_tcoffee_for_tcs.sh``` + (see ```tools/tcoffee/README.md```). +* For an Intel Mac this recipe may also work, but it is untested. Feedback welcome. +

    ### Apple Silicon Mac -#### Native installation (Faster, Gblocks unavailable) +#### Native installation (faster, Gblocks unavailable) 1. ```conda create -n FunVIP python=3.12``` -2. ```conda activate FunVIP ``` -3. ``` pip install FunVIP ``` -4. ``` conda install conda-forge::pyqt bioconda::fasttree bioconda::raxml bioconda::iqtree bioconda::mmseqs2 bioconda::blast conda-forge::mafft bioconda::trimal ``` +2. ```conda activate FunVIP``` +3. ```conda install conda-forge::pyqt bioconda::fasttree bioconda::raxml bioconda::iqtree bioconda::mmseqs2 bioconda::blast conda-forge::mafft bioconda::trimal``` +4. ```pip install FunVIP``` +5. run ```FunVIP --test terrei --email ``` to check installation -#### Rossetta installation (Slower, Gblocks available) +#### Rosetta installation (slower, Gblocks available) 1. ```softwareupdate --install-rosetta``` 2. ```CONDA_SUBDIR=osx-64 conda create -n FunVIP python=3.12``` 3. ```conda activate FunVIP``` 4. ```conda config --env --set subdir osx-64``` 5. ```conda install pyqt openssl ca-certificates``` -6. ```pip install FunVIP``` -7. ```conda install -c bioconda raxml iqtree "mmseqs2<=16" "blast>=2.12" mafft trimal gblocks``` -8. ```CONDA_SUBDIR=osx-arm64 conda install -c bioconda fasttree``` -9. run ```FunVIP --test Terrei --email [your email] ``` to check installation -- I personally recommend native installation because Gblocks can be substituted with TrimAl +6. ```conda install -c bioconda raxml iqtree "mmseqs2<=16" "blast>=2.12" mafft trimal gblocks``` +7. ```CONDA_SUBDIR=osx-arm64 conda install -c bioconda fasttree``` +8. ```pip install FunVIP``` +9. run ```FunVIP --test terrei --email ``` to check installation + +- Native installation is recommended; Gblocks can be substituted with trimAl. + +

    + +### Windows +1. ```conda create -n FunVIP python=3.12``` +2. ```conda activate FunVIP``` +3. ```pip install FunVIP``` +4. run ```FunVIP --test terrei --email ``` to check installation + +* Docker and WSL2 also work (build the image with `docker build -t funvip .`, or + open an Ubuntu WSL shell and follow the Linux recipe). +

    -### Installation from source (For developers and core users) -* this is for developmental steps + +### Installation from source (for developers) 1. ```git clone https://github.com/Changwanseo/FunVIP.git``` -2. Move to ```~/FunVIP``` +2. ```cd FunVIP``` 3. ```conda create -n FunVIP python=3.12``` 4. ```conda activate FunVIP``` -5. ```pip install ./``` -6. run ```FunVIP --test Terrei --email [your email]``` to check installation +5. ```conda config --add channels conda-forge``` +6. ```conda install -c bioconda raxml iqtree "modeltest-ng==0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree``` (t-coffee omitted on purpose; see the TCS note in the Linux section) +7. ```pip install -e ".[test]"``` (editable install + test dependencies; use ```pip install ./``` for a plain install) +8. run ```pytest``` for the unit tests, and ```FunVIP --test terrei --email ``` for an end-to-end check +

    -### Upgrade FunVIP -``` pip install FunVIP --upgrade ``` + +### Upgrade FunVIP (within the 1.x series) +Both sides use ete4, so a plain pip upgrade is fine: +```pip install FunVIP --upgrade``` + +### Migrating from 0.5.x to 1.0 + +**Do not `pip install --upgrade` across this jump.** FunVIP 0.5.x is built on +**ete3**; 1.0 switched to **ete4**. These are different packages, not two versions +of one, so a pip upgrade leaves the old `ete3` (and other stale 0.5.x dependencies) +behind alongside the new ones, which can conflict. Rebuild the conda environment +from scratch instead: + +``` +# 1. leave and delete the old environment (use your existing env's name) +conda deactivate +conda env remove -n FunVIP + +# 2. recreate it fresh, pick the recipe for your platform above, e.g. Linux: +conda create -n FunVIP python=3.12 +conda activate FunVIP +conda config --add channels conda-forge +conda install -c bioconda raxml iqtree "modeltest-ng==0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree +pip install FunVIP +FunVIP --test terrei --email +``` + +Only the Python environment is rebuilt: your input data, databases, and result +folders are untouched. If you installed with the one-file recipe, the equivalent is +`conda env remove -n funvip` followed by `conda env create -f environment.yml`. diff --git a/tutorial/tutorial.md b/tutorial/tutorial.md index 81a7e0f..749ed84 100644 --- a/tutorial/tutorial.md +++ b/tutorial/tutorial.md @@ -15,11 +15,11 @@ In this tutorial, we will run FunVIP with already prepared input files.

    ### 1. Install FunVIP by following instructions, depending on your os system -* [Windows](https://github.com/Changwanseo/FunVIP/#Windows) +* [Windows](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#windows) -* [Linux](https://github.com/Changwanseo/FunVIP/#Linux) +* [Linux](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#linux) -* [Mac - Apple Silicon](https://github.com/Changwanseo/FunVIP/#Apple-Silicon-Mac) +* [Mac - Apple Silicon](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md#apple-silicon-mac)

    diff --git a/tutorial/tutorial2.md b/tutorial/tutorial2.md index 43e23b2..4a6d39c 100644 --- a/tutorial/tutorial2.md +++ b/tutorial/tutorial2.md @@ -5,7 +5,7 @@ * [Part 3 - How to use FunVIP for final publication]() This is basic tutorial for how to run FunVIP -In this tutorial, we will prepare own Sanghuangporus database and identify unknown _Sanghuangporus_ sequneces from GenBank +In this tutorial, we will prepare our own Sanghuangporus database and identify unknown _Sanghuangporus_ sequences from GenBank ### 1. Generate _Sanghuangporus_ ITS database from [Zhou et al. 2021](https://link.springer.com/article/10.1186/s43008-021-00059-x) Open the following link about [work of Zhou et al. 2021](https://link.springer.com/article/10.1186/s43008-021-00059-x) @@ -20,7 +20,7 @@ Copy the table from [Table 1](https://link.springer.com/article/10.1186/s43008-0 Before editing, save this file to Sanghuangporus_db.xlsx -Now, we sould edit the file in the format of FunVIP database, following this format +Now, we should edit the file into the FunVIP database format, following this format ![image](https://github.com/user-attachments/assets/4d4b0d5a-af27-4fda-afa6-6e140d052eb4) @@ -33,7 +33,7 @@ First, we will make "Genus" and "Species" column. But before that, we have to un ![image](https://github.com/user-attachments/assets/ca684cfd-31dd-4c5e-8816-7e1a87e116ef) ![image](https://github.com/user-attachments/assets/a070fdf7-cbb2-49a6-9705-a0ed7000fd27) -You can use split by text to easily seperate genus name and specie name +You can use split by text to easily separate the genus name and species name ![image](https://github.com/user-attachments/assets/8572cbc1-5dca-4fff-a642-2b78254356b1) ![image](https://github.com/user-attachments/assets/1a3d44a9-3697-4722-a38e-348568f0016f) ![image](https://github.com/user-attachments/assets/f42c0532-c25e-4b84-89e9-73b3b7641f65)