Facelift for reproman to make CI pass and support recent pythons

.github/workflows/codespell.yml

@@ -14,6 +14,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v1

.github/workflows/docs.yml

@@ -16,9 +16,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - name: Install system dependencies

.github/workflows/lint.yml

@@ -16,9 +16,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - name: Install Black

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python: "3.9"
+          - python: "3.10"
             install_datalad: datalad
             # install_condor: condor
             # TODO 20.04 may be needed for condor
@@ -49,9 +49,9 @@ jobs:
             os: ubuntu-22.04
             reproman_tests_deps: full-except-datalad
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
       - name: Set up system
@@ -120,9 +120,7 @@ jobs:
           pip install -e .[tests]
           pip install -e .[devel-docs]
           pip install -e .[devel-utils]
-      - name: Verify setup.py build
-        run: python setup.py build
       - name: Run tests
         run: python -m pytest -s -v -rsfE --integration --cov=reproman --cov-report=xml reproman
       - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4

docs/source/conf.py

@@ -58,6 +58,12 @@
 # for the module reference
 autosummary_generate = True
 
+# Man page RST files under source/generated/man/ are not committed to git
+# (they are generated by `python setup.py build_manpage`), so references
+# to man page labels (e.g. man_reproman-login) will be undefined during
+# a normal docs build. Suppress those warnings.
+suppress_warnings = ["ref.ref", "toc.not_readable"]
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

pyproject.toml

@@ -1,3 +1,6 @@
 [tool.black]
 line-length = 100
-target-version = ['py39']
+target-version = ['py310']
+
+[tool.pytest.ini_options]
+testpaths = ["reproman"]

reproman/cmd.py

@@ -10,7 +10,6 @@
 
 """
 
-
 import subprocess
 import sys
 import logging

reproman/cmdline/helpers.py

@@ -171,8 +171,7 @@ def run_via_pbs(args, pbs):
         # TODO: we might need better way to join them, escaping spaces etc.  There must be a stock helper
         # exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else ''
         exe_args = " ".join(args[1:]) if len(args) > 1 else ""
-        f.write(
-            """\
+        f.write("""\
 Executable = %(exe)s
 Initialdir = %(pwd)s
 Output = %(logs)s
@@ -181,9 +180,7 @@ def run_via_pbs(args, pbs):
 
 arguments = %(exe_args)s
 queue
-"""
-            % locals()
-        )
+""" % locals())
         f.close()
         Runner().run(["condor_submit", f.name])
         lgr.info("Scheduled execution via %s.  Logs will be stored under %s" % (pbs, logs))

reproman/cmdline/main.py

@@ -73,11 +73,9 @@ def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter, return_su
     # main parser
     parser = argparse.ArgumentParser(
         fromfile_prefix_chars="@",
-        description=dedent_docstring(
-            """\
+        description=dedent_docstring("""\
             ReproMan aims to ease construction and execution of computation environments
-            based on collected provenance data."""
-        ),
+            based on collected provenance data."""),
         epilog='"Reproducibly Manage Your Environments"',
         formatter_class=formatter_class,
         add_help=False,
@@ -217,12 +215,10 @@ def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter, return_su
         parser.description,
         "\n".join(cmd_summary),
         textwrap.fill(
-            dedent_docstring(
-                """\
+            dedent_docstring("""\
     Detailed usage information for individual commands is
     available via command-specific --help, i.e.:
-    reproman <command> --help"""
-            ),
+    reproman <command> --help"""),
             75,
             initial_indent="",
             subsequent_indent="",

reproman/distributions/conda.py

@@ -6,6 +6,7 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Orchestrator sub-class to provide management of the localhost environment."""
+
 import json
 import os
 from collections import defaultdict
@@ -439,8 +440,8 @@ def identify_distributions(self, paths):
                 continue
             # Retrieve the environment details
             env_export = self._get_conda_env_export(root_path, conda_path)
-            (conda_package_details, file_to_pkg) = self._get_conda_package_details(conda_path)
-            (conda_pip_package_details, file_to_pip_pkg) = self._get_conda_pip_package_details(
+            conda_package_details, file_to_pkg = self._get_conda_package_details(conda_path)
+            conda_pip_package_details, file_to_pip_pkg = self._get_conda_pip_package_details(
                 env_export, conda_path
             )
             # Join our conda and pip packages

reproman/distributions/debian.py

@@ -6,6 +6,7 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Support for Debian(-based) distribution(s)."""
+
 import os
 import re
 
@@ -526,7 +527,7 @@ def _get_pkgs_install_date(self, pkg_dicts):
                 out = exc.stdout  # One file not found, so continue
             # Parse the output and store by filename
             for outlines in out.splitlines():
-                (fname, ftime) = outlines.split(": ")
+                fname, ftime = outlines.split(": ")
                 results[fname] = str(pytz.utc.localize(datetime.utcfromtimestamp(float(ftime))))
 
         # Now lookup the packages in the results

reproman/distributions/piputils.py

@@ -6,6 +6,7 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Utilities for working with pip."""
+
 import itertools
 import json
 import os

reproman/distributions/redhat.py

@@ -6,6 +6,7 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Support for Redhat(-based) distribution(s)."""
+
 import attr
 
 

reproman/distributions/singularity.py

@@ -14,7 +14,6 @@
 import tempfile
 import uuid
 
-
 lgr = logging.getLogger("reproman.distributions.singularity")
 
 from .base import Package

reproman/distributions/tests/test_conda.py

@@ -259,7 +259,7 @@ def test_conda_init_install_and_detect(tmpdir):
 
     assert len(dists) == 1, "Exactly one Conda distribution expected."
 
-    (distributions, unknown_files) = dists[0]
+    distributions, unknown_files = dists[0]
 
     # RepromanProvenance.write(sys.stdout, distributions)
 

reproman/distributions/tests/test_debian.py

@@ -35,7 +35,7 @@
 def test_dpkg_manager_identify_packages():
     files = [COMMON_SYSTEM_PATH]
     tracer = DebTracer()
-    (packages, unknown_files) = tracer.identify_packages_from_files(files)
+    packages, unknown_files = tracer.identify_packages_from_files(files)
     # Make sure that our common path was identified
     assert not unknown_files, "%s should be identified" % COMMON_SYSTEM_PATH
     assert len(packages) == 1
@@ -148,7 +148,7 @@ def test_utf8_file():
     ]
     manager = DebTracer()
     # Simple sanity check that the pipeline works with utf-8
-    (packages, unknown_files) = manager.identify_packages_from_files(files)
+    packages, unknown_files = manager.identify_packages_from_files(files)
     packages = manager.get_details_for_packages(packages)
     # If the file exists, it should be in ca-certificates
     if os.path.isfile(files[0]):
@@ -232,7 +232,7 @@ def setup_packages():
 
 
 def test_package_satisfies(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(p1, mode="satisfied_by")
     assert p1v10.compare(p1v10, mode="satisfied_by")
     assert not p1v10.compare(p1, mode="satisfied_by")
@@ -251,7 +251,7 @@ def test_package_satisfies(setup_packages):
 
 @pytest.fixture
 def setup_distributions(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     d1 = DebianDistribution(name="debian 1")
     d1.packages = [p1]
     d2 = DebianDistribution(name="debian 2")
@@ -260,8 +260,8 @@ def setup_distributions(setup_packages):
 
 
 def test_distribution_satisfies_package(setup_distributions, setup_packages):
-    (d1, d2) = setup_distributions
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    d1, d2 = setup_distributions
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(d1, mode="satisfied_by")
     assert not p1v10.compare(d1, mode="satisfied_by")
     assert p1.compare(d2, mode="satisfied_by")
@@ -270,13 +270,13 @@ def test_distribution_satisfies_package(setup_distributions, setup_packages):
 
 
 def test_distribution_statisfies(setup_distributions):
-    (d1, d2) = setup_distributions
+    d1, d2 = setup_distributions
     assert not d2.compare(d1, mode="satisfied_by")
     assert d1.compare(d2, mode="satisfied_by")
 
 
 def test_distribution_sub(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     d1 = DebianDistribution(name="debian 1")
     d1.packages = [p1, p2]
     d2 = DebianDistribution(name="debian 2")
@@ -288,7 +288,7 @@ def test_distribution_sub(setup_packages):
 
 
 def test_package_is_identical_to(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(p1, mode="identical_to")
     assert p1v10.compare(p1v10, mode="identical_to")
     assert p1v11.compare(p1v11, mode="identical_to")

reproman/distributions/tests/test_piputils.py

@@ -75,13 +75,10 @@ def test_parse_pip_show():
 """
 
     # Check parsing of show output for a "standard" package.
-    out_files = (
-        out_base
-        + """\
+    out_files = out_base + """\
 Files:
   pkg-0.3.0.dist-info/DESCRIPTION.rst
   pkg/__init__.py"""
-    )
     info_files = piputils.parse_pip_show(out_files)
 
     fields = {
@@ -103,12 +100,9 @@ def test_parse_pip_show():
 
     # Check parsing of show output for an editable packages that lacks
     # files.
-    out_no_files = (
-        out_base
-        + """\
+    out_no_files = out_base + """\
 Files:
 Cannot locate installed-files.txt"""
-    )
     info_nofiles = piputils.parse_pip_show(out_no_files)
     assert set(info_nofiles.keys()) == fields
     assert info_nofiles["Files"] == []

reproman/distributions/tests/test_redhat.py

@@ -36,8 +36,7 @@ def docker_container():
 def centos_spec():
     tmp = tempfile.NamedTemporaryFile(delete=False)
     with open(tmp.name, "w") as f:
-        f.write(
-            """# ReproMan Environment Configuration File
+        f.write("""# ReproMan Environment Configuration File
 # This file was created by ReproMan 0.0.5 on 2018-05-23 22:03:22.820177
 version: 0.0.1
 distributions:
@@ -95,8 +94,7 @@ def centos_spec():
     files:
     - /usr/sbin/fido
   version: CentOS Linux release 7.4.1708 (Core)
-"""
-        )
+""")
     return tmp.name
 
 
@@ -115,7 +113,7 @@ def setup_packages():
 
 @pytest.fixture
 def setup_distributions(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     d1 = RedhatDistribution(name="debian 1")
     d1.packages = [p1]
     d2 = RedhatDistribution(name="debian 2")
@@ -124,7 +122,7 @@ def setup_distributions(setup_packages):
 
 
 def test_package_satisfies(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(p1, mode="satisfied_by")
     assert p1v10.compare(p1v10, mode="satisfied_by")
     assert not p1v10.compare(p1, mode="satisfied_by")
@@ -142,8 +140,8 @@ def test_package_satisfies(setup_packages):
 
 
 def test_distribution_satisfies_package(setup_distributions, setup_packages):
-    (d1, d2) = setup_distributions
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    d1, d2 = setup_distributions
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(d1, mode="satisfied_by")
     assert not p1v10.compare(d1, mode="satisfied_by")
     assert p1.compare(d2, mode="satisfied_by")
@@ -152,13 +150,13 @@ def test_distribution_satisfies_package(setup_distributions, setup_packages):
 
 
 def test_distribution_statisfies(setup_distributions):
-    (d1, d2) = setup_distributions
+    d1, d2 = setup_distributions
     assert not d2.compare(d1, mode="satisfied_by")
     assert d1.compare(d2, mode="satisfied_by")
 
 
 def test_distribution_sub(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     d1 = RedhatDistribution(name="debian 1")
     d1.packages = [p1, p2]
     d2 = RedhatDistribution(name="debian 2")
@@ -223,7 +221,7 @@ def test_distribution(docker_container, centos_spec):
 
 
 def test_package_is_identical_to(setup_packages):
-    (p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2) = setup_packages
+    p1, p1v10, p1v11, p1ai, p1aa, p1v11ai, p2 = setup_packages
     assert p1.compare(p1, mode="identical_to")
     assert p1v10.compare(p1v10, mode="identical_to")
     assert p1v11.compare(p1v11, mode="identical_to")