Merge pull request #302 from mpsonntag/convScript

Add format conversion console script

LGTM

Author: achilleas-k

CHANGELOG.md

@@ -5,6 +5,25 @@ until the next release.
 
 # Latest changes in master
 
+## Uncertainty changes
+
+Uncertainty is now limited to float only. See #294.
+
+## Version converter changes
+
+The VersionConverter dealt with an edge case of XML test files with opening <B0> tags 
+that were missing their closing tag rendering them broken. Catching this one edge case 
+circumvented opening XML files via lxml, leaving the resulting document open to various 
+encoding problems.
+
+Support to resolve the specific tag edge cases is dropped in favour of properly opening 
+XML files via lxml. See #301.
+
+## Additional console script
+
+The `odmlconversion` convenience console script has been added to convert multiple 
+previous odML version files to the latest odML version.
+
 
 # Version 1.4.1
 

doc/conf.py

@@ -11,9 +11,22 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import json
 import os
+import pathlib
 import sys
 
+from distutils.version import LooseVersion as CheckVer
+
+currdir = os.path.dirname(os.path.abspath(__file__))
+parent = pathlib.Path(currdir).parent
+path = os.path.join(parent, "odml", "info.json")
+
+with open(path) as infofile:
+    infodict = json.load(infofile)
+
+version_str = infodict["VERSION"]
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -69,9 +82,9 @@ def __new__(meta, name, bases, clsdict):
 # built documents.
 #
 # The short X.Y version.
-version = '1.4'
+version = "%s.%s" % (CheckVer(version_str).version[0], CheckVer(version_str).version[1])
 # The full version, including alpha/beta/rc tags.
-release = '1.4.1'
+release = version_str
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

odml/info.json

@@ -1,5 +1,5 @@
 {
-  "VERSION": "1.4.1",
+  "VERSION": "1.4.2",
   "FORMAT_VERSION": "1.1",
   "AUTHOR": "Hagen Fritsch, Jan Grewe, Christian Kellner, Achilleas Koutsou, Michael Sonntag, Lyuba Zehl",
   "COPYRIGHT": "(c) 2011-2018, German Neuroinformatics Node",

odml/scripts/odml_conversion.py

@@ -0,0 +1,127 @@
+"""odmlConversion
+
+odmlConversion searches for odML files within a provided SEARCHDIR
+and converts them to the newest odML format version.
+Original files will never be overwritten. New files will be
+written either to a new directory at the current or a specified
+location.
+
+Usage: odmlconversion [-r] [-o OUT] SEARCHDIR
+
+Arguments:
+    SEARCHDIR       Directory to search for odML files.
+
+Options:
+    -o OUT          Output directory. Must exist if specified.
+                    If not specified, output files will be
+                    written to the current directory.
+    -r              Search recursively. Directory structures
+                    will not be retained.
+    -h --help       Show this screen.
+    --version       Show version.
+"""
+
+import os
+import pathlib
+import sys
+import tempfile
+
+from docopt import docopt
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+import odml
+
+from odml.tools.version_converter import VersionConverter as VerConf
+
+try:
+    unicode = unicode
+except NameError:
+    unicode = str
+
+
+def run_conversion(file_list, output_dir, report, source_format="XML"):
+    """
+    Convert a list of odML files to the latest odML version.
+    :param file_list: list of files to be converted.
+    :param output_dir: Directory where odML files converted to
+                       the latest odML version will be saved.
+    :param report: Reporting StringIO.
+    :param source_format: Original file format of the odML source files.
+                          XML, JSON and YAML are supported, default is XML.
+    """
+    # Exceptions are kept as broad as possible to ignore any non-odML or
+    # invalid odML files and ensuring everything that can be will be converted.
+    for curr_file in file_list:
+        file_path = unicode(curr_file.absolute())
+        report.write("[Info] Handling file '%s'\n" % file_path)
+        # When loading the current file succeeds, it is
+        # a recent odML format file and can be ignored.
+        try:
+            odml.load(file_path, source_format)
+            report.write("[Info] Skip recent version file '%s'" % file_path)
+        except Exception as exc:
+            out_name = os.path.splitext(os.path.basename(file_path))[0]
+            outfile = os.path.join(output_dir, "%s_conv.xml" % out_name)
+            try:
+                VerConf(file_path).write_to_file(outfile, source_format)
+            except Exception as exc:
+                # Ignore files we cannot parse or convert
+                report.write("[Error] version converting file '%s': '%s'\n" %
+                             (file_path, exc))
+
+
+def main(args=None):
+    """
+    Convenience script to automatically convert odML files
+    within a directory (tree) to the newest file version.
+    Check the cli help for details.
+    :param args: Command line arguments
+    """
+    parser = docopt(__doc__, argv=args, version="0.1.0")
+
+    root = parser['SEARCHDIR']
+    if not os.path.isdir(root):
+        print(docopt(__doc__, "-h"))
+        exit(1)
+
+    # Handle all supported odML file formats.
+    if parser['-r']:
+        xfiles = list(pathlib.Path(root).rglob('*.odml'))
+        xfiles.extend(list(pathlib.Path(root).rglob('*.xml')))
+        jfiles = list(pathlib.Path(root).rglob('*.json'))
+        yfiles = list(pathlib.Path(root).rglob('*.yaml'))
+    else:
+        xfiles = list(pathlib.Path(root).glob('*.odml'))
+        xfiles.extend(list(pathlib.Path(root).glob('*.xml')))
+        jfiles = list(pathlib.Path(root).glob('*.json'))
+        yfiles = list(pathlib.Path(root).glob('*.yaml'))
+
+    out_root = os.getcwd()
+    if parser["-o"]:
+        if not os.path.isdir(parser["-o"]):
+            print("[Error] Could not find output directory '%s'" % parser["-o"])
+            exit(1)
+
+        out_root = parser["-o"]
+
+    out_dir = tempfile.mkdtemp(prefix="odmlconv_", dir=out_root)
+
+    # Use this monkeypatch reporter until there is a way
+    # to run the converters silently.
+    report = StringIO()
+    report.write("[Info] Files will be saved to '%s'\n" % out_dir)
+
+    run_conversion(xfiles, out_dir, report)
+    run_conversion(jfiles, out_dir, report, "JSON")
+    run_conversion(yfiles, out_dir, report, "YAML")
+
+    print(report.getvalue())
+    report.close()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

odml/scripts/odml_to_rdf.py

@@ -3,7 +3,9 @@
 odmlToRDF searches for odML files within a provided SEARCHDIR
 and converts them to the newest odML format version and
 exports all found and resulting odML files to XML formatted RDF.
-Original files will never be overwritten.
+Original files will never be overwritten. New files will be
+written either to a new directory at the current or a specified
+location.
 
 Usage: odmltordf [-r] [-o OUT] SEARCHDIR
 
@@ -14,7 +16,8 @@
     -o OUT          Output directory. Must exist if specified.
                     If not specified, output files will be
                     written to the current directory.
-    -r              Search recursively.
+    -r              Search recursively. Directory structures
+                    will not be retained.
     -h --help       Show this screen.
     --version       Show version.
 """
@@ -24,14 +27,15 @@
 import sys
 import tempfile
 
+from docopt import docopt
+
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
 import odml
 
-from docopt import docopt
 from odml.tools.odmlparser import ODMLReader, ODMLWriter
 from odml.tools.version_converter import VersionConverter as VerConf
 

setup.py

@@ -28,7 +28,7 @@
 with open('README.rst') as f:
     description_text = f.read()
 
-install_req = ["lxml", "pyyaml==3.13", "rdflib", "docopt"]
+install_req = ["lxml", "pyyaml==3.13", "rdflib", "docopt", "pathlib"]
 
 if sys.version_info < (3, 4):
     install_req += ["enum34"]
@@ -47,5 +47,6 @@
     long_description=description_text,
     classifiers=CLASSIFIERS,
     license="BSD",
-    entry_points={'console_scripts': ['odmltordf=odml.scripts.odml_to_rdf:main']}
+    entry_points={'console_scripts': ['odmltordf=odml.scripts.odml_to_rdf:main',
+                                      'odmlconversion=odml.scripts.odml_conversion:main']}
 )