Retire XDMF from the archive: the writers build ymf grid dicts, no module imports xml (C2)

proteus/AuxiliaryVariables.py

@@ -7,7 +7,6 @@
 import numpy
 from . import Viewers
 from . import Archiver
-from xml.etree.ElementTree import *
 
 from . import Profiling
 from .Profiling import logEvent
@@ -31,21 +30,19 @@ def attachModel(self,model,ar):
     def attachAuxiliaryVariables(self,avDict):
         pass
     def writeScalarXdmf(self,u,name):
-        #get the current spatial grid
-        for child in self.ar.domain:
-            if child.tag == "Grid" and child.attrib["Name"] == "Mesh Spatial_Domain":
-                self.arGridCollection = child
-        assert self.arGridCollection is not None
-        self.arGrid = self.arGridCollection[-1]
-        #write the attribute
-        attribute = SubElement(self.arGrid,"Attribute",{"Name":name,
-                                                        "AttributeType":"Scalar",
-                                                        "Center":"Grid"})
-        values    = SubElement(attribute,"DataItem",
-                               {"Format":"XML",
-                                **Archiver.xdmf_number_type(numpy.asarray(u)),
-                                "Dimensions":"1"})
-        values.text = str(u)
+        """Record one number for the step, inline in the archive document."""
+        from ymf.archive import attribute, data_item_for
+        #the current spatial grid
+        self.arGridCollection = self.ar.domain.get("Mesh Spatial_Domain")
+        assert self.arGridCollection is not None, \
+            "no Mesh Spatial_Domain collection; the mesh is archived first"
+        self.arGrid = self.arGridCollection.grid
+        assert self.arGrid is not None, \
+            "Mesh Spatial_Domain has no grid for this step yet"
+        #a grid-centred attribute holding the value itself
+        item = data_item_for(numpy.asarray(u).reshape((1,)), inline=True)
+        self.arGrid["attributes"].append(
+            attribute(name, item, center="Grid", attribute_type="Scalar"))
     def calculate_init(self):
         self.calculate()
     def calculate(self):

proteus/FemTools.py

@@ -4176,8 +4176,6 @@ def readFunctionXdmf(self,ar,u,tCount=0):
             u.dof[permute] = dataset[numbering[permute].tolist()]
         else:
             u.dof[:] = dataset[:]
-            #numpy.savetxt(ar.textDataDir+"/"+"{0:s}{0:d}".format(u.name, tCount)+".txt",u.dof)
-            #SubElement(values,"xi:include",{"parse":"text","href":"./"+ar.textDataDir+"/"+"{0:s}{0:d}".format(u.name, tCount)+".txt"})
     def writeVectorFunctionXdmf(self,ar,uList,components,vectorName,tCount=0,init=True):
         #XDMF vectors are always 3-component, so a 1D or 2D problem pads the
         #components it doesn't have with zeros. This was duplicated verbatim

proteus/NumericalSolution.py

@@ -608,8 +608,8 @@ def calculateSolution(self,runName):
             if not self.opts.hotStart:
                 logEvent("Archiving initial conditions")
                 self.archiveInitialSolution(m,index)
-            else:
-                self.ar[index].domain = self.ar[index].tree.find("Domain")
+            #a hot-started archive seeded its domain with the collections
+            #the archive already has when it was opened
             #if(not hasattr(self.pList[0].domain,'PUMIMesh') and not self.opts.hotStart):
             self.initializeViewSolution(m)
             logEvent("Estimating initial time derivative and initializing time history for model "+p.name)
@@ -1124,13 +1124,12 @@ def restrictFromFineMesh(self,model):
     ##save model's initial solution values to archive
     def archiveInitialSolution(self,model,index):
         if True if self.fastArchive == False else 'clsvof' in model.name:
-            import xml.etree.ElementTree as ElementTree
             if self.archiveFlag == ArchiveFlags.UNDEFINED:
                 return
             logEvent("Writing initial mesh for  model = "+model.name,level=3)
             logEvent("Writing initial conditions for  model = "+model.name,level=3)
-            if not self.so.useOneArchive or index==0:
-                self.ar[index].domain = ElementTree.SubElement(self.ar[index].tree.getroot(),"Domain")
+            #the archive owns its domain (Archiver.ArchiveDomain) from
+            #construction; the writers add their collections to it
             if self.so.useOneArchive:
                 model.levelModelList[-1].archiveFiniteElementSolutions(self.ar[index],self.tnList[0],self.tCount,initialPhase=True,
                                                                        writeVectors=True,meshChanged=True,femSpaceWritten=self.femSpaceWritten,

proteus/mbd/CouplingFSI.pyx

@@ -50,10 +50,33 @@ import h5py
 cimport ChronoHeaders as ch
 # chrono Python headers
 from proteus.mprans import BodyDynamics as bd
-from proteus.Archiver import indentXML
 import pychrono as chrono
 from pychrono import fea as chrono_fea
 
+
+def indentXML(elem, level=0):
+    """Pretty-print an ElementTree in place.
+
+    This module still writes its own XDMF for the bodies and cables it
+    moves (one ``.xmf`` per object, with the per-step fragments stored in
+    the HDF5 file the way proteus <= 1.9.x did). proteus's own archive no
+    longer builds XML, so the helper that used to live in Archiver lives
+    here until these writers go through the archive too.
+    """
+    i = "\n" + level*"  "
+    if len(elem):
+        if not elem.text or not elem.text.strip():
+            elem.text = i + "  "
+        for e in elem:
+            indentXML(e, level+1)
+            if not e.tail or not e.tail.strip():
+                e.tail = i + "  "
+        if not e.tail or not e.tail.strip():
+            e.tail = i
+    else:
+        if level and (not elem.tail or not elem.tail.strip()):
+            elem.tail = i
+
 # needed for sphinx docs
 __all__ = ['ProtChSystem',
            'ProtChBody',

test/mesh_tests/test_interpolatedbathy.py

@@ -4,7 +4,6 @@
 from proteus.MeshTools import InterpolatedBathymetryMesh
 from proteus.Archiver import XdmfArchive
 
-import xml.etree.ElementTree as ElementTree
 import pytest
 
 @pytest.mark.MeshTools
@@ -99,7 +98,6 @@ def test_L1(self):
                                           bathyAssignmentScheme="localAveraging",errorNormType="L1")
         outfile = "interpolatedBathySimpleTest_L1_"
         archive = XdmfArchive(dataDir='.',filename = outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -117,7 +115,6 @@ def test_L2(self):
                                           errorNormType="L2")
         outfile = "interpolatedBathySimpleTest_L2_"
         archive = XdmfArchive(dataDir='.',filename = outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -134,7 +131,6 @@ def test_Linfty(self):
                                           errorNormType="Linfty")
         outfile = "interpolatedBathySimpleTest_Linfty_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -152,7 +148,6 @@ def test_L1_interp(self):
                                           errorNormType="L1")
         outfile = "interpolatedBathySimpleTest_L1_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -170,7 +165,6 @@ def test_L2_interp(self):
                                           errorNormType="L2")
         outfile = "interpolatedBathySimpleTest_L2_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -188,7 +182,6 @@ def test_Linfty_interp(self):
                                           errorNormType="Linfty")
         outfile = "interpolatedBathySimpleTest_Linfty_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -206,7 +199,6 @@ def test_L1_grid(self):
                                           errorNormType="L1")
         outfile = "interpolatedBathySimpleTest_grid_L1_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -224,7 +216,6 @@ def test_L2_grid(self):
                                           errorNormType="L2")
         outfile = "interpolatedBathySimpleTest_grid_L2_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -242,7 +233,6 @@ def test_Linfty_grid(self):
                                           errorNormType="Linfty")
         outfile = "interpolatedBathySimpleTest_grid_Linfty_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -260,7 +250,6 @@ def test_L1_interp_grid(self):
                                           errorNormType="L1")
         outfile = "interpolatedBathySimpleTest_grid_L1_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -278,7 +267,6 @@ def test_L2_interp_grid(self):
                                           errorNormType="L2")
         outfile = "interpolatedBathySimpleTest_grid_L2_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         self.aux_names.append(outfile)
@@ -296,7 +284,6 @@ def test_Linfty_interp_grid(self):
                                           errorNormType="Linfty")
         outfile = "interpolatedBathySimpleTest_grid_Linfty_interp_"
         archive = XdmfArchive(dataDir='.',filename=outfile, global_sync=False)
-        archive.domain = ElementTree.SubElement(archive.tree.getroot(),"Domain")
         mesh.meshList[-1].writeMeshXdmf(ar=archive,init=True)
         archive.sync(); archive.close()
         mesh.meshList[-1].writeMeshADH("interpolatedBathySimpleTest_grid_Linfty_interp_")

test/test_archive_dtypes.py

@@ -13,7 +13,6 @@
 finding out afterwards with archives already written.
 """
 import os
-from xml.etree.ElementTree import Element, SubElement
 
 import numpy as np
 import pytest
@@ -58,13 +57,14 @@ def test_a_dtype_with_no_xdmf_type_is_refused_not_guessed():
 def test_topology_and_geometry_cannot_be_written_without_a_type():
     ar = Archiver.AR_base.__new__(Archiver.AR_base)
     ar.dataItemFormat, ar.hdfFile, ar.hdfFilename = "HDF", object(), "out.h5"
-    grid = Element("Grid")
+    grid = Archiver.ArchiveCollection("Mesh").new_grid(0.0, 0)
     with pytest.raises(TypeError):
         ar.write_topology(grid, "Triangle", 2, [2, 3], "elements", "elements")
     with pytest.raises(TypeError):
         ar.write_geometry(grid, [4, 3], "nodes", "nodes")
     item = ar.write_geometry(grid, [4, 3], "nodes", "nodes", dtype="float32")
-    assert (item.get("DataType"), item.get("Precision")) == ("Float", "4")
+    assert (item["data_type"], item["precision"]) == ("Float", 4)
+    assert grid["geometry"]["data_item"] is item
 
 
 # --------------------------------------------------------------------------
@@ -100,9 +100,7 @@ def __init__(self, mesh):
 @pytest.fixture
 def archive(tmp_path):
     Comm.init()
-    ar = Archiver.XdmfArchive(str(tmp_path), "dtypes", global_sync=False)
-    ar.domain = SubElement(ar.tree.getroot(), "Domain")
-    return ar
+    return Archiver.XdmfArchive(str(tmp_path), "dtypes", global_sync=False)
 
 
 def written_arrays(path):

test/test_archive_no_xml.py

@@ -0,0 +1,88 @@
+"""proteus builds its archive as ymf dicts and writes no XML.
+
+C2 of the campaign roster: only ``ymf2xmf`` should need XDMF. The archive
+(``Archiver.py``), the mesh writers (``MeshTools.py``), the finite-element
+writers (``FemTools.py``), the models that drive them and the auxiliary
+variables build ymf grid dicts through the archive's builders; the ``.xmf``
+a viewer wants is derived from the ``.ymf`` on demand. These tests pin that
+no module on the archive path imports an XML library, and that the running
+archive is one step of dicts, not a tree.
+
+The one writer that still builds XDMF, ``mbd/CouplingFSI.pyx``, keeps its
+own files for the bodies it moves and is outside this check.
+"""
+import ast
+import os
+
+import numpy as np
+import pytest
+
+PROTEUS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "proteus")
+
+ARCHIVE_PATH = ["Archiver.py", "MeshTools.py", "FemTools.py", "AuxiliaryVariables.py",
+                "NumericalSolution.py", "Transport.py", "SplitOperator.py"]
+
+
+def imports_of(path):
+    tree = ast.parse(open(path).read(), path)
+    found = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            found.update(alias.name for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            found.add(node.module)
+    return found
+
+
+@pytest.mark.parametrize("module", ARCHIVE_PATH)
+def test_the_archive_path_imports_no_xml_library(module):
+    names = imports_of(os.path.join(PROTEUS, module))
+    assert not [n for n in names if n == "xml" or n.startswith("xml.")], module
+
+
+def test_no_module_of_the_package_builds_xdmf_elements():
+    """``SubElement`` was the signature of the hand-built XDMF; none is left."""
+    offenders = []
+    for root, _, files in os.walk(PROTEUS):
+        for name in files:
+            if name.endswith(".py"):
+                path = os.path.join(root, name)
+                with open(path) as f:
+                    if "SubElement(" in f.read():
+                        offenders.append(os.path.relpath(path, PROTEUS))
+    assert offenders == []
+
+
+def test_the_running_archive_is_one_step_of_dicts():
+    from proteus.Archiver import ArchiveCollection, ArchiveDomain
+
+    domain = ArchiveDomain()
+    mesh = domain.collection("Mesh Spatial_Domain")
+    space = domain.collection("Mesh_c0p2_Lagrange")
+    assert domain.collection("Mesh Spatial_Domain") is mesh      # found, not duplicated
+    assert domain.collection("Mesh_dgp1", create=False) is None   # a continuing run expects it
+    assert domain.names() == ["Mesh Spatial_Domain", "Mesh_c0p2_Lagrange"]
+    assert domain[0] is mesh and len(domain) == 2
+    g = mesh.new_grid(0.25, 1)
+    assert mesh.is_current(0.25) and not space.is_current(0.25)
+    domain.clear_step()
+    assert mesh.grid is None and not mesh.is_current(0.25)
+    assert isinstance(g, dict) and g["attributes"] == []
+
+
+def test_a_grid_scalar_is_inlined_in_the_document():
+    """AuxiliaryVariables.writeScalarXdmf used to put the value in a DataItem's text."""
+    from proteus.Archiver import ArchiveDomain
+    from proteus.AuxiliaryVariables import AV_base
+
+    class Archive(object):
+        domain = ArchiveDomain()
+
+    av = AV_base()
+    av.ar = Archive()
+    av.ar.domain.collection("Mesh Spatial_Domain").new_grid(0.0, 0)
+    av.writeScalarXdmf(3.5, "force")
+    (attr,) = av.ar.domain[0].grid["attributes"]
+    assert attr["name"] == "force" and attr["center"] == "Grid"
+    assert attr["data_item"]["values"] == [3.5]
+    assert attr["data_item"]["format"] == "XML"

test/test_archive_parallel_metadata.py

@@ -54,7 +54,6 @@ def _find_mpiexec():
 RANK_SCRIPT = textwrap.dedent(
     '''
     import os, shutil, sys
-    from xml.etree.ElementTree import SubElement
     import numpy as np
 
     from proteus import Comm
@@ -79,23 +78,18 @@ def _find_mpiexec():
     if hot:
         # what NumericalSolution does: continue after the archive's last step
         ar.n_datasets = len(ar.archived_times)
-        ar.domain = ar.tree.getroot().find("Domain")
-    else:
-        ar.domain = SubElement(ar.tree.getroot(), "Domain")
 
     # Sizes differ per rank on purpose: the metadata dataset is fixed-width
     # and created collectively, so if the width agreement is wrong the
     # longest rank's YAML is silently truncated.
     n_elements, n_nodes = 4 + rank, 6 + rank
     if hot:
-        # found by name, as XdmfWriter.setGridCollectionAndGridElements does
-        (collection,) = [c for c in ar.domain
-                         if c.attrib["Name"] == "Mesh Spatial_Domain"]
+        # found by name, as XdmfWriter.setGridCollectionAndGridElements does;
+        # the archive seeded it from the .h5 when it was opened
+        collection = ar.domain.collection("Mesh Spatial_Domain", create=False)
+        assert collection is not None, ar.domain.names()
     else:
-        collection = SubElement(ar.domain, "Grid",
-                                {"Name": "Mesh Spatial_Domain",
-                                 "GridType": "Collection",
-                                 "CollectionType": "Temporal"})
+        collection = ar.domain.collection("Mesh Spatial_Domain")
 
     # global_sync describes one assembled global array, so the metadata
     # carries global counts and each rank contributes only the slice it owns.
@@ -107,7 +101,7 @@ def _find_mpiexec():
 
     times = [1.0] if hot else [0.0, 0.5]
     for tCount, t in enumerate(times, start=ar.n_datasets):
-        grid, _ = ar.write_grid(collection, "Grid_p%d" % rank, t, tCount)
+        grid = ar.write_grid(collection, "Grid_p%d" % rank, t, tCount)
         if global_sync:
             ar.write_topology(grid, "Triangle", n_elements_global,
                               [n_elements_global, 3],