Split publishing into template / publish commands

Author: stefanv

devstats/.gitignore

@@ -0,0 +1,2 @@
+source
+build

devstats/__main__.py

@@ -3,15 +3,25 @@
 import re
 import sys
 from glob import glob
+import collections
 
 import click
 import requests
 
 from .query import GithubGrabber
-from .publish import publish
+from .publish import template, publish
 
 
-@click.group()
+class OrderedGroup(click.Group):
+    def __init__(self, name=None, commands=None, **attrs):
+        super().__init__(name, commands, **attrs)
+        self.commands = commands or collections.OrderedDict()
+
+    def list_commands(self, ctx):
+        return self.commands
+
+
+@click.group(cls=OrderedGroup)
 def cli():
     pass
 
@@ -20,7 +30,7 @@ def cli():
 @click.argument("repo_owner")
 @click.argument("repo_name")
 def query(repo_owner, repo_name):
-    """Download and save issue and pr data for `repo_owner`/`repo_name`."""
+    """Download and save issue and pr data for `repo_owner`/`repo_name`"""
 
     try:
         token = os.environ["GRAPH_API_KEY"]
@@ -62,4 +72,5 @@ def query(repo_owner, repo_name):
         data.dump(f"{repo_name}_{ftype.get(qtype, qtype)}.json")
 
 
+cli.add_command(template)
 cli.add_command(publish)

devstats/publish.py

@@ -1,31 +1,77 @@
 import os
 import sys
 from glob import glob
+import shutil
+import re
+import functools
+
 import click
 
 
 @click.command()
 @click.argument("project")
 @click.option(
-    "-o", "--outdir", default="build", help="Output directory", show_default=True
+    "-o", "--outdir", default="source", help="Output directory", show_default=True
 )
-def publish(project, outdir):
-    """Generate myst report for `repo_owner`/`repo_name`."""
+def template(project, outdir):
+    """Generate myst report template for `repo_owner`/`repo_name`"""
     os.makedirs(outdir, exist_ok=True)
     os.makedirs(os.path.join(outdir, project), exist_ok=True)
 
+    print(f"Populating [{outdir}] with templates for [{project}] report:", flush=True)
+
     report_files = glob(os.path.join(os.path.dirname(__file__), "reports/*.md"))
+    for f in report_files:
+        dest = f"{outdir}/{project}/{os.path.basename(f)}"
+        print(f" - {dest}")
+        shutil.copyfile(f, dest)
+
+
+def _include_file(basedir, x):
+    fn = x.group(1)
+    with open(os.path.join(basedir, fn)) as f:
+        return f.read()
+
+
+@click.command()
+@click.argument("project")
+@click.option(
+    "-t",
+    "--templatedir",
+    default="source",
+    help="Template directory",
+    show_default=True,
+)
+@click.option(
+    "-o", "--outdir", default="build", help="Output directory", show_default=True
+)
+def publish(project, templatedir, outdir):
+    """Compile templates into markdown files ready for sphinx / myst"""
+    os.makedirs(outdir, exist_ok=True)
+    os.makedirs(os.path.join(outdir, project), exist_ok=True)
 
     variables = {"project": project}
 
-    print(f"Generating [{project}] report in [{outdir}]...", end="", flush=True)
+    print(f"Templating [{project}] report from [{templatedir}] to [{outdir}]...")
+
+    template_files = glob(f"{templatedir}/{project}/*.md")
 
-    for report in report_files:
-        with open(report) as fh:
+    for f in template_files:
+        with open(f) as fh:
             template = fh.read()
-        with open(f"{outdir}/{project}/{os.path.basename(report)}", "w") as fh:
+        dest_dir = f"{outdir}/{project}"
+        dest = f"{dest_dir}/{os.path.basename(f)}"
+        with open(dest, "w") as fh:
+            print(f" - {dest}")
             for v in variables:
                 template = template.replace("{{ " + v + " }}", variables[v])
-            fh.write(template)
 
-    print("OK")
+            # Handle myst includes
+            template = re.sub(
+                r"```{include}\s*(.*)\s*```",
+                functools.partial(_include_file, dest_dir),
+                template,
+                flags=re.MULTILINE,
+            )
+
+            fh.write(template)