Merge pull request #25 from stefanv/template-publish

Publish from templates

Author: jarrodmillman

devstats/.gitignore

@@ -0,0 +1,2 @@
+source
+build

devstats/__main__.py

@@ -3,24 +3,42 @@
 import re
 import sys
 from glob import glob
+import collections
 
 import click
 import requests
 
 from .query import GithubGrabber
-from .publish import publisher
+from .publish import template, publish
 
 
-@click.group()
+class OrderedGroup(click.Group):
+    def __init__(self, name=None, commands=None, **attrs):
+        super().__init__(name, commands, **attrs)
+        self.commands = commands or collections.OrderedDict()
+
+    def list_commands(self, ctx):
+        return self.commands
+
+
+@click.group(cls=OrderedGroup)
 def cli():
     pass
 
 
 @cli.command("query")
 @click.argument("repo_owner")
 @click.argument("repo_name")
-def query(repo_owner, repo_name):
-    """Download and save issue and pr data for `repo_owner`/`repo_name`."""
+@click.option(
+    "-o",
+    "--outdir",
+    default="devstats-data",
+    help="Output directory",
+    show_default=True,
+)
+def query(repo_owner, repo_name, outdir):
+    """Download and save issue and pr data for `repo_owner`/`repo_name`"""
+    os.makedirs(outdir, exist_ok=True)
 
     try:
         token = os.environ["GRAPH_API_KEY"]
@@ -59,11 +77,8 @@ def query(repo_owner, repo_name):
         )
         data.get()
         ftype = {"issues": "issues", "pullRequests": "PRs"}
-        data.dump(f"{repo_name}_{ftype.get(qtype, qtype)}.json")
+        data.dump(f"{outdir}/{repo_name}_{ftype.get(qtype, qtype)}.json")
 
 
-@cli.command("publish")
-@click.argument("project")
-def publish(project):
-    """Generate myst report for `repo_owner`/`repo_name`."""
-    publisher(project)
+cli.add_command(template)
+cli.add_command(publish)

devstats/publish.py

@@ -1,33 +1,94 @@
 import os
 import sys
 from glob import glob
-from pathlib import Path
+import shutil
+import re
+import functools
 
+import click
 
-def publisher(project):
-    print(f"Generating {project} report...", end="")
-    basedir = os.path.dirname(__file__)
-    with open(os.path.join(basedir, "template.md")) as fh:
-        template = fh.read()
 
-    issues = glob(os.path.join(basedir, "reports/issues/*.md"))
-    for issue in issues:
-        with open(issue) as fh:
-            issue_text = fh.read()
-        issue_name = Path(issue).stem
-        template = template.replace(f"{{{{ {issue_name} }}}}", issue_text)
+@click.command()
+@click.argument("project")
+@click.option(
+    "-o", "--outdir", default="source", help="Output directory", show_default=True
+)
+def template(project, outdir):
+    """Generate myst report templates
 
-    prs = glob(os.path.join(basedir, "reports/pull_requests/*.md"))
-    for pr in prs:
-        with open(pr) as fh:
-            pr_text = fh.read()
-        pr_name = Path(pr).stem
-        template = template.replace(f"{{{{ {pr_name} }}}}", pr_text)
+    These templates are copied from `devstats`, and still need to be compiled
+    to substitute variables.
+    """
+    os.makedirs(outdir, exist_ok=True)
+    os.makedirs(os.path.join(outdir, project), exist_ok=True)
 
-    template = template.replace("{{ project }}", project)
+    print(f"Populating [{outdir}] with templates for [{project}] report:", flush=True)
 
-    os.makedirs("_generated", exist_ok=True)
-    with open(f"_generated/{project}.md", "w") as fh:
-        fh.write(template)
+    report_files = glob(os.path.join(os.path.dirname(__file__), "reports/*.md"))
+    for f in report_files:
+        dest = f"{outdir}/{project}/{os.path.basename(f)}"
+        print(f" - {dest}")
+        shutil.copyfile(f, dest)
 
-    print("OK")
+
+def _include_file(basedir, x):
+    fn = x.group(1)
+    with open(os.path.join(basedir, fn)) as f:
+        return f.read()
+
+
+@click.command()
+@click.argument("project")
+@click.option(
+    "-t",
+    "--templatedir",
+    default="source",
+    help="Template directory",
+    show_default=True,
+)
+@click.option(
+    "-o", "--outdir", default="build", help="Output directory", show_default=True
+)
+def publish(project, templatedir, outdir):
+    """Compile templates (substitute variables) into markdown files ready for sphinx / myst
+
+    Include sections like the following are executed:
+
+    ```
+    {include} filename.md
+    ```
+
+    Thereafter, the following variables are substituted:
+
+      - `{{ project }}`: Name of the project
+
+    """
+    os.makedirs(outdir, exist_ok=True)
+    os.makedirs(os.path.join(outdir, project), exist_ok=True)
+
+    variables = {"project": project}
+
+    print(f"Templating [{project}] report from [{templatedir}] to [{outdir}]...")
+
+    templatedir = f"{templatedir}/{project}"
+    template_files = [f"{templatedir}/index.md"]
+
+    for f in template_files:
+        with open(f) as fh:
+            template = fh.read()
+        dest_dir = f"{outdir}/{project}"
+        dest = f"{dest_dir}/{os.path.basename(f)}"
+        with open(dest, "w") as fh:
+            print(f" - {dest}")
+            # Handle myst includes
+            template = re.sub(
+                r"```{include}\s*(.*)\s*```",
+                functools.partial(_include_file, templatedir),
+                template,
+                flags=re.MULTILINE,
+            )
+
+            for v in variables:
+                template = template.replace("{{ " + v + " }}", variables[v])
+
+            fh.write(template)

devstats/reports/index.md

@@ -0,0 +1,105 @@
+---
+file_format: mystnb
+kernelspec:
+  display_name: Python 3
+  name: python3
+---
+
+# `{{ project }}`
+
+```{include} preamble.md
+
+```
+
+A snapshot of the development on the {{ project }} project.
+
+## Issues
+
+% TODO: query_date should be synced up with the query that generates data,
+% rather than specified manually
+
+% TODO improve handling of datetimes (super annoying)
+
+```{code-cell}
+query_date = np.datetime64("2020-01-01 00:00:00")
+
+# Load data
+with open("devstats-data/{{ project }}_issues.json", "r") as fh:
+    issues = [item["node"] for item in json.loads(fh.read())]
+
+glue("devstats-data/{{ project }}_query_date", str(query_date.astype("M8[D]")))
+```
+
+```{include} new_issues.md
+
+```
+
+```{include} issue_time_to_response.md
+
+```
+
+```{include} issue_first_responders.md
+
+```
+
+## Pull Requests
+
+```{code-cell}
+---
+tags: [hide-input]
+---
+with open("devstats-data/{{ project }}_prs.json", "r") as fh:
+    prs = [item["node"] for item in json.loads(fh.read())]
+
+# Filters
+
+# The following filters are applied to the PRs for the following analysis:
+#
+# - Only PRs to the default development branch (e.g `main`)[^master_to_main]
+#  are considered.
+# - Only PRs from users with _active_ GitHub accounts are considered. For example,
+#   if a user opened a Pull Request in 2016, but then deleted their GitHub account
+#   in 2017, then this PR is excluded from the analysis.
+# - PRs opened by dependabot are excluded.
+
+# Only look at PRs to the main development branch - ignore backports,
+# gh-pages, etc.
+default_branches = {"main", "master"}  # Account for default branch update
+prs = [pr for pr in prs if pr["baseRefName"] in default_branches]
+
+# Drop data where PR author is unknown (e.g. github account no longer exists)
+prs = [pr for pr in prs if pr["author"]]  # Failed author query results in None
+
+# Filter out PRs by bots
+bot_filter = {
+  "dependabot-preview",
+  "github-actions",
+  "meeseeksmachine",
+  "pre-commit-ci[bot]"
+}
+prs = [pr for pr in prs if pr["author"]["login"] not in bot_filter]
+```
+
+```{include} prs_merged_over_time.md
+
+```
+
+```{include} prs_lifetime.md
+
+```
+
+```{include} prs_mergeability.md
+
+```
+
+```{include} prs_participants.md
+
+```
+
+```{include} prs_contributor_origin.md
+
+```
+
+```{include} prs_pony_factor.md
+
+```

devstats/reports/issue_first_responders.md

@@ -0,0 +1,25 @@
+#### First responders
+
+```{code-cell}
+---
+tags: [hide-input]
+---
+```
+
+```{code-cell}
+---
+tags: [hide-input]
+---
+first_commenter_tab = pd.DataFrame(
+    {
+        k: v
+        for k, v in zip(
+            ("Contributor", "# of times commented first"),
+            np.unique(first_commenters, return_counts=True),
+        )
+    }
+)
+first_commenter_tab.sort_values(
+    "# of times commented first", ascending=False
+).head(10)
+```