Add function/target documentation based on Sphinx/Jinja2. Replace command line function listings by hyperlink

Author: mlell

.github/workflows/docs.yml

@@ -13,7 +13,7 @@ jobs:
           python-version: '3.11'
       - uses: actions/checkout@v4
       - run: python -m pip install .[docs]
-      - run: python -m sphinx -W -b html docs/ build/html/
+      - run: python docs/make_bql_doc.py
       - uses: actions/upload-pages-artifact@v3
         with:
           path: build/html

README.rst

@@ -5,3 +5,18 @@ beanquery is a customizable and extensible lightweight SQL query tool
 that works on tabular data, including `Beancount`__ ledger data.
 
 __ https://beancount.github.io/
+
+With this tool you can write *queries* to extract information from your
+Beancount ledger. This is the documentation of functions and field names
+which are available for queries. 
+
+Please read the Manual of the `Beancount Query Language (BQL)`__ if you
+do not yet know how to write queries.
+
+
+__ http://furius.ca/beancount/doc/query
+
+After you have learned the BQL, you can use the 
+`list and documentation of available functions and columns`__
+
+__ https://beancount.github.io/beanquery

beanquery/query_compile.py

@@ -406,6 +406,14 @@ def __call__(self, context):
 
 
 class EvalFunction(EvalNode):
+    """Base class for function evaluation nodes.
+
+    Class Attributes:
+        __intypes__: List of input parameter types for type checking.
+        __outtype__: Output type of the function. None means no type
+          annotation. To annotate that the function returns None, use
+          types.NoneType.
+    """
     __slots__ = ('operands',)
 
     # Type constraints on the input arguments.

beanquery/query_env.py

@@ -80,10 +80,6 @@ def _extract_param_names(func):
     sig = inspect.signature(func)
     param_names = list(sig.parameters.keys())
 
-    # Remove 'self' if present (for class methods)
-    if param_names and param_names[0] == 'self':
-        param_names = param_names[1:]
-
     return param_names
 
 
@@ -154,6 +150,8 @@ def __call__(self, row):
         Func.__name__ = name if name is not None else func.__name__
         Func.__doc__ = func.__doc__
         query_compile.FUNCTIONS[Func.__name__].append(Func)
+        _add_to_doc_groups(Func, intypes, groups)
+
         return func
     return decorator
 
@@ -173,6 +171,8 @@ def decorator(cls):
         if name is not None:
             cls.__name__ = name
         query_compile.FUNCTIONS[cls.__name__].append(cls)
+        _add_to_doc_groups(cls, cls.__intypes__, groups)
+
         return cls
     return decorator
 
@@ -857,11 +857,7 @@ def interval(x):
 
 @function([relativedelta, datetime.date, datetime.date], datetime.date, groups = ['date'])
 def date_bin(stride, source, origin):
-    """Bin a date into the specified stride aligned with the specified origin.
 
-    As an extension to the the SQL standard ``date_bin()`` function this
-    function also accepts strides containing units of months and years.
-    """
     if stride.months or stride.years:
         if origin + stride <= origin:
             # FIXME: this should raise and error: stride must be greater than zero
@@ -1067,3 +1063,47 @@ def update(self, store, context):
             cur = store[self.handle]
             if cur is None or value > cur:
                 store[self.handle] = value
+
+def _describe_functions(functions, aggregates=False, type_filter=None):
+    """Describe functions, optionally filtered by input type category.
+
+    Args:
+        functions: Dictionary of (function name: EvalFunction subclass),
+          the actual class, not an object, which would represent a particular
+          function call.
+        aggregates: If True, show aggregates; if False, show regular functions
+        type_filter: Optional filter by input type category (see TYPE_CATEGORIES)
+    """
+    # Determine which functions to iterate over
+    if type_filter:
+        # Use the pre-populated FUNCTION_DOC_GROUPS for filtering
+        funcs_to_process = FUNCTION_DOC_GROUPS.get(type_filter, [])
+    else:
+        # Collect all functions from all groups
+        funcs_to_process = []
+        for name, funcs in functions.items():
+            funcs_to_process.extend(funcs)
+
+    entries = []
+    for func in funcs_to_process:
+        # Filter by aggregate vs non-aggregate
+        if aggregates != issubclass(func, query_compile.EvalAggregator):
+            continue
+
+        # Get the function name
+        name = func.__name__.lower()
+
+        # Assemble function signature for output using parameter names
+        args = ', '.join(f'{param_name}: {types.name(dtype)}'
+                        for param_name, dtype in zip(func.__param_names__, func.__intypes__))
+
+        if func.__outtype__:
+            outtype = types.name(func.__outtype__)
+        else:
+            outtype = None
+
+        doc = func.__doc__ or ''
+        entries.append((name, doc, args, outtype))
+
+    entries.sort()
+    return entries

beanquery/shell.py

@@ -629,69 +629,50 @@ def help_targets(self):
         template = textwrap.dedent("""
 
           The list of comma-separated target expressions may consist of columns,
-          simple functions and aggregate functions. If you use any aggregate
-          function, you must also provide a GROUP-BY clause.
+          simple functions and aggregate functions. You can use AS to determine
+          the output column name, for example:
 
-          Columns
-          -------
+              SELECT yearmonth(date) AS month ....
 
-          {columns}
+          If you use any aggregate function, you must also provide a GROUP-BY
+          clause.
 
-          Functions
-          ---------
+          See the online Beanquery documentation for the full list of columns,
+          functions, and aggregates:
 
-          {functions}
-
-          Aggregate functions
-          -------------------
-
-          {aggregates}
+              https://beancount.github.io/beanquery/
 
         """)
-        print(template.format(**_describe(self.context.tables['postings'],
-                                          query_compile.FUNCTIONS)), file=self.outfile)
+
+        print(template, file=self.outfile)
 
     def help_from(self):
         template = textwrap.dedent("""
 
           A logical expression that consist of columns on directives (mostly
           transactions) and simple functions.
 
-          Columns
-          -------
-
-          {columns}
-
-          Functions
-          ---------
+          See the online Beanquery documentation for the full list of columns,
+          functions, and aggregates:
 
-          {functions}
+              https://beancount.github.io/beanquery/
 
         """)
-        print(template.format(**_describe(self.context.tables['entries'],
-                                          query_compile.FUNCTIONS)),
-              file=self.outfile)
+        print(template, file=self.outfile)
 
     def help_where(self):
         template = textwrap.dedent("""
 
           A logical expression that consist of columns on postings and simple
           functions.
 
-          Columns
-          -------
+          See the online Beanquery documentation for the full list of columns,
+          functions, and aggregates:
 
-          {columns}
-
-          Functions
-          ---------
-
-          {functions}
+              https://beancount.github.io/beanquery/
 
         """)
-        print(template.format(**_describe(self.context.tables['postings'],
-                                          query_compile.FUNCTIONS)), file=self.outfile)
-
+        print(template, file=self.outfile)
 
 def _describe_columns(columns):
     out = io.StringIO()
@@ -702,35 +683,6 @@ def _describe_columns(columns):
         print(file=out)
     return out.getvalue().rstrip()
 
-
-def _describe_functions(functions, aggregates=False):
-    entries = []
-    for name, funcs in functions.items():
-        if aggregates != issubclass(funcs[0], query_compile.EvalAggregator):
-            continue
-        name = name.lower()
-        for func in funcs:
-            args = ', '.join(types.name(d) for d in func.__intypes__)
-            doc = re.sub(r'[ \n\t]+', ' ', func.__doc__ or '')
-            entries.append((name, doc, args))
-    entries.sort()
-    out = io.StringIO()
-    wrapper = textwrap.TextWrapper(initial_indent='  ', subsequent_indent='  ', width=80)
-    for key, entries in itertools.groupby(entries, key=lambda x: x[:2]):  # noqa: B020
-        for name, doc, args in entries:
-            print(f'{name}({args})', file=out)
-        print(wrapper.fill(doc), file=out)
-        print(file=out)
-    return out.getvalue().rstrip()
-
-
-def _describe(table, functions):
-    return dict(
-        columns=_describe_columns(table.columns),
-        functions=_describe_functions(functions, aggregates=False),
-        aggregates=_describe_functions(functions, aggregates=True))
-
-
 def summary_statistics(entries):
     """Calculate basic summary statistics to output a brief welcome message.
 

beanquery/sources/beancount.py

@@ -104,6 +104,13 @@ class Position(types.Structure):
 
 
 class Cost(types.Structure):
+    """The amount which was payed for a position. This object
+    saves besides the amount also the date and label (if assigned).
+    This serves to identify the position to sell according to
+    a `given booking rule`__.
+
+    __ https://beancount.github.io/docs/beancount_language_syntax.html#reducing-positions
+    """
     name = 'cost'
     columns = _typed_namedtuple_to_columns(data.Cost)
 
@@ -296,7 +303,8 @@ def id(entry):
 
     @columns.register(str)
     def type(entry):
-        """The data type of the directive."""
+        """The data type of the directive. Currently, beanquery only can list
+        the directives of type 'transaction'."""
         return type(entry).__name__.lower()
 
     @columns.register(str)

docs/bql_functions.rst.j2

@@ -0,0 +1,58 @@
+List of all BQL Functions
+==========================
+
+This page documents all available BQL functions.
+
+..
+    Disambiguate functions and types by pretending they are in 
+    different modules (we are documenting BQL with the Sphinx Python
+    domain). We put functions in the virtual module "fun" (thus
+    functions must be explicitly linked like :func:`~fun.myfunction`).
+    Types have no module such that they are auto-linked by the 
+    `.. function:` directives.
+
+.. currentmodule:: fun
+
+{# Render documentation for all functions. This uses the function docstrings
+defined in beancount/query_env.py. There are multiple variants of many functions,
+differing in the type and number of arguments. We group the variants which
+have the same docstrings to render them en bloc. If all variants of a function
+of a certain name have the same docstring, we list all signatures directly in
+the RST function:: directive. Otherwise, we write .. function:: myfunction(...) 
+and render code blocks with the signatures, followed by the documentation for
+this group of variants.
+#}
+{% macro render_functions(functions) %}
+{% for name, args, has_multiple_docs, docs in preprocess_function_documentation(functions) %}
+{% if has_multiple_docs %}
+.. function:: {{ name }}(...)
+
+{% for variant in docs %}
+  ::
+
+    {{ variant.signatures }}
+
+{{ variant.doc_text | indent(2, first=True) }}
+
+{% endfor %}
+{% else %}
+.. function::
+{% for sig in args %}
+   {{ sig }}
+{% endfor %}
+
+{{ docs | indent(2, first=True) }}
+
+{% endif %}
+{% endfor %}
+{% endmacro %}
+
+Simple functions
+----------------
+
+{{ render_functions(_describe_functions(FUNCTIONS)) }}
+
+Aggregation functions
+---------------------
+
+{{ render_functions(_describe_functions(FUNCTIONS, aggregates=True)) }}

docs/columns_entry.rst.j2

@@ -0,0 +1,14 @@
+Targets: Entry
+==============
+
+This is the list of all the fields you can use in the 
+SELECT ... clause and the FROM ... clause of a BQL query.
+
+For available functions and aggregates to use with these 
+columns, see :doc:`functions_index`.
+
+{% for name, type_name, doc in preprocess_targets(EntriesTable) -%}
+* **{{ name }}**: ({{ type_name }}) {{ doc }}
+
+{% endfor %}
+

docs/columns_postings.rst.j2

@@ -0,0 +1,14 @@
+Fields: Posting
+===============
+
+This is the list of all the fields you can use in the 
+WHERE ... clause of a BQL query.
+
+For available functions and aggregates to use with these 
+columns, see :doc:`functions_index`.
+
+{% for name, type_name, doc in preprocess_targets(PostingsTable) -%}
+* **{{ name }}**: ({{ type_name }}) {{ doc }}
+
+{% endfor %}
+

docs/conf.py

@@ -1,17 +1,19 @@
+import beanquery 
+
 project = 'beanquery'
 copyright = '2014-2022, beanquery Contributors'
 author = 'beanquery Contributors'
-version = '0.1'
+version = beanquery.__version__
 language = 'en'
 html_theme = 'furo'
 html_title = f'{project} {version}'
 html_logo = 'logo.svg'
 extensions = [
-    'sphinx.ext.autodoc',
     'sphinx.ext.napoleon',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.extlinks',
-    'sphinx.ext.githubpages',
+    'sphinx.ext.autodoc',
+    #'sphinx.ext.intersphinx',
+    #'sphinx.ext.extlinks',
+    #'sphinx.ext.githubpages',
 ]
 extlinks = {
     'issue': ('https://github.com/beancount/beanquery/issues/%s', '#'),
@@ -25,3 +27,5 @@
 napoleon_use_param = False
 autodoc_typehints = 'none'
 autodoc_member_order = 'bysource'
+# see make_bql_doc.py, we use virtual module "fun" for BQL functions
+add_module_names = False