From 3be0c4b2c14838510ed0e393cbe103d816670192 Mon Sep 17 00:00:00 2001
From: Lon Hohberger <lon@metamorphism.com>
Date: Wed, 14 Aug 2024 20:09:39 -0400
Subject: [PATCH] Use Jinja2 for variable templates

Rather than maintaining our own parser, jinja2 can be used.
Miguel G. pointed out that we can simply change the glyphs used
to denote a variable declaration to avoid conflicts with Jira
syntax.

This has numerous advantages.

Contrast to 2 commits ago:
 - variable declartions -> @@var@@ to {@var@}
 - No way to specify an optional value (that's OK I think)
 - Use Jinja2 templating by passing raw text from reading
   the template instead of the dict loaded by yaml.safe_load()
---
 contrib/example-template.yaml      |   6 +-
 jirate/jira_cli.py                 |  22 ++--
 jirate/template_vars.py            | 158 +++++++++++++++++++----------
 jirate/tests/test_template_vars.py | 116 +++++++--------------
 requirements.txt                   |   1 +
 5 files changed, 157 insertions(+), 146 deletions(-)

diff --git a/contrib/example-template.yaml b/contrib/example-template.yaml
index 8b04798..d3f363b 100644
--- a/contrib/example-template.yaml
+++ b/contrib/example-template.yaml
@@ -1,6 +1,7 @@
+{% set type=type or "Story" %}
 issues:
-  - issue_type: "Story"
-    summary: "Set up Jirate templates on my machine"
+  - issue_type: {@type@}
+    summary: "Set up {@name@} templates {@version|default('1.0')@} on my machine"
     # Description terminated by two newlines
     description: |
       * {*}This is a multi-line description.{*}
@@ -11,7 +12,6 @@ issues:
        * {*}Termination{*}.
        ** You can terminate multi-line strings with two newlines.
 
-
     subtasks:
       - summary: "Clone the repo and install it"
         description: "Description for subtask 1"
diff --git a/jirate/jira_cli.py b/jirate/jira_cli.py
index 05dab89..89ac28d 100644
--- a/jirate/jira_cli.py
+++ b/jirate/jira_cli.py
@@ -467,7 +467,7 @@ def _parse_creation_args(issue_data, required_fields=None, reserved_fields=None,
 
 def create_from_template(args):
     with open(args.template_file, 'r') as yaml_file:
-        template = yaml.safe_load(yaml_file)
+        template = yaml_file.read()
 
     values = {}
     # Always render. there should be defaults.
@@ -486,8 +486,10 @@ def create_from_template(args):
     interactive = sys.stdin.isatty() and not args.non_interactive
     template_output = apply_values(template, values, interactive)
 
+    template = yaml.safe_load(template_output)
+
     try:
-        validate_template(args)
+        _validate_template(args.project, template)
     except jsonschema.exceptions.ValidationError as e:
         print(f"Provided template file is not valid: {args.template_file}")
         raise e
@@ -561,12 +563,9 @@ def _create_from_template(args, template):
     return all_filed
 
 
-def validate_template(args):
-    with open(args.template_file, 'r') as yaml_file:
-        template = yaml.safe_load(yaml_file)
-
+def _validate_template(project, template):
     for i, issue in enumerate(template['issues']):
-        template['issues'][i] = {args.project.field_to_id(name): value for name, value in issue.items()}
+        template['issues'][i] = {project.field_to_id(name): value for name, value in issue.items()}
 
     schema_dir = files('jirate').joinpath('schemas')
     schemas = {}
@@ -579,6 +578,15 @@ def validate_template(args):
 
     # Will raise a ValidationError with details on what failed:
     validator.validate(template)
+    return True
+
+
+def validate_template(args):
+    with open(args.template_file, 'r') as yaml_file:
+        template = yaml.safe_load(yaml_file)
+
+    _validate_template(args.project, template)
+
     # If we get here it means validation succeeded.
     print(f"Template {args.template_file} is valid.")
     return (0, True)
diff --git a/jirate/template_vars.py b/jirate/template_vars.py
index 2e6e27e..661c9b2 100644
--- a/jirate/template_vars.py
+++ b/jirate/template_vars.py
@@ -1,73 +1,126 @@
 #!/usr/bin/python3
 #
-# Jinja2 variable substitution uses glyphs that are native to JIRA so
-# use direct/simple substitution instead.  CPaaS uses @@var@@, so we
-# will follow suit.
 #
-# Assigning a default value:  @@var:value@@
 
 import copy
 import re
 
-_sub_left = '@@'
-#_sub_right = '((:([^@]+))|(\\?([^@]+)?))?@@'  # for allowing null/empty
-_sub_right = '(:([^@]+))?@@'
-_base_pattern = _sub_left + '([a-z_]+)' + _sub_right
-
-
-def _apply_values(inp, values):
-    if isinstance(inp, str):
-        match = re.findall(_base_pattern, inp)
-        for glyph in match:
-            varname, _, __ = glyph
-            inp = re.sub(_sub_left + varname + _sub_right, values[varname], inp)
-    elif isinstance(inp, list):
-        ret = []
-        for val in inp:
-            ret.append(_apply_values(val, values))
-        inp = ret
-    elif isinstance(inp, dict):
-        for key in inp:
-            inp[key] = _apply_values(inp[key], values)
-
-    return inp
-
-
-def _populate_defaults(inp, values):
-    if isinstance(inp, str):
-        match = re.findall(_base_pattern, inp)
-        for glyph in match:
-            key, _, value = glyph
-            if key in values and values[key]:
-                continue
-            values[key] = value
-    elif isinstance(inp, list):
-        for val in inp:
-            _populate_defaults(val, values)
-    elif isinstance(inp, dict):
-        for key in inp:
-            _populate_defaults(inp[key], values)
-
-
-def update_values_interactive(values):
+from jinja2 import Environment, BaseLoader, meta, nodes
+
+
+# If a value is not provided on the command line, ask for it here.
+def update_values_interactive(values, cli_values):
     ret = {}
     for key in values:
+        # If provided via CLI, move on
+        if key in cli_values:
+            ret[key] = cli_values[key]
+            continue
+
+        # If a default was set in the jinja2 template, start with that
         if values[key]:
-            ret[key] = input(f'Value for "{key}" (default: "{values[key]}"):')
+            ret[key] = input(f'Value for "{key}" (default: "{values[key]}"): ')
         else:
-            ret[key] = input(f'Value for "{key}":')
+            ret[key] = input(f'Value for "{key}": ')
         if not ret[key]:
             ret[key] = values[key]
 
     return ret
 
 
+# In Jinja2, you can set a default globally like this:
+# {% set var = "value" %}
+# But, if you do that, you no longer can override the default
+# when you render the template.
+# You also can set a default per variable instance, but we will
+# overwrite it with the first.  So, for setting a default value
+# in a template, you can do:
+#
+# {% set var=var or "value" %}  # form 1; code at top of file
+# {{var|default("value")}}      # form 2; inline var w/ default
+#
+# Process code block variable default (form 1)
+# Input:
+#   {% set version=version or "1.0" %}
+# Output:
+#   Assign(target=Name(name='abc', ctx='store'), node=Or(left=Name(name='version', ctx='load'), right=Const(value='1.0'))),
+def __assign_default(node, ret):
+    if not isinstance(node.node, nodes.Or):
+        return False
+    if not isinstance(node.node.left, nodes.Name):
+        return False
+    if not isinstance(node.node.right, nodes.Const):
+        return False
+    ret[node.target.name] = node.node.right.value
+    return True
+
+
+# Process inline variable default (form 2)
+# Input:
+#   {{version|default('1.0')}}
+# Output node tree:
+#   Output(nodes=[Filter(node=Name(name='version', ctx='load'), name='default', args=[Const(value='1.0')]
+def __filter_default(node, ret):
+    if not isinstance(node, nodes.Filter):
+        return False
+    if not isinstance(node.node, nodes.Name):
+        return False
+    if node.name != 'default':
+        return False
+    if len(node.args) > 1:
+        return False
+    if not isinstance(node.args[0], nodes.Const):
+        return False
+
+    ret[node.node.name] = node.args[0].value
+    return True
+
+
+def __assemble_from_tree(node, ret):
+    if isinstance(node, nodes.Assign):
+        if __assign_default(node, ret):
+            # Successfully found a mutable variable in form 1
+            return ret
+    elif isinstance(node, nodes.Output):
+        for item in node.nodes:
+            __assemble_from_tree(item, ret)
+    elif isinstance(node, nodes.Filter):
+        if __filter_default(node, ret):
+            # Successfully found a mutable variable in form 2
+            return ret
+    elif isinstance(node, nodes.Template):
+        for item in node.body:
+            __assemble_from_tree(item, ret)
+
+    return ret
+
+
+def assemble_from_tree(tree):
+    ret = {}  # initialize
+    return __assemble_from_tree(tree, ret)
+
+
 def apply_values(inp, values={}, interactive=False):
-    template_values = {}
-    _populate_defaults(inp, template_values)
+    # Jinja2 variable substitution uses glyphs that are native to JIRA so
+    # use a modified one. {{ -> {@, }} -> @}
+    env = Environment(loader=BaseLoader,
+                      trim_blocks=True, lstrip_blocks=True,
+                      variable_start_string='{@',
+                      variable_end_string='@}')
+    jinja_template = env.from_string(inp)
+    ast = env.parse(inp)
+
+    # Pass 1: store all values w/ defaults
+    template_values = assemble_from_tree(ast)
+
+    # pass 2: store unassigned variables too
+    unset_keys = meta.find_undeclared_variables(ast)
+    for key in unset_keys:
+        if key not in template_values:
+            template_values[key] = ''
 
     if interactive:
-        template_values = update_values_interactive(template_values)
+        values = update_values_interactive(template_values, values)
 
     extra = []
     for key in values:
@@ -84,7 +137,4 @@ def apply_values(inp, values={}, interactive=False):
     if missing:
         raise ValueError(f'Missing value(s) for {missing}')
 
-    outp = copy.deepcopy(inp)
-    outp = _apply_values(outp, template_values)
-
-    return outp
+    return jinja_template.render(template_values)
diff --git a/jirate/tests/test_template_vars.py b/jirate/tests/test_template_vars.py
index 223365f..05e6fbd 100644
--- a/jirate/tests/test_template_vars.py
+++ b/jirate/tests/test_template_vars.py
@@ -1,94 +1,46 @@
 #!/usr/bin/python3
 
-from jirate.template_vars import apply_values, _populate_defaults
+from jirate.template_vars import apply_values
 
 import pytest  # NOQA
 import types
 
 
-def test_populate_defaults_simple():
-    inp = '@@a:b@@'
-    values = {}
+def test_apply_var_type1():
+    inp = """
+{% set var=var or "1.0" %}
+fork: {@var@}
+"""
+    exp = 'fork: 1.0'
+    assert apply_values(inp).strip() == exp.strip()
 
-    _populate_defaults(inp, values)
-    assert values == {'a': 'b'}
 
+def test_apply_var_type2():
+    inp = """
+fork: {@var|default('1.0')@}
+"""
+    exp = 'fork: 1.0'
+    assert apply_values(inp).strip() == exp.strip()
 
-def test_ok_variable_no_default():
-    inp = '@@a@@'
-    values = {'a': 'b'}
-
-    ret = apply_values(inp, values)
-    assert ret == 'b'
-
-
-def test_populate_defaults_no_overwrite():
-    inp = '@@a:b@@'
-    values = {'a': 'c'}
-
-    ret = apply_values(inp, values)
-    assert ret == 'c'
-
-
-def test_populate_defaults_first_value():
-    inp = '@@a:b@@ @@a:c@@'
-    values = {}
-
-    ret = apply_values(inp, values)
-    assert ret == 'b b'
-
-
-def test_populate_defaults_no_overwrite2():
-    inp = '@@a:b@@ @@a:c@@'
-    values = {'a': 'd'}
-
-    ret = apply_values(inp, values)
-    assert ret == 'd d'
-
-
-def test_populate_defaults_second_def():
-    inp = '@@a@@ @@a:c@@'
-    values = {}
-
-    ret = apply_values(inp, values)
-    assert ret == 'c c'
-
-
-def test_multi_str1():
-    inp = 'abc@@a@@def@@b@@ghi@@a@@jkl@@c@@'
-    values = {'a': '1', 'b': '2', 'c': '3'}
-
-    ret = apply_values(inp, values)
-    assert ret == 'abc1def2ghi1jkl3'
-
-
-def test_list1():
-    inp = ['one', '@@ver@@', 'two']
-    values = {'ver': '1.0'}
-
-    ret = apply_values(inp, values)
-    assert ret == ['one', '1.0', 'two']
-
-
-def test_dict1():
-    inp = {'top': '@@ver@@', 'bottom': '@@old:0.1@@'}
-    values = {'ver': '1.0'}
-
-    ret = apply_values(inp, values)
-    assert ret == {'top': '1.0', 'bottom': '0.1'}
-
-
-def test_complex1():
-    inp = {'top': '@@ver@@', 'bottom': ['@@old:0.1@@', '@@date@@', {'pork': '@@bacon@@'}]}
-    values = {'ver': '1.0', 'date': '2024-07-25', 'bacon': 'yum'}
-
-    ret = apply_values(inp, values)
-    assert ret == {'top': '1.0', 'bottom': ['0.1', '2024-07-25', {'pork': 'yum'}]}
-
-
-def test_invalid_var():
-    inp = {'fun': '@@bar@@'}
-    values = {'baz': '123'}
 
+def test_missing_var():
+    inp = """
+fork: {@var|default('1.0')@}
+knife: {@butter@}
+"""
     with pytest.raises(ValueError):
-        ret = apply_values(inp, values)
+        apply_values(inp)
+
+
+def test_provided_var():
+    inp = """
+fork: {@var|default('1.0')@}
+knife: {@butter@}
+"""
+    exp = """
+fork: 2.0
+knife: salted
+"""
+    values = {'var': '2.0',
+              'butter': 'salted'}
+    assert apply_values(inp, values).strip() == exp.strip()
diff --git a/requirements.txt b/requirements.txt
index 58bc046..49a3d38 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,6 +4,7 @@ editor>=1.2.1
 PyYAML
 rich
 jira>=3.8.0
+Jinja2>=3.0.0
 python-dateutil
 toolchest
 prettytable