863: add expression delimiter syntax

pyxform/entities/entities_parsing.py

@@ -7,8 +7,8 @@
 from pyxform.elements import action
 from pyxform.errors import ErrorCode, PyXFormError
 from pyxform.parsing.expression import is_xml_tag
+from pyxform.parsing.variable_reference import parse_pyxform_references
 from pyxform.question_type_dictionary import get_meta_group
-from pyxform.validators.pyxform.pyxform_reference import parse_pyxform_references
 
 EC = const.EntityColumns
 

pyxform/errors.py

@@ -246,6 +246,30 @@ class ErrorCode(Enum):
             "path '{path}'."
         ),
     )
+    INTERNAL_003 = Detail(
+        name="Internal error - delimited expression parsing failed, grammar error",
+        msg=(
+            "Delimited expression parsing failed. "
+            "Check the expression syntax, or try using delimiter escapes. "
+            "Input string: {s}"
+        ),
+    )
+    INTERNAL_004 = Detail(
+        name="Internal error - delimited expression parsing failed, nested expression",
+        msg=(
+            "Delimited expression parsing failed. "
+            "Expressions must not be nested inside another expression. "
+            "Input string: {s}"
+        ),
+    )
+    INTERNAL_005 = Detail(
+        name="Internal error - delimited expression parsing failed, empty expression",
+        msg=(
+            "Delimited expression parsing failed. "
+            "Expressions must not be empty or only contain whitespace characters. "
+            "Input string: {s}"
+        ),
+    )
     LABEL_001: Detail = Detail(
         name="Labels - invalid missing label in the choices sheet",
         msg=(

pyxform/parsing/expression.py

@@ -1,6 +1,5 @@
 import re
 from functools import lru_cache
-from typing import Any
 
 from lark import Lark, Token
 
@@ -74,7 +73,7 @@
     CLOSE_PAREN.16: /\)/
     BRACKET.15: /[\[\{{\}}]/
     PARENT_REF.14: /\.\./
-    SELF_REF.13: /\./\
+    SELF_REF.13: /\./
     // # javarosa.xpath says "//" is an "unsupported construct".
     PATH_SEP.12: /\//
     SYSTEM_LITERAL.11: /"[^"]*"|'[^']*'/
@@ -88,9 +87,9 @@
     // Must be lower priority than rules containing ncname_regex.
     NAME.3: /{ncname_regex_ns}/
     PYXFORM_REF_START.2: /\$\{{/
-    PYXFORM_REF_END.1: /\}}/\
+    PYXFORM_REF_END.1: /\}}/
     // Catch any other character so that parsing doesn't stop.
-    OTHER.0: /.+?/\
+    OTHER.0: /.+?/
 """
 
 RE_NCNAME_NAMESPACED = re.compile(ncname_regex_ns_named)
@@ -105,32 +104,18 @@
 
 
 @lru_cache(maxsize=128)
-def parse_expression(text: str) -> tuple[Token, ...]:
+def lex_expression(text: str) -> tuple[Token, ...]:
     """
-    Parse an expression.
+    Lex an expression.
 
     Use this function instead of _EXPRESSION_LEXER to take advantage of caching.
 
     :param text: The expression.
-    :return: The parsed tokens, and any remaining unparsed text.
+    :return: The lexed tokens, and any remaining text.
     """
     return tuple(_EXPRESSION_LEXER.lex(text))
 
 
 def is_xml_tag(value: str) -> bool:
     """Check if the input string contains only a valid XML tag / element name."""
-    return value and bool(RE_NCNAME_NAMESPACED.fullmatch(value))
-
-
-def maybe_strip(value: Any) -> Any:
-    """
-    If the value is a string and looks like it has whitespace at either end, strip it.
-
-    If a string was "interned" (cached) by Python, string.strip() should generally return
-    the existing string if no leading/trailing whitespace was found. But strings may or
-    may not be interned by Python, and there may be a large cache for many unique values
-    (which is likely for XLSForms), so this function tries to avoid calling strip().
-    """
-    if isinstance(value, str) and value and (value[0].isspace() or value[-1].isspace()):
-        return value.strip()
-    return value
+    return bool(value) and bool(RE_NCNAME_NAMESPACED.fullmatch(value))

pyxform/parsing/expression_delimited.py

@@ -0,0 +1,144 @@
+from typing import TYPE_CHECKING
+
+from lark import Lark, LarkError, Transformer
+
+from pyxform.errors import ErrorCode, PyXFormError
+from pyxform.parsing.instance_expression import (
+    replace_with_output as replace_with_output_old,
+)
+from pyxform.parsing.utils import maybe_strip
+from pyxform.parsing.variable_reference import (
+    is_pyxform_reference_candidate,
+    resolve_variables,
+)
+from pyxform.utils import node
+
+if TYPE_CHECKING:
+    from lark import Token
+
+    from pyxform.survey import Survey
+    from pyxform.survey_element import SurveyElement
+
+
+expression_grammar = r"""
+start: (expression | escaped | text)*
+
+expression: _OPEN (invalid_nesting | escaped | TEXT_NON_DELIMITER | CHAR_NON_DELIMITER)* _CLOSE
+invalid_nesting: expression
+escaped: ESCAPED_DELIMITER
+text: (TEXT_NON_DELIMITER | CHAR_NON_DELIMITER)+
+
+_OPEN: "{{"
+_CLOSE: /}\}(?!})/
+ESCAPED_DELIMITER: "\\{{" | "\\}}"
+TEXT_NON_DELIMITER: /[^{}\\]+/
+CHAR_NON_DELIMITER: /[{}\\]/
+"""
+
+
+_EXPRESSION_PARSER = Lark(
+    expression_grammar, parser="lalr", lexer="contextual", cache=True
+)
+
+
+class ExpressionTransformer(Transformer):
+    def __init__(
+        self, context: "SurveyElement", survey: "Survey", visit_tokens: bool = True
+    ):
+        super().__init__(visit_tokens=visit_tokens)
+        self.context: SurveyElement = context
+        self.survey: Survey = survey
+        self.expression_modified: bool = False
+
+    @staticmethod
+    def _flatten(children: list["Token"]) -> str:
+        """Combine leaf child strings into one string."""
+        if len(children) == 0:
+            return ""
+        elif len(children) > 1:
+            return "".join(children)
+        else:
+            return str(children[0])
+
+    def start(self, children: list["Token"]) -> str:
+        """Join all children strings."""
+        return self._flatten(children=children)
+
+    def text(self, children: list["Token"]) -> str:
+        """Join all children strings, resolve variable references."""
+        return resolve_variables(
+            xml_text=self._flatten(children=children),
+            survey=self.survey,
+            context=self.context,
+            wrap=True,
+        )
+
+    def escaped(self, children: list["Token"]) -> str:
+        r"""Drop the leading backslash: `'\{{' -> '{{'`, `'\}}' -> '}}'`."""
+        resolved = {r"\{{": "{{", r"\}}": "}}"}[children[0]]
+        self.expression_modified = True
+        return resolved
+
+    def invalid_nesting(self, children: list["Token"]) -> str:
+        """Raise an error if a nested expression is found."""
+        raise PyXFormError(code=ErrorCode.INTERNAL_004)
+
+    def expression(self, children: list["Token"]) -> str:
+        """Join/wrap children in XML: `<output value='children123'/>`, resolve variable references."""
+        flat = self._flatten(children=children)
+        if len(maybe_strip(flat)) == 0:
+            raise PyXFormError(code=ErrorCode.INTERNAL_005)
+        resolved = resolve_variables(
+            xml_text=flat,
+            survey=self.survey,
+            context=self.context,
+        )
+        self.expression_modified = True
+        return node("output", value=resolved).toxml()
+
+
+def replace_with_output(xml_text: str, context: "SurveyElement", survey: "Survey") -> str:
+    """
+    Wrap expressions in output element: `{{contents}} -> <output value="contents"/>`.
+
+    Resolves variable references inside the expression, but not outside, so an extra pass
+    over the string is required.
+
+    :param xml_text: Text that may contain an expression.
+    :param context: The element context that variable references should be resolved in,
+      for example to determine relative references.
+    :param survey: The parent Survey object.
+    """
+    # Minimum length token that could be changed.
+    if len(xml_text) < 3:
+        return xml_text
+
+    try:
+        if "{" in xml_text or "}" in xml_text:
+            transformer = ExpressionTransformer(survey=survey, context=context)
+            out_str = transformer.transform(_EXPRESSION_PARSER.parse(xml_text))
+            if transformer.expression_modified:
+                return out_str
+
+        # Fallback for old func that only looks for non-delimited `instance()`.
+        fallback = replace_with_output_old(xml_text, context, survey)
+        if is_pyxform_reference_candidate(fallback):
+            fallback = resolve_variables(
+                xml_text=fallback, survey=survey, context=context, wrap=True
+            )
+        if fallback != xml_text:
+            return fallback
+        else:
+            return xml_text
+    except LarkError as e:
+        if isinstance(e.__context__, PyXFormError):
+            ctx = e.__context__
+            if ctx.code:
+                ctx.context["s"] = xml_text
+                raise PyXFormError(code=ctx.code, context=ctx.context) from e
+            else:
+                raise PyXFormError(ctx.args[0]) from e
+        else:
+            raise PyXFormError(
+                code=ErrorCode.INTERNAL_003, context={"s": xml_text}
+            ) from e

pyxform/parsing/instance_expression.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING
 
-from pyxform.parsing.expression import RE_PYXFORM_REF, parse_expression
+from pyxform.parsing.expression import RE_PYXFORM_REF, lex_expression
 from pyxform.utils import node
 
 if TYPE_CHECKING:
@@ -21,7 +21,7 @@ def find_boundaries(xml_text: str) -> list[tuple[int, int]]:
     :param xml_text: XML text that may contain an instance expression.
     :return: Tokens in instance expression, and the string position boundaries.
     """
-    tokens = parse_expression(xml_text)
+    tokens = lex_expression(xml_text)
     if not tokens:
         return []
     instance_enter = False

pyxform/parsing/parameters.py

@@ -3,7 +3,7 @@
 
 from pyxform import constants as co
 from pyxform.errors import ErrorCode, PyXFormError
-from pyxform.parsing.expression import maybe_strip
+from pyxform.parsing.utils import maybe_strip
 
 # Parameters for which the value should not be lower-cased.
 CASE_SENSITIVE_VALUES = {

pyxform/parsing/sheet_headers.py

@@ -5,7 +5,7 @@
 
 from pyxform import constants
 from pyxform.errors import ErrorCode, PyXFormError
-from pyxform.parsing.expression import maybe_strip
+from pyxform.parsing.utils import maybe_strip
 from pyxform.xls2json_backends import RE_WHITESPACE
 
 SMART_QUOTES = {"\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"'}

pyxform/parsing/utils.py

@@ -0,0 +1,15 @@
+from typing import Any
+
+
+def maybe_strip(value: Any) -> Any:
+    """
+    If the value is a string and looks like it has whitespace at either end, strip it.
+
+    If a string was "interned" (cached) by Python, string.strip() should generally return
+    the existing string if no leading/trailing whitespace was found. But strings may or
+    may not be interned by Python, and there may be a large cache for many unique values
+    (which is likely for XLSForms), so this function tries to avoid calling strip().
+    """
+    if isinstance(value, str) and value and (value[0].isspace() or value[-1].isspace()):
+        return value.strip()
+    return value