From f8c7b4460ef7c9e90f40f1f39aefc8cb616f6150 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Fri, 7 Aug 2026 15:35:26 +1000 Subject: [PATCH 1/8] fix: typo backslash in expression lark grammar - Does not appear to effect the grammar behaviour since the test cases in tests/parsing/test_expression.py identify these terminals (e.g. SELF_REF) but the backslash is still a typo / unnecessary. - A backslash is automatically added when pressing enter inside a multi-line f-string in pycharm so that's probably how it was added. --- pyxform/parsing/expression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyxform/parsing/expression.py b/pyxform/parsing/expression.py index 9ce27f87..d66e1548 100644 --- a/pyxform/parsing/expression.py +++ b/pyxform/parsing/expression.py @@ -74,7 +74,7 @@ CLOSE_PAREN.16: /\)/ BRACKET.15: /[\[\{{\}}]/ PARENT_REF.14: /\.\./ - SELF_REF.13: /\./\ + SELF_REF.13: /\./ // # javarosa.xpath says "//" is an "unsupported construct". PATH_SEP.12: /\// SYSTEM_LITERAL.11: /"[^"]*"|'[^']*'/ @@ -88,9 +88,9 @@ // Must be lower priority than rules containing ncname_regex. NAME.3: /{ncname_regex_ns}/ PYXFORM_REF_START.2: /\$\{{/ - PYXFORM_REF_END.1: /\}}/\ + PYXFORM_REF_END.1: /\}}/ // Catch any other character so that parsing doesn't stop. - OTHER.0: /.+?/\ + OTHER.0: /.+?/ """ RE_NCNAME_NAMESPACED = re.compile(ncname_regex_ns_named) From ceb60eb94251ca992f2bd8135c106686b8a1523a Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 23 Sep 2026 14:34:09 +1000 Subject: [PATCH 2/8] chg: move `maybe_strip` func to utils module - not related to expression.py specifically --- pyxform/parsing/expression.py | 16 +--------------- pyxform/parsing/parameters.py | 2 +- pyxform/parsing/sheet_headers.py | 2 +- pyxform/parsing/utils.py | 15 +++++++++++++++ pyxform/question.py | 2 +- .../pyxform/iana_subtags/validation.py | 2 +- 6 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 pyxform/parsing/utils.py diff --git a/pyxform/parsing/expression.py b/pyxform/parsing/expression.py index d66e1548..6a1b40ea 100644 --- a/pyxform/parsing/expression.py +++ b/pyxform/parsing/expression.py @@ -119,18 +119,4 @@ def parse_expression(text: str) -> tuple[Token, ...]: def is_xml_tag(value: str) -> bool: """Check if the input string contains only a valid XML tag / element name.""" - return value and bool(RE_NCNAME_NAMESPACED.fullmatch(value)) - - -def maybe_strip(value: Any) -> Any: - """ - If the value is a string and looks like it has whitespace at either end, strip it. - - If a string was "interned" (cached) by Python, string.strip() should generally return - the existing string if no leading/trailing whitespace was found. But strings may or - may not be interned by Python, and there may be a large cache for many unique values - (which is likely for XLSForms), so this function tries to avoid calling strip(). - """ - if isinstance(value, str) and value and (value[0].isspace() or value[-1].isspace()): - return value.strip() - return value + return bool(value) and bool(RE_NCNAME_NAMESPACED.fullmatch(value)) diff --git a/pyxform/parsing/parameters.py b/pyxform/parsing/parameters.py index 31978a73..78cc7435 100644 --- a/pyxform/parsing/parameters.py +++ b/pyxform/parsing/parameters.py @@ -3,7 +3,7 @@ from pyxform import constants as co from pyxform.errors import ErrorCode, PyXFormError -from pyxform.parsing.expression import maybe_strip +from pyxform.parsing.utils import maybe_strip # Parameters for which the value should not be lower-cased. CASE_SENSITIVE_VALUES = { diff --git a/pyxform/parsing/sheet_headers.py b/pyxform/parsing/sheet_headers.py index 4242d7e3..900af65b 100644 --- a/pyxform/parsing/sheet_headers.py +++ b/pyxform/parsing/sheet_headers.py @@ -5,7 +5,7 @@ from pyxform import constants from pyxform.errors import ErrorCode, PyXFormError -from pyxform.parsing.expression import maybe_strip +from pyxform.parsing.utils import maybe_strip from pyxform.xls2json_backends import RE_WHITESPACE SMART_QUOTES = {"\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"'} diff --git a/pyxform/parsing/utils.py b/pyxform/parsing/utils.py new file mode 100644 index 00000000..70208ebc --- /dev/null +++ b/pyxform/parsing/utils.py @@ -0,0 +1,15 @@ +from typing import Any + + +def maybe_strip(value: Any) -> Any: + """ + If the value is a string and looks like it has whitespace at either end, strip it. + + If a string was "interned" (cached) by Python, string.strip() should generally return + the existing string if no leading/trailing whitespace was found. But strings may or + may not be interned by Python, and there may be a large cache for many unique values + (which is likely for XLSForms), so this function tries to avoid calling strip(). + """ + if isinstance(value, str) and value and (value[0].isspace() or value[-1].isspace()): + return value.strip() + return value diff --git a/pyxform/question.py b/pyxform/question.py index cd00b831..6ca4c86b 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -15,7 +15,7 @@ ) from pyxform.elements import action from pyxform.errors import PyXFormError -from pyxform.parsing.expression import maybe_strip +from pyxform.parsing.utils import maybe_strip from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement from pyxform.utils import ( diff --git a/pyxform/validators/pyxform/iana_subtags/validation.py b/pyxform/validators/pyxform/iana_subtags/validation.py index a968dd56..da4019d2 100644 --- a/pyxform/validators/pyxform/iana_subtags/validation.py +++ b/pyxform/validators/pyxform/iana_subtags/validation.py @@ -2,7 +2,7 @@ from functools import lru_cache from pathlib import Path -from pyxform.parsing.expression import maybe_strip +from pyxform.parsing.utils import maybe_strip LANG_CODE_REGEX = re.compile(r"\((.*)\)$") HERE = Path(__file__).parent From 35cb43d1488c6fc012630f5e42c30e72bfdfc1db Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 23 Sep 2026 19:00:46 +1000 Subject: [PATCH 3/8] chg: rename `parse_expression` to `lex_expression` since it only lexes - the actual parsing happens imperatively in other modules based on the tokens from this function, such as in instance_expression.py --- pyxform/parsing/expression.py | 7 +++---- pyxform/parsing/instance_expression.py | 4 ++-- pyxform/utils.py | 4 ++-- tests/parsing/test_expression.py | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pyxform/parsing/expression.py b/pyxform/parsing/expression.py index 6a1b40ea..bc144aaf 100644 --- a/pyxform/parsing/expression.py +++ b/pyxform/parsing/expression.py @@ -1,6 +1,5 @@ import re from functools import lru_cache -from typing import Any from lark import Lark, Token @@ -105,14 +104,14 @@ @lru_cache(maxsize=128) -def parse_expression(text: str) -> tuple[Token, ...]: +def lex_expression(text: str) -> tuple[Token, ...]: """ - Parse an expression. + Lex an expression. Use this function instead of _EXPRESSION_LEXER to take advantage of caching. :param text: The expression. - :return: The parsed tokens, and any remaining unparsed text. + :return: The lexed tokens, and any remaining text. """ return tuple(_EXPRESSION_LEXER.lex(text)) diff --git a/pyxform/parsing/instance_expression.py b/pyxform/parsing/instance_expression.py index f7cfc861..c9637952 100644 --- a/pyxform/parsing/instance_expression.py +++ b/pyxform/parsing/instance_expression.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from pyxform.parsing.expression import RE_PYXFORM_REF, parse_expression +from pyxform.parsing.expression import RE_PYXFORM_REF, lex_expression from pyxform.utils import node if TYPE_CHECKING: @@ -21,7 +21,7 @@ def find_boundaries(xml_text: str) -> list[tuple[int, int]]: :param xml_text: XML text that may contain an instance expression. :return: Tokens in instance expression, and the string position boundaries. """ - tokens = parse_expression(xml_text) + tokens = lex_expression(xml_text) if not tokens: return [] instance_enter = False diff --git a/pyxform/utils.py b/pyxform/utils.py index 6cc73071..8b4fa70f 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -16,7 +16,7 @@ from pyxform import constants as const from pyxform.errors import PyXFormError -from pyxform.parsing.expression import parse_expression +from pyxform.parsing.expression import lex_expression from pyxform.xls2json_backends import DefinitionData LAST_SAVED_INSTANCE_NAME = "__last-saved" @@ -226,7 +226,7 @@ def default_is_dynamic(element_default, element_type=None): if not element_default or not isinstance(element_default, str): return False - tokens = parse_expression(element_default) + tokens = lex_expression(element_default) for t in tokens: # Data types which are likely to have non-dynamic defaults containing a hyphen. if element_type in {"date", "dateTime", "geopoint", "geotrace", "geoshape"}: diff --git a/tests/parsing/test_expression.py b/tests/parsing/test_expression.py index 59dc646c..d09f5d5c 100644 --- a/tests/parsing/test_expression.py +++ b/tests/parsing/test_expression.py @@ -1,6 +1,6 @@ from enum import Enum -from pyxform.parsing.expression import is_xml_tag, parse_expression +from pyxform.parsing.expression import is_xml_tag, lex_expression from tests.fixtures.lexer_cases import LexerCases from tests.pyxform_test_case import PyxformTestCase @@ -402,11 +402,11 @@ def test_is_xml_tag__negative(self): with self.subTest(case=case, description=description): self.assertFalse(is_xml_tag(case)) - def test_parse_expression(self): + def test_lex_expression(self): """Should find expected sequence of token types for each input.""" for lexer_case, token_types in (i.value for i in ExpectedTokens): description, case = lexer_case.value - with self.subTest(case=case, description=description): + with self.subTest(name=lexer_case.name, case=case, description=description): self.assertEqual( - token_types, tuple(t.type for t in parse_expression(text=case)) + token_types, tuple(t.type for t in lex_expression(text=case)) ) From 79141424b4b0894919d768c0f1d5fa3c7ec41ee6 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 23 Sep 2026 20:11:48 +1000 Subject: [PATCH 4/8] chg: move _setup_xpath_dictionary to group up post-init in `validate()` - the _setup_xpath_dictionary function indexes all child SurveyElements into one list, and thereby validates that names are not re-used. - this is a validation task and the result could be useful to other functions prior to calling the final `xml()` output steps function. --- pyxform/survey.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyxform/survey.py b/pyxform/survey.py index ccd00258..f4e47e5d 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -243,6 +243,7 @@ def validate(self): raise PyXFormError("Survey cannot have an empty id_string") super().validate() self._validate_uniqueness_of_section_names() + self._setup_xpath_dictionary() def _validate_uniqueness_of_section_names(self): root_node_name = self.name @@ -287,7 +288,6 @@ def get_nsmap(self): def xml(self): """Calls necessary preparation methods, then returns the xml.""" self.validate() - self._setup_xpath_dictionary() body_kwargs = {} if self.style: @@ -983,7 +983,7 @@ def _to_pretty_xml(self) -> str: return f"""\n{self.xml().toprettyxml(indent=" ")}""" def _setup_xpath_dictionary(self): - if self._xpath: + if self._xpath is not None: return xpaths = {} for element in self.iter_descendants(lambda i: isinstance(i, Question | Section)): From fc19c5efb38bd3f9018d476efaeed8040d665e8b Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 23 Sep 2026 20:15:33 +1000 Subject: [PATCH 5/8] add: delimited expression parsing - accept an expression wrapped in double curly braces as a string delimiter to wrap in an `output` element. - allows wider range of expressions than was possible with previous detection of `instance()` calls or ${} variable references. - move the logic for variable resolution and fallback parsing over to the new func, to simplify survey.py and testing - remove maybe_strip of space chars in expression in order to match old behaviour of variable resolution - the simpler thing seems to be to not add spaces in `Survey._var_repl_function` but that may have other consequences, and it breaks a lot of tests, so it is left for a later changeset. - add test cases for delimited expressions in various unit scenarios - test usage works in survey/choices translatable text/media columns. - test escaped delims in text, instance() fallback behaviour --- pyxform/errors.py | 11 + pyxform/parsing/expression_delimited.py | 171 +++ pyxform/survey.py | 17 +- tests/parsing/test_expression_delimited.py | 1517 ++++++++++++++++++++ tests/test_notes.py | 50 +- tests/xpath_helpers/choices.py | 20 + tests/xpath_helpers/questions.py | 32 + 7 files changed, 1801 insertions(+), 17 deletions(-) create mode 100644 pyxform/parsing/expression_delimited.py create mode 100644 tests/parsing/test_expression_delimited.py diff --git a/pyxform/errors.py b/pyxform/errors.py index 65818189..514ac88b 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -246,6 +246,17 @@ class ErrorCode(Enum): "path '{path}'." ), ) + INTERNAL_003 = Detail( + name="Internal error - expression parsing failed", + msg="Internal error: expression parsing failed. Input string: {s}", + ) + INTERNAL_004 = Detail( + name="Internal error - expression parsing failed", + msg=( + "Internal error: expression parsing failed. " + "Nested expressions are not allowed. Input string: {s}" + ), + ) LABEL_001: Detail = Detail( name="Labels - invalid missing label in the choices sheet", msg=( diff --git a/pyxform/parsing/expression_delimited.py b/pyxform/parsing/expression_delimited.py new file mode 100644 index 00000000..80afa7a1 --- /dev/null +++ b/pyxform/parsing/expression_delimited.py @@ -0,0 +1,171 @@ +from typing import TYPE_CHECKING + +from lark import Lark, LarkError, Transformer, Tree + +from pyxform.errors import ErrorCode, PyXFormError +from pyxform.parsing.expression import RE_PYXFORM_REF +from pyxform.parsing.instance_expression import ( + replace_with_output as replace_with_output_old, +) +from pyxform.utils import node +from pyxform.validators.pyxform.pyxform_reference import is_pyxform_reference_candidate + +if TYPE_CHECKING: + from lark import Token + + from pyxform.survey import Survey + from pyxform.survey_element import SurveyElement + + +expression_grammar = r""" +start: (expression | escaped | text)* + +expression: _OPEN (invalid_nesting | escaped | TEXT_NON_DELIMITER | CHAR_NON_DELIMITER)* _CLOSE +invalid_nesting: expression +escaped: ESCAPED_DELIMITER +text: (TEXT_NON_DELIMITER | CHAR_NON_DELIMITER)+ + +_OPEN: "{{" +_CLOSE: /}\}(?!})/ +ESCAPED_DELIMITER: "\\{{" | "\\}}" +TEXT_NON_DELIMITER: /[^{}\\]+/ +CHAR_NON_DELIMITER: /[{}\\]/ +""" + + +_EXPRESSION_PARSER = Lark( + expression_grammar, parser="lalr", lexer="contextual", cache=True +) + + +def resolve_variables( + xml_text: str, + survey: "Survey", + context: "SurveyElement | None" = None, + wrap: bool = False, +): + """ + Replace ${} variables with XPath references, optionally wrapped in . + + :param xml_text: Input string to process. + :param survey: Survey object for reference lookup. + :param context: The document node that the text belongs to. + :param wrap: If True, enclose resolved references in tags. + """ + + def resolver(match): + replaced = survey._var_repl_function(match, context) + if wrap: + return node("output", value=replaced).toxml() + else: + return replaced + + if is_pyxform_reference_candidate(value=xml_text): + return RE_PYXFORM_REF.sub(resolver, xml_text) + return xml_text + + +class ExpressionTransformer(Transformer): + def __init__( + self, context: "SurveyElement", survey: "Survey", visit_tokens: bool = True + ): + super().__init__(visit_tokens=visit_tokens) + self.context: SurveyElement = context + self.survey: Survey = survey + self.expression_modified: bool = False + + @staticmethod + def _flatten(children: list["Token"]) -> str: + """Combine leaf child strings into one string.""" + if len(children) == 0: + return "" + elif len(children) > 1: + return "".join(children) + else: + return str(children[0]) + + def start(self, children: list["Token"]) -> str: + """Join all children strings.""" + return self._flatten(children=children) + + def text(self, children: list["Token"]) -> str: + """Join all children strings.""" + original = self._flatten(children=children) + resolved = resolve_variables( + xml_text=original, survey=self.survey, context=self.context, wrap=True + ) + if original != resolved: + return resolved + else: + return original + + def escaped(self, children: list["Token"]) -> str: + r"""Drop the leading backslash: `'\{{' -> '{{'`, `'\}}' -> '}}'`.""" + original = children[0] + resolved = {r"\{{": "{{", r"\}}": "}}"}[original] + if original != resolved: + self.expression_modified = True + return resolved + else: + return original + + def invalid_nesting(self, children: list["Token"]) -> str: + """Raise an error if a nested expression is found.""" + raise PyXFormError(code=ErrorCode.INTERNAL_004) + + def expression(self, children: list["Token"]) -> str: + """Join/wrap children in XML: ``.""" + resolved = resolve_variables( + xml_text=self._flatten(children=children), + survey=self.survey, + context=self.context, + ) + self.expression_modified = True + return node("output", value=resolved).toxml() + + +def replace_with_output(xml_text: str, context: "SurveyElement", survey: "Survey") -> str: + """ + Wrap expressions in output element: `{{contents}} -> `. + + Resolves variable references inside the expression, but not outside, so an extra pass + over the string is required. + + :param xml_text: Text that may contain an expression. + :param context: The element context that variable references should be resolved in, + for example to determine relative references. + :param survey: The parent Survey object. + """ + # Minimum length token that could be changed. + if len(xml_text) < 3: + return xml_text + + try: + if "{" in xml_text or "}" in xml_text: + transformer = ExpressionTransformer(survey=survey, context=context) + out_str = transformer.transform(_EXPRESSION_PARSER.parse(xml_text)) + if transformer.expression_modified: + return out_str + + # Fallback for old func that only looks for non-delimited `instance()`. + fallback = replace_with_output_old(xml_text, context, survey) + if is_pyxform_reference_candidate(fallback): + fallback = resolve_variables( + xml_text=fallback, survey=survey, context=context, wrap=True + ) + if fallback != xml_text: + return fallback + else: + return xml_text + except LarkError as e: + if isinstance(e.__context__, PyXFormError): + ctx = e.__context__ + if ctx.code: + ctx.context["s"] = xml_text + raise PyXFormError(code=ctx.code, context=ctx.context) from e + else: + raise PyXFormError(ctx.args[0]) from e + else: + raise PyXFormError( + code=ErrorCode.INTERNAL_003, context={"s": xml_text} + ) from e diff --git a/pyxform/survey.py b/pyxform/survey.py index f4e47e5d..0c2b3dc9 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -16,7 +16,7 @@ from pyxform.external_instance import ExternalInstance from pyxform.instance import SurveyInstance from pyxform.parsing.expression import RE_PYXFORM_REF -from pyxform.parsing.instance_expression import replace_with_output +from pyxform.parsing.expression_delimited import replace_with_output from pyxform.question import Itemset, MultipleChoiceQuestion, Option, Question, Tag from pyxform.section import SECTION_EXTRA_FIELDS, RepeatingSection, Section from pyxform.survey_element import _GET_SENTINEL, SURVEY_ELEMENT_FIELDS, SurveyElement @@ -1159,13 +1159,6 @@ def _var_repl_function(matchobj): return re.sub(RE_PYXFORM_REF, _var_repl_function, value) - def _var_repl_output_function(self, matchobj, context): - """ - A regex substitution function that will replace - ${varname} with an output element that has the xpath to varname. - """ - return f"""""" - def insert_output_values( self, text: str, @@ -1183,9 +1176,6 @@ def insert_output_values( if text == "-": return text, False - def _var_repl_output_function(matchobj): - return self._var_repl_output_function(matchobj, context) - # There was a bug where escaping is completely turned off in labels # where variable replacement is used. # For exampke, `${name} < 3` causes an error but `< 3` does not. @@ -1193,12 +1183,7 @@ def _var_repl_output_function(matchobj): # variable replacement: original_xml = escape_text_for_xml(text=text) - # need to make sure we have reason to replace - # since at this point < is <, - # the net effect < gets translated again to &lt; value = replace_with_output(original_xml, context, self) - if is_pyxform_reference_candidate(value): - value = re.sub(RE_PYXFORM_REF, _var_repl_output_function, value) changed = value != original_xml if changed: return value, True diff --git a/tests/parsing/test_expression_delimited.py b/tests/parsing/test_expression_delimited.py new file mode 100644 index 00000000..579eb8a4 --- /dev/null +++ b/tests/parsing/test_expression_delimited.py @@ -0,0 +1,1517 @@ +""" +## Delimited expression traceability. + +Each test should reference one (or more) requirements from these lists. + +- feature delimited expressions: + - Validation + - EDV01: an opening expression delimiter must have a matching closing delimiter. + - EDV02: expressions must not be nested. + - Behaviour + - EDB01: delimited expression contents are wrapped in an XML element. + - EDB02: whitespace in expression contents is preserved. + - EDB03: text containing an expression is XML-escaped. + - EDB04: escaped expression delimiters outside of an expression are unescaped. + - EDB05: escaped expression delimiters inside of an expression are preserved. + - EDB06: text containing an expression may contain multiple expressions. + - EDB08: variable references inside an expression string are resolved to a XPath. + - EDB10: variable references outside an expression are resolved and wrapped in an XML element. + - EDB11: text strings shorter than 3 characters is returned unchanged. + - EDB12: text without an (escaped) expression delimiter may be returned unchanged. + - EDB13: text without an (escaped) expression may fallback to old `instance()` processing. + - EDB14: backslashes and partial delimiter/escape sequences are preserved as text. + - EDB15: expressions in survey translatable text/media columns are processed. + - EDB16: expressions in choice translatable text/media are processed. +""" + +from unittest import expectedFailure + +from pyxform.errors import ErrorCode, PyXFormError +from pyxform.parsing.expression_delimited import replace_with_output +from pyxform.question import InputQuestion +from pyxform.survey import Survey + +from tests.pyxform_test_case import PyxformTestCase +from tests.xpath_helpers.choices import xpc +from tests.xpath_helpers.questions import xpq + +SURVEY = Survey(name="test_name") +Q1 = InputQuestion(name="q1", type="string") +Q2 = InputQuestion(name="q2", type="string") +PROXIMO = InputQuestion(name="próxiMO", type="string") +ELEMENT = InputQuestion(name="élémeNT", type="string") +SURVEY.add_children((Q1, Q2, PROXIMO, ELEMENT)) +SURVEY.validate() + +MD_SURVEY = """ +| survey | +| | type | name | label | +| | text | q1 | {label} | +| | text | q2 | Q2 | +""" +MD_SURVEY_CHOICES = f""" +{MD_SURVEY} + +| choices | +| | list_name | name | label | +| | c1 | n1 | N1 | +""" + + +class TestParsingErrors(PyxformTestCase): + """Should raise an error when malformed expression template syntax is found.""" + + def test_unclosed_template_with_trailing_content(self): + """Unclosed template with trailing content.""" + # EDV01 + inp = "Total cost: {{ ${q1} * ${q2}" + with self.assertRaises(PyXFormError): + replace_with_output(inp, Q1, SURVEY) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + ) + + def test_valid_template_followed_by_unclosed_template(self): + """Valid template followed by unclosed template.""" + # EDV01 + inp = "User {{ ${q1} }} has pending {{ ${q2}" + with self.assertRaises(PyXFormError): + replace_with_output(inp, Q1, SURVEY) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + ) + + def test_unclosed_expression_following_escaped_delimiter(self): + """Unclosed template following escaped delimiter.""" + # EDV01 + inp = r"\{{ escaped \}} but {{ unclosed template" + with self.assertRaises(PyXFormError): + replace_with_output(inp, Q1, SURVEY) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + ) + + def test_repeated_delimiters(self): + """Doubled up delimiters inside expression.""" + # EDV01 + inp = "Value: {{{{ ${q1} }}}}" + with self.assertRaises(PyXFormError): + replace_with_output(inp, Q1, SURVEY) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + ) + + def test_nested_delimiters_inside_expression(self): + """Nested delimiters inside expression.""" + # EDV02 + inp = "Value: {{ {{ ${q1} }} }}" + with self.assertRaises(PyXFormError): + replace_with_output(inp, Q1, SURVEY) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_004.value.format(s=inp)], + ) + + +class TestOutputText(PyxformTestCase): + """Should return unchanged text when no expression template is found.""" + + def test_empty_string(self): + """Should accept an empty string.""" + # EDB11 EDB12 + inp = "" + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + # Label required, so XPath assertion not testable. + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=["The survey element named 'q1' has no label or hint."], + ) + + def test_plain_text(self): + """Should accept plain text.""" + # EDB12 + inp = "Welcome to the survey!" + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + def test_multiline_text(self): + """Should accept multiline text.""" + # EDB12 + inp = """Line 1\n Line 2\twith tabs""" + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + ss_structure = { + "survey_header": [{"type": None, "name": None, "label": None}], + "survey": [ + {"type": "text", "name": "q1", "label": inp}, + {"type": "text", "name": "q2", "label": "Q2"}, + ], + } + self.assertPyxformXform( + ss_structure=ss_structure, + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {"Line 1\n Line 2\twith tabs"}) + ], + ) + + def test_variable_references(self): + """Should accept variable references.""" + # EDB10 + inp = "Hello ${q1}, your score is ${q2}." + expected = """Hello , your score is .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Hello ", ", your score is ", ". "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 ", " /test_name/q2 "}, + ), + ], + ) + + def test_single_delimiter_tokens(self): + """Should ignore single delimiter tokens.""" + # EDB12 EDB14 + inp = "Item costs $10 or {5} credits." + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + def test_unclosed_opening_delimiter(self): + """Should ignore unclosed opening delimiter.""" + # EDB11 EDB12 EDB14 + inp = "{{" + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + def test_unmatched_closing_delimiter(self): + """Should ignore unmatched closing delimiter.""" + # EDB12 EDB14 + inp = "Unmatched closing delimiter }} kept as text." + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + def test_file_path_or_single_backslash(self): + """Should ignore file paths or single backslash.""" + # EDB12 EDB14 + inp = r"C:\Users\test\file.txt \ " + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp.strip()})], + ) + + def test_partial_escape_sequence(self): + """Should ignore partial escape sequences.""" + # EDB12 EDB14 + inp = r"\{ escaped \} but \\" + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + +class TestOutputBasicExpressions(PyxformTestCase): + """Should replace {{ expr }} content wrapped in an element.""" + + def test_no_surrounding_text(self): + """Should accept an expression with no surrounding text.""" + # EDB01 EDB02 + inp = "{{ 1 + 1 }}" + expected = '' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_output_value("q1"), {" 1 + 1 "})], + ) + + def test_adjacent_templates(self): + """Should accept adjacent templates.""" + # EDB01 EDB02 EDB06 + inp = "{{ 1 + 1 }}{{ 2 + 2 }}" + expected = '' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_output_value("q1"), + {" 1 + 1 ", " 2 + 2 "}, + ) + ], + ) + + def test_empty_template(self): + """Should accept an empty template.""" + # EDB01 + inp = "Empty: {{}}" + expected = 'Empty: ' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Empty: ", " "}), + (xpq.body_input_label_output_value("q1"), {""}), + ], + ) + + def test_no_inner_spaces(self): + """Should accept expressions with no spaces inside the delimiters.""" + # EDB01 + inp = "Value: {{today()}}" + expected = 'Value: ' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Value: ", " "}), + (xpq.body_input_label_output_value("q1"), {"today()"}), + ], + ) + + def test_inner_spaces_not_stripped(self): + """Should accept but not strip spaces inside expression delimiters.""" + # EDB01 EDB02 + inp = "Value: {{ count(/data/item) }}" + expected = 'Value: ' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Value: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" count(/data/item) "}, + ), + ], + ) + + def test_expression_xpath_preserved(self): + """Should accept and preserve XPath expressions.""" + # EDB01 + inp = "Selected item: {{ instance('cities')/root/item[id = 1]/name }}" + expected = """Selected item: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Selected item: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" instance('cities')/root/item[id = 1]/name "}, + ), + ], + ) + + def test_multiline_expression_with_internal_whitespace(self): + """Should accept multiline expression with internal whitespace.""" + # EDB01 EDB02 EDB03 EDB08 + inp = "Result: {{\n if(\n ${q1} >= 18,\n 'Adult',\n 'Minor'\n )\n}}" + expected = """Result: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + ss_structure = { + "survey_header": [{"type": None, "name": None, "label": None}], + "survey": [ + {"type": "text", "name": "q1", "label": inp}, + {"type": "text", "name": "q2", "label": "Q2"}, + ], + } + self.assertPyxformXform( + ss_structure=ss_structure, + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Result: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" if( /test_name/q1 >= 18, 'Adult', 'Minor' ) "}, + ), + ], + ) + + +class TestOutputVariableResolution(PyxformTestCase): + """Should resolve ${} variable references found inside the expression template.""" + + def test_single_variable(self): + """Should accept single variable reference.""" + # EDB01 EDB08 + inp = "Value: {{${q1}}}" + expected = """Value: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Value: ", " "}), + (xpq.body_input_label_output_value("q1"), {" /test_name/q1 "}), + ], + ) + + def test_repeated_variable(self): + """Should accept repeated variable references.""" + # EDB01 EDB02 EDB08 + inp = "Double: {{ ${q1} + ${q1} }}" + expected = """Double: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Double: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 + /test_name/q1 "}, + ), + ], + ) + + def test_multiple_different_variables(self): + """Should accept multiple different variable references.""" + # EDB01 EDB02 EDB08 + inp = "Total cost: {{ ${q1} * ${q2} }} USD" + expected = ( + """Total cost: USD""" + ) + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Total cost: ", " USD "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 * /test_name/q2 "}, + ), + ], + ) + + def test_variable_inside_and_outside(self): + """Should accept variable references inside and outside expression.""" + # EDB01 EDB02 EDB08 EDB10 + inp = "User ${q1}, your total is {{ ${q1} * 2 }}." + expected = """User , your total is .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" User ", ", your total is ", ". "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 ", " /test_name/q1 * 2 "}, + ), + ], + ) + + def test_unicode_ncname_variable_references(self): + """Should accept variable references with a unicode NCName.""" + # EDB01 EDB02 EDB08 EDB10 + inp = "Hola ${próxiMO}, total: {{ ${élémeNT} + 1 }}" + expected = """Hola , total: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + md_extra = """ + | | text | próxiMO | PróxiMO | + | | text | élémeNT | ÉlémeNT | + """ + self.assertPyxformXform( + md=(MD_SURVEY + md_extra).format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Hola ", ", total: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/próxiMO ", " /test_name/élémeNT + 1 "}, + ), + ], + ) + + def test_last_saved_variables(self): + """Should accept last-saved variable references.""" + # EDB01 EDB02 EDB08 EDB10 + inp = "Previous: ${last-saved#q1}, Calculated: {{ ${last-saved#q2} * 1.1 }}" + expected = """Previous: , Calculated: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Previous: ", ", Calculated: ", " "}, + ), + ( + xpq.body_input_label_output_value("q1"), + { + " instance('__last-saved')/test_name/q1 ", + " instance('__last-saved')/test_name/q2 * 1.1 ", + }, + ), + ], + ) + + def test_adjacent_variables(self): + """Should accept adjacent variable references.""" + # EDB01 EDB02 EDB08 EDB10 + inp = "${q1}{{ ${q1} * ${q2} }}${q2}" + expected = """""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {"\n ", "\n "}), + ( + xpq.body_input_label_output_value("q1"), + { + " /test_name/q1 ", + " /test_name/q1 * /test_name/q2 ", + " /test_name/q2 ", + }, + ), + ], + ) + + +class TestOutputXpathExpressions(PyxformTestCase): + """Should replace {{ expr }} content wrapped in an element.""" + + def test_function_with_variable(self): + """Should accept function with variable references.""" + # EDB01 EDB02 EDB08 + inp = "Welcome {{ coalesce(upper-case(${q1}), 'Guest') }}!" + expected = """Welcome !""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Welcome ", "! "}), + ( + xpq.body_input_label_output_value("q1"), + {" coalesce(upper-case( /test_name/q1 ), 'Guest') "}, + ), + ], + ) + + def test_operator_escaping(self): + """Should escape XML characters.""" + # EDB01 EDB02 EDB03 EDB08 + inp = "Valid: {{ ${q1} < 10 and ${q2} <= 20 }}" + expected = """Valid: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Valid: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 < 10 and /test_name/q2 <= 20 "}, + ), + ], + ) + + def test_literal_ampersand_escaping(self): + """Should escape XML characters.""" + # EDB01 EDB02 EDB03 EDB08 + inp = "Menu: {{ concat('Fish & Chips: ', ${q1}) }}" + expected = ( + """Menu: """ + ) + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Menu: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat('Fish & Chips: ', /test_name/q1 ) "}, + ), + ], + ) + + def test_secondary_instance_lookup_with_predicate_filtering(self): + """Should accept secondary instance lookups with predicate filtering.""" + # EDB01 EDB02 EDB08 + inp = "Facility: {{ instance('facilities')/root/item[id=${q1}]/label }}" + expected = """Facility: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Facility: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" instance('facilities')/root/item[id= /test_name/q1 ]/label "}, + ), + ], + ) + + def test_complex_nested_functions_with_operators_and_variable_references(self): + """Should accept complex nested functions with operators and variable references.""" + # EDB01 EDB02 EDB03 EDB08 + inp = "Score: {{ if(${q1} >= 80, concat('PASS: ', round(${q1} * 1.1, 2), '%'), if(${q1} >= 50, 'RETRY', 'FAIL')) }}" + expected = """Score: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Score: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + { + """ if( /test_name/q1 >= 80, concat('PASS: ', round( /test_name/q1 * 1.1, 2), '%'), if( /test_name/q1 >= 50, 'RETRY', 'FAIL')) """ + }, + ), + ], + ) + + def test_date_formatting_string(self): + """Should accept and preserve date formatting specifiers.""" + # EDB01 EDB02 EDB08 + inp = "Visit scheduled for {{ format-date(${q1}, '%A, %b %e, %Y') }}." + expected = """Visit scheduled for .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Visit scheduled for ", ". "}), + ( + xpq.body_input_label_output_value("q1"), + {" format-date( /test_name/q1 , '%A, %b %e, %Y') "}, + ), + ], + ) + + def test_indexed_repeat_lookup(self): + """Should accept indexed repeat lookups.""" + # EDB01 EDB02 EDB08 + inp = "Primary Contact: {{ indexed-repeat(${q1}, ${q2}, 1) }}" + expected = """Primary Contact: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Primary Contact: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" indexed-repeat( /test_name/q1 , /test_name/q2 , 1) "}, + ), + ], + ) + + def test_multiple_expressions(self): + """Should accept multiple expressions and emit each in output elements.""" + # EDB01 EDB02 EDB06 EDB08 + # pyxform/#844 alternative strategy with new feature. + inp = """Thing's label: {{instance('c1')/root/item[name=${q1} or true()]/label}} Thing's denomination: {{instance('c1')/root/item[name=${q1}]/denomination}} and more stuff.""" + expected = """Thing's label: Thing's denomination: and more stuff.""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + { + " Thing's label: ", + " Thing's denomination: ", + " and more stuff. ", + }, + ), + ( + xpq.body_input_label_output_value("q1"), + { + "instance('c1')/root/item[name= /test_name/q1 or true()]/label", + "instance('c1')/root/item[name= /test_name/q1 ]/denomination", + }, + ), + ], + ) + + +class TestOutputDelimiterEscaping(PyxformTestCase): + """Should detect and resolve escaped delimiters when an expression template is found.""" + + def test_escaped_delimiters(self): + """Should accept escaped delimiters.""" + # EDB04 EDB10 + inp = r"Literal \{{ ${q1} \}} display." + expected = """Literal {{ }} display.""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Literal {{ ", " }} display. "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 "}, + ), + ], + ) + + def test_escaped_closing_delimiter_outside_expression(self): + """Should accept escaped closing delimiter outside an expression.""" + # EDB04 + inp = r"Hello \}} there" + expected = """Hello }} there""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_escaped_opening_delimiter_outside_expression(self): + """Should accept escaped opening delimiter outside an expression.""" + # EDB04 + inp = r"Hello \{{ there" + expected = r"Hello {{ there" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_escaped_closing_delimiter_inside_expression(self): + """Should accept escaped closing delimiter inside an expression.""" + # EDB01 EDB05 + inp = r"Result: {{ concat('foo', '\}}') }}" + expected = """Result: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Result: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat('foo', '}}') "}, + ), + ], + ) + + def test_escaped_opening_delimiter_inside_expression(self): + """Should accept escaped opening delimiter inside an expression.""" + # EDB01 EDB05 EDB08 + inp = r"Result: {{ concat('\{{', ${q1}) }}" + expected = """Result: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Result: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat('{{', /test_name/q1 ) "}, + ), + ], + ) + + def test_escaped_delimiter_with_expression(self): + """Should accept string with escaped an unescaped delimiters.""" + # EDB01 EDB04 EDB08 EDB10 + inp = r"User ${q1}: \{{ literal \}} evaluates to {{ ${q2} * 100 }}%" + expected = """User : {{ literal }} evaluates to %""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" User ", ": {{ literal }} evaluates to ", "% "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 ", " /test_name/q2 * 100 "}, + ), + ], + ) + + def test_adjacent_escaped_delimiters(self): + """Should ignore adjacent escaped delimiters.""" + # EDB04 + inp = r"\{{\{{\}}\}}" + expected = r"{{{{}}}}" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_double_escaped_delimiters(self): + """Should ignore double-escaped delimiters.""" + # EDB04 + inp = r"\\{{\\}}" + expected = r"\{{\}}" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_escaped_delimiters_multiline(self): + """Should ignore escaped delimiters in multi-line strings.""" + # EDB04 + inp = "Line 1: \\{{\nLine 2: \\}}" + expected = "Line 1: {{\nLine 2: }}" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + ss_structure = { + "survey_header": [{"type": None, "name": None, "label": None}], + "survey": [ + {"type": "text", "name": "q1", "label": inp}, + {"type": "text", "name": "q2", "label": "Q2"}, + ], + } + self.assertPyxformXform( + ss_structure=ss_structure, + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_escaped_delimiters_adjacent_variable_references(self): + """Should ignore escaped delimiters adjacent to variable references.""" + # EDB04 EDB10 + inp = r"${q1}\{{ ${q2} \}}${q1}" + expected = """{{ }}""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {"{{ ", " }}", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q1 ", " /test_name/q2 "}, + ), + ], + ) + + +class TestOutputDelimiterAmbiguity(PyxformTestCase): + """Should preserve single delimiter token sequences.""" + + def test_single_delimiter_tokens_in_expression_in_single_quotes(self): + """Should ignore single delimiter tokens in single quotes in an expression.""" + # EDB01 EDB08 EDB14 + inp = "Output: {{ concat('Val: $', ${q1}, ' {tax incl.}') }}" + expected = """Output: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Output: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat('Val: $', /test_name/q1 , ' {tax incl.}') "}, + ), + ], + ) + + def test_single_delimiter_tokens_in_expression_in_double_quotes(self): + """Should ignore single delimiter tokens in double quotes in an expression.""" + # EDB01 EDB03 EDB08 EDB14 + inp = 'Output: {{ concat("Val: $", ${q1}, " {tax incl.}") }}' + expected = """Output: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Output: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {""" concat("Val: $", /test_name/q1 , " {tax incl.}") """}, + ), + ], + ) + + def test_single_delimiter_tokens_in_expression_in_xpath(self): + """Should ignore single delimiter tokens in XPath in an expression.""" + # EDB01 EDB03 EDB08 EDB14 + inp = "Data: {{ instance('items')/root/item[code = 'x}y' and ${q1} > 0]/name }}" + expected = """Data: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Data: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + { + " instance('items')/root/item[code = 'x}y' and /test_name/q1 > 0]/name " + }, + ), + ], + ) + + def test_triple_delimiter_tokens(self): + """Should ignore extra delimiter tokens and accept outer pair of delimiter tokens.""" + # EDB01 EDB08 EDB14 + inp = "Triple open: {{{ ${q1} }}}" + expected = 'Triple open: ' + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Triple open: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {"{ /test_name/q1 }"}, + ), + ], + ) + + def test_triple_closing_delimiter(self): + """Should ignore extra delimiter token and accept outer closing delimiter.""" + # EDB01 EDB08 EDB14 + inp = "Triple close: {{ ${q1} }}}" + expected = """Triple close: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Triple close: ", " "}), + (xpq.body_input_label_output_value("q1"), {" /test_name/q1 }"}), + ], + ) + + def test_extra_closing_delimiter_near_expression_end(self): + """Should ignore single quote near accepted closing delimiter.""" + # EDB01 EDB08 EDB14 + inp = "Ending brace: {{ concat(${q1}, '}') }}" + expected = """Ending brace: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Ending brace: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat( /test_name/q1 , '}') "}, + ), + ], + ) + + def test_backlash_inside_expression(self): + """Should ignore backslash inside expression not adjacent to delimiter.""" + # EDB01 EDB08 EDB14 + inp = r"Path: {{ concat('C:\temp\node', ${q1}) }}" + expected = ( + r"""Path: """ + ) + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Path: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {r" concat('C:\temp\node', /test_name/q1 ) "}, + ), + ], + ) + + +class TestOutputFallbackInstanceExpression(PyxformTestCase): + """Should attempt to process `instance()` expressions if no expression delimiter found.""" + + def test_ignored_when__delimited_expression(self): + """Should ignore instance() in text adjacent to delimited expression.""" + # EDB01 EDB13 + inp = "Lookup {{instance('c1')/root/item[./text() = 'OK!']}} instance('c1')/root/item[./text() = 'OK!'] ." + expected = """Lookup instance('c1')/root/item[./text() = 'OK!'] .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Lookup ", " instance('c1')/root/item[./text() = 'OK!'] . "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {"instance('c1')/root/item[./text() = 'OK!']"}, + ), + ], + ) + + def test_ignored_when__escaped_opening_delimiter(self): + """Should ignore instance() in text adjacent to escaped expression opening delimiter.""" + # EDB04 EDB13 + inp = r"Lookup \{{ instance('c1')/root/item[./text() = 'OK!'] ." + expected = """Lookup {{ instance('c1')/root/item[./text() = 'OK!'] .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_ignored_when__escaped_closing_delimiter(self): + """Should ignore instance() in text adjacent to escaped expression closing delimiter.""" + # EDB04 EDB13 + inp = r"Lookup \}} instance('c1')/root/item[./text() = 'OK!'] ." + expected = """Lookup }} instance('c1')/root/item[./text() = 'OK!'] .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {expected})], + ) + + def test_instance_expression(self): + """Should process instance() with no other triggers present.""" + # EDB13 + inp = "Lookup instance('c1')/root/item[./text() = 'OK!'] ." + expected = ( + """Lookup .""" + ) + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Lookup ", " . "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {"instance('c1')/root/item[./text() = 'OK!']"}, + ), + ], + ) + + def test_instance_expression_adjacent_to_variable_reference(self): + """Should process instance() with variable reference adjacent to the expression.""" + # EDB10 EDB13 + inp = "Lookup ${q2} instance('c1')/root/item[./text() = 'OK!'] ." + expected = """Lookup .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Lookup ", " ", " . "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {" /test_name/q2 ", "instance('c1')/root/item[./text() = 'OK!']"}, + ), + ], + ) + + def test_instance_expression_containing_variable_reference(self): + """Should process instance() with variable reference inside of the expression.""" + # EDB10 EDB13 + inp = "Lookup instance('c1')/root/item[${q2} = 'OK!'] ." + expected = """Lookup .""" + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[ + ( + xpq.body_input_label_text("q1"), + {" Lookup ", " . "}, + ), + ( + xpq.body_input_label_output_value("q1"), + {"instance('c1')/root/item[ /test_name/q2 = 'OK!']"}, + ), + ], + ) + + def test_no_processing(self): + """Should not modify text with no triggers present.""" + # EDB12 + inp = "Lookup at the sky." + self.assertEqual(inp, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY_CHOICES.format(label=inp), + xml__xpath_exact=[(xpq.body_input_label_text("q1"), {inp})], + ) + + +class TestOutputSupportedColumns(PyxformTestCase): + """ + Should allow expression usage in all translatable text/media columns. + + For the media columns, the `jr://` prefix makes the test cases look strange but there + probably are meaningful use cases for dynamically setting the image name. + """ + + def setUp(self): + # Test expression with a delimited expression, and variable refs inside and out. + self.exp = "Test q1 ${q1}: {{ ${q1} + 1}}" + # Expected contents of the corresponding elements @value attribute. + self.outputs = {" /test_name/q1 ", " /test_name/q1 + 1"} + + def test_survey_label_default(self): + """Should emit the processed label in the body control.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | + | | text | q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Test q1 ", ": ", " "}), + ( + xpq.body_input_label_output_value("q1"), + self.outputs, + ), + ], + ) + + def test_survey_hint_default(self): + """Should emit the processed hint in the body control.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | hint | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + """/h:html/h:body/x:input[@ref='/test_name/q1']/x:hint/text()""", + {" Test q1 ", ": ", " "}, + ), + ( + """/h:html/h:body/x:input[@ref='/test_name/q1']/x:hint/x:output/@value""", + self.outputs, + ), + ], + ) + + def test_survey_label_itext(self): + """Should emit the processed label in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label::English (en) | + | | text | q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "English (en)", "label", "/text()"), + {" Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "English (en)", "label", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_hint_itext(self): + """Should emit the processed hint in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | hint::English (en) | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "English (en)", "hint", "/text()"), + {" Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "English (en)", "hint", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_guidance_hint(self): + """Should emit the processed guidance hint in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | guidance_hint | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "default", "guidance", "/text()"), + {" Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "guidance", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_constraint_message(self): + """Should emit the processed constraint message in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | constraint | constraint_message | + | | text | q1 | Q1 | length(.) > 0 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value( + "q1", "default", "constraint_msg", "/text()" + ), + {" Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "constraint_msg", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_required_message(self): + """Should emit the processed required message in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | required_message | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value( + "q1", "default", "required_msg", "/text()" + ), + {" Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "required_msg", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_image(self): + """Should emit the processed image file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | image | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "default", "image", "/text()"), + {" jr://images/Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "image", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_big_image(self): + """Should emit the processed big-image file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | image | big-image | + | | text | q1 | Q1 | q1.png | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "default", "big-image", "/text()"), + {" jr://images/Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "big-image", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_audio(self): + """Should emit the processed audio file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | audio | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "default", "audio", "/text()"), + {" jr://audio/Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "audio", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_survey_video(self): + """Should emit the processed video file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB15 + md = """ + | survey | + | | type | name | label | video | + | | text | q1 | Q1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpq.model_itext_form_value("q1", "default", "video", "/text()"), + {" jr://video/Test q1 ", ": ", " "}, + ), + ( + xpq.model_itext_form_value( + "q1", "default", "video", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_label_default(self): + """Should emit the processed label in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | label | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "label", "/text()"), + {" Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "label", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_label_itext(self): + """Should emit the processed label in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | label::English (en) | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value( + "c1-0", "English (en)", "label", "/text()" + ), + {" Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "English (en)", "label", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_image(self): + """Should emit the processed image file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | image | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "image", "/text()"), + {" jr://images/Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "image", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_big_image(self): + """Should emit the processed big-image file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | image | big-image | + | | c1 | n1 | n1.png | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "big-image", "/text()"), + {" jr://images/Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "big-image", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_audio(self): + """Should emit the processed audio file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | audio | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "audio", "/text()"), + {" jr://audio/Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "audio", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + def test_choice_video(self): + """Should emit the processed video file name in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | video | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "video", "/text()"), + {" jr://video/Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "video", "/x:output/@value" + ), + self.outputs, + ), + ], + ) + + # Not (yet?) supported: `geometry` is not placed in itext and not translatable. + @expectedFailure + def test_choice_geometry(self): + """Should emit the processed geometry value in the model itext.""" + # EDB01 EDB08 EDB10 EDB16 + md = """ + | survey | + | | type | name | label | + | | text | q1 | Q1 | + + | choices | + | | list_name | name | geometry | + | | c1 | n1 | {exp} | + """ + self.assertPyxformXform( + md=md.format(exp=self.exp), + xml__xpath_exact=[ + ( + xpc.model_itext_form_value("c1-0", "default", "geometry", "/text()"), + {" Test q1 ", ": ", " "}, + ), + ( + xpc.model_itext_form_value( + "c1-0", "default", "geometry", "/x:output/@value" + ), + self.outputs, + ), + ], + ) diff --git a/tests/test_notes.py b/tests/test_notes.py index 7da0570a..15778943 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -1,6 +1,7 @@ """Test the "note" question type.""" from dataclasses import dataclass, field +from unittest import expectedFailure from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.questions import xpq @@ -16,7 +17,7 @@ class Case: class TestNotes(PyxformTestCase): - def test_instance_expression__original_problem_scenario(self): + def test_instance_expression__original_problem_scenario_646(self): """Should produce expected output for scenario similar to pyxform/#646.""" md = """ | survey | | | | @@ -57,6 +58,53 @@ def test_instance_expression__original_problem_scenario(self): ], ) + @expectedFailure + def test_instance_expression__original_problem_scenario_844(self): + """Documentation of pyxform/#844 bug behaviour due to compound predicate/tokens.""" + md = """ + | survey | + | | type | name | label | + | | select_one_from_file c1.csv | q1 | Q1 | + | | note | q2 | Thing's label: instance('c1')/root/item[name=${q1} or true()]/label Thing's denomination: instance('c1')/root/item[name=${q1}]/denomination and more stuff. | + """ + self.assertPyxformXform( + md=md, + xml__xpath_exact=[ + # What should not happen: parsing stops after first `instance()`. + ( + xpq.body_input_label_text("q2"), + { + """ Thing's label: instance('c1')/root/item[name=""", + " or true()]/label Thing's denomination: ", + " stuff. ", + }, + ), + ( + xpq.body_input_label_output_value("q2"), + { + " /test_name/q1 ", + "instance('c1')/root/item[name= /test_name/q1 ]/denomination and more", + }, + ), + # What should happen: both `instance()` usages detected accurately. + ( + xpq.body_input_label_text("q2"), + { + " Thing's label: ", + " Thing's denomination: ", + " and more stuff. ", + }, + ), + ( + xpq.body_input_label_output_value("q2"), + { + "instance('c1')/root/item[name= /test_name/q1 or true()]/label", + "instance('c1')/root/item[name= /test_name/q1 ]/denomination", + }, + ), + ], + ) + def test_instance_expression__permutations(self): """Should produce expected output for various combinations of instance usages.""" md = """ diff --git a/tests/xpath_helpers/choices.py b/tests/xpath_helpers/choices.py index 66bb4ccb..13aec967 100644 --- a/tests/xpath_helpers/choices.py +++ b/tests/xpath_helpers/choices.py @@ -9,6 +9,26 @@ class XPathHelper: """XPath expressions for choices assertions.""" + @staticmethod + def model_itext_form_value(cname: str, lang: str, form: str, suffix: str = "") -> str: + """Model itext contains an alternate form itext for the label or hint.""" + prefix = { + "audio": ("label", True), + "image": ("label", True), + "big-image": ("label", True), + "video": ("label", True), + "label": ("label", False), + "geometry": ("geometry", False), + } + value_predicate = "not(@form)" + if prefix[form][1]: + value_predicate = f"@form='{form}'" + return f""" + /h:html/h:head/x:model/x:itext/x:translation[@lang='{lang}'] + /x:text[@id='{cname}'] + /x:value[{value_predicate}]{suffix} + """ + @staticmethod def model_instance_choices_label(cname: str, choices: tuple[tuple[str, str], ...]): """Model instance has choices elements with name and label.""" diff --git a/tests/xpath_helpers/questions.py b/tests/xpath_helpers/questions.py index d140a512..92e08c1a 100644 --- a/tests/xpath_helpers/questions.py +++ b/tests/xpath_helpers/questions.py @@ -88,6 +88,31 @@ def model_itext_form(q_name: str, lang: str, form: str, fname: str) -> str: /x:value[@form='{form}' and text()='{prefix[form][1]}{fname}'] """ + @staticmethod + def model_itext_form_value( + q_name: str, lang: str, form: str, suffix: str = "" + ) -> str: + """Model itext contains an alternate form itext for the label or hint.""" + prefix = { + "audio": ("label", True), + "image": ("label", True), + "big-image": ("label", True), + "video": ("label", True), + "guidance": ("hint", True), + "label": ("label", False), + "hint": ("hint", False), + "constraint_msg": ("jr:constraintMsg", False), + "required_msg": ("jr:requiredMsg", False), + } + value_predicate = "not(@form)" + if prefix[form][1]: + value_predicate = f"@form='{form}'" + return f""" + /h:html/h:head/x:model/x:itext/x:translation[@lang='{lang}'] + /x:text[@id='/test_name/{q_name}:{prefix[form][0]}'] + /x:value[{value_predicate}]{suffix} + """ + @staticmethod def body_label_inline(q_type: str, q_name: str, q_label: str) -> str: """Body element contains the question label.""" @@ -178,6 +203,13 @@ def body_odk_rank_itemset(q_name: str) -> str: ] """ + @staticmethod + def body_input_label_text(q_name: str) -> str: + """Body has an input with label text.""" + return rf""" + /h:html/h:body/x:input[@ref='/test_name/{q_name}']/x:label/text() + """ + @staticmethod def body_input_label_output_value(q_name: str) -> str: """Body has an input (note) with output reference in the label.""" From 746f186685ef6a3b726a2a7c66a05746eda6ea72 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 30 Sep 2026 15:29:07 +1000 Subject: [PATCH 6/8] chg: reorganise variable references parsing/validation - use 'variable references' name since they're called 'variables' in the docs, rather than 'pyxform references', and it seems distinct enough from expressions to not have to differentiate further. - move parsing funcs to new parsing module - rename existing validation module - move `resolve_variables` func from expression_delimited.py into parsing/variable_reference.py --- pyxform/entities/entities_parsing.py | 2 +- pyxform/parsing/expression_delimited.py | 39 +--- pyxform/parsing/variable_reference.py | 192 ++++++++++++++++++ pyxform/question.py | 4 +- pyxform/survey.py | 8 +- pyxform/survey_element.py | 2 +- .../pyxform/question_types/__init__.py | 2 +- .../validators/pyxform/question_types/geo.py | 6 +- ...orm_reference.py => variable_reference.py} | 145 +------------ pyxform/xls2json.py | 9 +- .../pyxform/test_pyxform_reference.py | 12 +- 11 files changed, 223 insertions(+), 198 deletions(-) create mode 100644 pyxform/parsing/variable_reference.py rename pyxform/validators/pyxform/{pyxform_reference.py => variable_reference.py} (58%) diff --git a/pyxform/entities/entities_parsing.py b/pyxform/entities/entities_parsing.py index 4268c9b2..1ce36047 100644 --- a/pyxform/entities/entities_parsing.py +++ b/pyxform/entities/entities_parsing.py @@ -7,8 +7,8 @@ from pyxform.elements import action from pyxform.errors import ErrorCode, PyXFormError from pyxform.parsing.expression import is_xml_tag +from pyxform.parsing.variable_reference import parse_pyxform_references from pyxform.question_type_dictionary import get_meta_group -from pyxform.validators.pyxform.pyxform_reference import parse_pyxform_references EC = const.EntityColumns diff --git a/pyxform/parsing/expression_delimited.py b/pyxform/parsing/expression_delimited.py index 80afa7a1..fd74c7e0 100644 --- a/pyxform/parsing/expression_delimited.py +++ b/pyxform/parsing/expression_delimited.py @@ -1,14 +1,16 @@ from typing import TYPE_CHECKING -from lark import Lark, LarkError, Transformer, Tree +from lark import Lark, LarkError, Transformer from pyxform.errors import ErrorCode, PyXFormError -from pyxform.parsing.expression import RE_PYXFORM_REF from pyxform.parsing.instance_expression import ( replace_with_output as replace_with_output_old, ) +from pyxform.parsing.variable_reference import ( + is_pyxform_reference_candidate, + resolve_variables, +) from pyxform.utils import node -from pyxform.validators.pyxform.pyxform_reference import is_pyxform_reference_candidate if TYPE_CHECKING: from lark import Token @@ -38,33 +40,6 @@ ) -def resolve_variables( - xml_text: str, - survey: "Survey", - context: "SurveyElement | None" = None, - wrap: bool = False, -): - """ - Replace ${} variables with XPath references, optionally wrapped in . - - :param xml_text: Input string to process. - :param survey: Survey object for reference lookup. - :param context: The document node that the text belongs to. - :param wrap: If True, enclose resolved references in tags. - """ - - def resolver(match): - replaced = survey._var_repl_function(match, context) - if wrap: - return node("output", value=replaced).toxml() - else: - return replaced - - if is_pyxform_reference_candidate(value=xml_text): - return RE_PYXFORM_REF.sub(resolver, xml_text) - return xml_text - - class ExpressionTransformer(Transformer): def __init__( self, context: "SurveyElement", survey: "Survey", visit_tokens: bool = True @@ -89,7 +64,7 @@ def start(self, children: list["Token"]) -> str: return self._flatten(children=children) def text(self, children: list["Token"]) -> str: - """Join all children strings.""" + """Join all children strings, resolve variable references.""" original = self._flatten(children=children) resolved = resolve_variables( xml_text=original, survey=self.survey, context=self.context, wrap=True @@ -114,7 +89,7 @@ def invalid_nesting(self, children: list["Token"]) -> str: raise PyXFormError(code=ErrorCode.INTERNAL_004) def expression(self, children: list["Token"]) -> str: - """Join/wrap children in XML: ``.""" + """Join/wrap children in XML: ``, resolve variable references.""" resolved = resolve_variables( xml_text=self._flatten(children=children), survey=self.survey, diff --git a/pyxform/parsing/variable_reference.py b/pyxform/parsing/variable_reference.py new file mode 100644 index 00000000..e59dfe52 --- /dev/null +++ b/pyxform/parsing/variable_reference.py @@ -0,0 +1,192 @@ +""" +Helpers for parsing variable references from strings. + +The relatively small LRU cache sizes used here attempt to balance: +a) how expensive is the function? Regex and/or Scanner is worse than len and membership, + and there is also a cost to hash/lookup from the cache. +b) how much memory is it reasonable to spend on getting a high cache hit ratio vs. + lower ratio with extra time re-parsing; and the memory for the key and return value? +c) how likely is it that a large variety of unique strings are present in a XLSForm, and + how likely is it that similar strings are close to each other vs. randomly dispersed? +""" + +from collections.abc import Generator +from functools import lru_cache +from typing import TYPE_CHECKING + +from pyxform.errors import ErrorCode, PyXFormError +from pyxform.parsing.expression import ( + RE_PYXFORM_REF, + RE_PYXFORM_REF_INNER, + RE_PYXFORM_REF_OUTER, +) +from pyxform.utils import node + +if TYPE_CHECKING: + from pyxform.survey import Survey + from pyxform.survey_element import SurveyElement + + +class ParsedReference: + __slots__ = ("last_saved", "name") + + def __init__(self, name: str, last_saved: bool = False): + self.name: str = name + self.last_saved: bool = last_saved + + +@lru_cache(maxsize=128) +def is_pyxform_reference_candidate(value: str) -> bool: + """ + Check if the string looks like a pyxform reference. + + Needs 2 characters for "${", plus at least 1 more for a name inside. Does not look + for closing brace because full parsing will try to detect malformed references. This + pre-check can help avoid more expensive full parsing. + + :param value: The string to inspect. + """ + return len(value) > 2 and "${" in value + + +def _parse( + value: str, + match_limit: int | None = None, + match_full: bool = False, +) -> Generator[ParsedReference, None, None]: + """ + Parse the string and return reference target(s) e.g. `name` from `${name}`. + + It is an error if the reference token contains anything other than a valid ncname + (a question name), optionally with the `last-saved#` prefix. + + Does not otherwise treat `last-saved#` differently since https://docs.getodk.org/form-logic/ + says: "References to the last saved record could be used as part of any expression + wherever expressions are allowed". + + :param value: The string to inspect. + :param match_limit: If provided, parse only this many references in the string, and if + more references than the limit are found, then raise an error. + :param match_full: If True, require that the string contains a reference and nothing + else (no other characters or references). + """ + if not is_pyxform_reference_candidate(value): + return None + + if match_full: + outer_matches = RE_PYXFORM_REF_OUTER.fullmatch(value) + if not outer_matches: + # Expression may contain a reference but has other characters e.g. func call. + return None + outer_matches = (outer_matches,) + else: + outer_matches = RE_PYXFORM_REF_OUTER.finditer(value) + + count = 0 + # Look for any possible matches, then check each one for valid reference syntax. + for match in outer_matches: + ref_candidate = match.group("pyxform_ref") + # Although it's an "any" match pattern, fullmatch is used to require "only". + # Return the ref_candidate since it has original string start/end positions. + if ref_candidate: + ref_inner = RE_PYXFORM_REF_INNER.fullmatch(ref_candidate) + if ref_inner: + if match_limit is not None and count >= match_limit: + raise PyXFormError(code=ErrorCode.PYREF_002) + + yield ParsedReference( + name=ref_inner.group("ncname"), + last_saved=ref_inner.group("last_saved") is not None, + ) + count += 1 + else: + raise PyXFormError(code=ErrorCode.PYREF_001) + else: + raise PyXFormError(code=ErrorCode.PYREF_001) + + +@lru_cache(maxsize=128) +def is_pyxform_reference(value: str) -> bool: + """ + Does the input string contain only a valid Pyxform reference? e.g. `${my_question}`. + + :param value: The string to inspect. + """ + try: + return next(_parse(value=value, match_full=True), None) is not None + except (StopIteration, PyXFormError): + return False + + +@lru_cache(maxsize=128) +def has_pyxform_reference(value: str) -> bool: + """ + Does the input string contain a valid Pyxform reference? e.g. `hi ${name}`. + + :param value: The string to inspect. + """ + try: + return next(_parse(value=value), None) is not None + except (StopIteration, PyXFormError): + return False + + +@lru_cache(maxsize=128) +def has_pyxform_reference_with_last_saved(value: str) -> bool: + """ + Does the input string contain a valid '#last-saved'? e.g. `${last-saved#my_question}`. + + Needs 14 characters for "${last-saved#}", plus a name inside. This pre-check can help + avoid more expensive full parsing. + + :param value: The string to inspect. + """ + try: + return len(value) > 14 and any(i.last_saved for i in _parse(value=value)) + except (StopIteration, PyXFormError): + return False + + +@lru_cache(maxsize=128) +def parse_pyxform_references( + value: str, + match_limit: int | None = None, + match_full: bool = False, +) -> tuple[ParsedReference, ...]: + """ + Parse all pyxform references in a string. + + :param value: The string to inspect. + :param match_limit: If provided, parse only this many references in the string, and if + more references than the limit are found, then raise an error. + :param match_full: If True, require that the string contains a reference and nothing + else (no other characters or references). + """ + return tuple(_parse(value=value, match_limit=match_limit, match_full=match_full)) + + +def resolve_variables( + xml_text: str, + survey: "Survey", + context: "SurveyElement | None" = None, + wrap: bool = False, +): + """ + Replace ${} variables with XPath references, optionally wrapped in . + + :param xml_text: Input string to process. + :param survey: Survey object for reference lookup. + :param context: The document node that the text belongs to. + :param wrap: If True, enclose resolved references in tags. + """ + + def resolver(match): + replaced = survey._var_repl_function(match, context) + if wrap: + return node("output", value=replaced).toxml() + else: + return replaced + + if is_pyxform_reference_candidate(value=xml_text): + return RE_PYXFORM_REF.sub(resolver, xml_text) + return xml_text diff --git a/pyxform/question.py b/pyxform/question.py index 6ca4c86b..6a661026 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -16,6 +16,7 @@ from pyxform.elements import action from pyxform.errors import PyXFormError from pyxform.parsing.utils import maybe_strip +from pyxform.parsing.variable_reference import has_pyxform_reference from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement from pyxform.utils import ( @@ -25,9 +26,6 @@ default_is_dynamic, node, ) -from pyxform.validators.pyxform.pyxform_reference import ( - has_pyxform_reference, -) if TYPE_CHECKING: from pyxform.survey import Survey diff --git a/pyxform/survey.py b/pyxform/survey.py index 0c2b3dc9..9da4b1d7 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -17,6 +17,10 @@ from pyxform.instance import SurveyInstance from pyxform.parsing.expression import RE_PYXFORM_REF from pyxform.parsing.expression_delimited import replace_with_output +from pyxform.parsing.variable_reference import ( + has_pyxform_reference_with_last_saved, + is_pyxform_reference_candidate, +) from pyxform.question import Itemset, MultipleChoiceQuestion, Option, Question, Tag from pyxform.section import SECTION_EXTRA_FIELDS, RepeatingSection, Section from pyxform.survey_element import _GET_SENTINEL, SURVEY_ELEMENT_FIELDS, SurveyElement @@ -30,10 +34,6 @@ from pyxform.validators import odk_validate from pyxform.validators.pyxform import unique_names from pyxform.validators.pyxform.iana_subtags.validation import get_languages_with_bad_tags -from pyxform.validators.pyxform.pyxform_reference import ( - has_pyxform_reference_with_last_saved, - is_pyxform_reference_candidate, -) RE_BRACKET = re.compile(r"\[([^]]+)\]") RE_FUNCTION_ARGS = re.compile(r"\b[^()]+\((.*)\)$") diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index c7bcf6fc..eb4e7031 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -10,12 +10,12 @@ from pyxform import constants as const from pyxform.errors import ErrorCode, PyXFormError from pyxform.parsing.expression import is_xml_tag +from pyxform.parsing.variable_reference import has_pyxform_reference from pyxform.utils import ( DetachableElement, node, print_pyobj_to_json, ) -from pyxform.validators.pyxform.pyxform_reference import has_pyxform_reference if TYPE_CHECKING: from pyxform.survey import Survey diff --git a/pyxform/validators/pyxform/question_types/__init__.py b/pyxform/validators/pyxform/question_types/__init__.py index ac6fa10a..fa2bf5f2 100644 --- a/pyxform/validators/pyxform/question_types/__init__.py +++ b/pyxform/validators/pyxform/question_types/__init__.py @@ -3,7 +3,7 @@ from collections.abc import Collection, Iterable from pyxform.errors import ErrorCode, PyXFormError -from pyxform.validators.pyxform.pyxform_reference import ( +from pyxform.parsing.variable_reference import ( is_pyxform_reference_candidate, parse_pyxform_references, ) diff --git a/pyxform/validators/pyxform/question_types/geo.py b/pyxform/validators/pyxform/question_types/geo.py index b4432927..dd1b3af3 100644 --- a/pyxform/validators/pyxform/question_types/geo.py +++ b/pyxform/validators/pyxform/question_types/geo.py @@ -4,12 +4,12 @@ from pyxform import aliases from pyxform import constants as co from pyxform.errors import ErrorCode, PyXFormError -from pyxform.validators.pyxform import parameters as pv -from pyxform.validators.pyxform.parameters import PARAMETERS_TYPE -from pyxform.validators.pyxform.pyxform_reference import ( +from pyxform.parsing.variable_reference import ( is_pyxform_reference_candidate, parse_pyxform_references, ) +from pyxform.validators.pyxform import parameters as pv +from pyxform.validators.pyxform.parameters import PARAMETERS_TYPE def validate_parameter_incremental(value: str) -> None: diff --git a/pyxform/validators/pyxform/pyxform_reference.py b/pyxform/validators/pyxform/variable_reference.py similarity index 58% rename from pyxform/validators/pyxform/pyxform_reference.py rename to pyxform/validators/pyxform/variable_reference.py index 484a81fe..53a2dff6 100644 --- a/pyxform/validators/pyxform/pyxform_reference.py +++ b/pyxform/validators/pyxform/variable_reference.py @@ -11,159 +11,18 @@ """ from collections import Counter -from collections.abc import Generator, Sequence -from functools import lru_cache +from collections.abc import Sequence from typing import TYPE_CHECKING from pyxform import aliases from pyxform import constants as co from pyxform.errors import ErrorCode, PyXFormError -from pyxform.parsing.expression import ( - RE_PYXFORM_REF_INNER, - RE_PYXFORM_REF_OUTER, -) +from pyxform.parsing.variable_reference import parse_pyxform_references if TYPE_CHECKING: from pyxform.xls2json_backends import DefinitionData -class ParsedReference: - __slots__ = ("last_saved", "name") - - def __init__(self, name: str, last_saved: bool = False): - self.name: str = name - self.last_saved: bool = last_saved - - -def is_pyxform_reference_candidate(value: str) -> bool: - """ - Check if the string looks like a pyxform reference. - - Needs 2 characters for "${", plus at least 1 more for a name inside. Does not look - for closing brace because full parsing will try to detect malformed references. This - pre-check can help avoid more expensive full parsing. - - :param value: The string to inspect. - """ - return len(value) > 2 and "${" in value - - -def _parse( - value: str, - match_limit: int | None = None, - match_full: bool = False, -) -> Generator[ParsedReference, None, None]: - """ - Parse the string and return reference target(s) e.g. `name` from `${name}`. - - It is an error if the reference token contains anything other than a valid ncname - (a question name), optionally with the `last-saved#` prefix. - - Does not otherwise treat `last-saved#` differently since https://docs.getodk.org/form-logic/ - says: "References to the last saved record could be used as part of any expression - wherever expressions are allowed". - - :param value: The string to inspect. - :param match_limit: If provided, parse only this many references in the string, and if - more references than the limit are found, then raise an error. - :param match_full: If True, require that the string contains a reference and nothing - else (no other characters or references). - """ - if not is_pyxform_reference_candidate(value): - return None - - if match_full: - outer_matches = RE_PYXFORM_REF_OUTER.fullmatch(value) - if not outer_matches: - # Expression may contain a reference but has other characters e.g. func call. - return None - outer_matches = (outer_matches,) - else: - outer_matches = RE_PYXFORM_REF_OUTER.finditer(value) - - count = 0 - # Look for any possible matches, then check each one for valid reference syntax. - for match in outer_matches: - ref_candidate = match.group("pyxform_ref") - # Although it's an "any" match pattern, fullmatch is used to require "only". - # Return the ref_candidate since it has original string start/end positions. - if ref_candidate: - ref_inner = RE_PYXFORM_REF_INNER.fullmatch(ref_candidate) - if ref_inner: - if match_limit is not None and count >= match_limit: - raise PyXFormError(code=ErrorCode.PYREF_002) - - yield ParsedReference( - name=ref_inner.group("ncname"), - last_saved=ref_inner.group("last_saved") is not None, - ) - count += 1 - else: - raise PyXFormError(code=ErrorCode.PYREF_001) - else: - raise PyXFormError(code=ErrorCode.PYREF_001) - - -@lru_cache(maxsize=128) -def is_pyxform_reference(value: str) -> bool: - """ - Does the input string contain only a valid Pyxform reference? e.g. `${my_question}`. - - :param value: The string to inspect. - """ - try: - return next(_parse(value=value, match_full=True), None) is not None - except (StopIteration, PyXFormError): - return False - - -@lru_cache(maxsize=128) -def has_pyxform_reference(value: str) -> bool: - """ - Does the input string contain a valid Pyxform reference? e.g. `hi ${name}`. - - :param value: The string to inspect. - """ - try: - return next(_parse(value=value), None) is not None - except (StopIteration, PyXFormError): - return False - - -@lru_cache(maxsize=128) -def has_pyxform_reference_with_last_saved(value: str) -> bool: - """ - Does the input string contain a valid '#last-saved'? e.g. `${last-saved#my_question}`. - - Needs 14 characters for "${last-saved#}", plus a name inside. This pre-check can help - avoid more expensive full parsing. - - :param value: The string to inspect. - """ - try: - return len(value) > 14 and any(i.last_saved for i in _parse(value=value)) - except (StopIteration, PyXFormError): - return False - - -@lru_cache(maxsize=128) -def parse_pyxform_references( - value: str, - match_limit: int | None = None, - match_full: bool = False, -) -> tuple[ParsedReference, ...]: - """ - Parse all pyxform references in a string. - - :param value: The string to inspect. - :param match_limit: If provided, parse only this many references in the string, and if - more references than the limit are found, then raise an error. - :param match_full: If True, require that the string contains a reference and nothing - else (no other characters or references). - """ - return tuple(_parse(value=value, match_limit=match_limit, match_full=match_full)) - - def validate_pyxform_reference_syntax( sheet_name: str, sheet_data: Sequence[dict[str, str]], diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 093c6d40..df5f11e1 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -27,6 +27,7 @@ from pyxform.parsing.expression import is_xml_tag from pyxform.parsing.parameters import parse as parameters_parse from pyxform.parsing.sheet_headers import dealias_and_group_headers +from pyxform.parsing.variable_reference import has_pyxform_reference, is_pyxform_reference from pyxform.question_type_dictionary import get_meta_group from pyxform.utils import ( coalesce, @@ -39,15 +40,13 @@ from pyxform.validators.pyxform import settings as validate_settings from pyxform.validators.pyxform.android_package_name import validate_android_package_name from pyxform.validators.pyxform.choices import validate_and_clean_choices -from pyxform.validators.pyxform.pyxform_reference import ( - has_pyxform_reference, - is_pyxform_reference, - validate_pyxform_references_in_workbook, -) from pyxform.validators.pyxform.question_types import geo as qt_geo from pyxform.validators.pyxform.question_types import range as qt_range from pyxform.validators.pyxform.sheet_misspellings import find_sheet_misspellings from pyxform.validators.pyxform.translations_checks import SheetTranslations +from pyxform.validators.pyxform.variable_reference import ( + validate_pyxform_references_in_workbook, +) from pyxform.xls2json_backends import DefinitionData, get_xlsform RE_BEGIN_CONTROL = re.compile( diff --git a/tests/validators/pyxform/test_pyxform_reference.py b/tests/validators/pyxform/test_pyxform_reference.py index 2a17e059..700577e9 100644 --- a/tests/validators/pyxform/test_pyxform_reference.py +++ b/tests/validators/pyxform/test_pyxform_reference.py @@ -2,7 +2,9 @@ from itertools import chain, product from pyxform.errors import ErrorCode, PyXFormError -from pyxform.validators.pyxform import pyxform_reference as pr +from pyxform.validators.pyxform.variable_reference import ( + validate_pyxform_reference_syntax, +) from tests.pyxform_test_case import PyxformTestCase @@ -40,7 +42,7 @@ def test_single_reference__ok(self): for token, tok_desc in ok_tokens: with self.subTest(c=context, ctx=ctx_desc, t=token, tok=tok_desc): case = context.format(token) - pr.validate_pyxform_reference_syntax( + validate_pyxform_reference_syntax( sheet_name="test", sheet_data=({"label": case},), element_names=ELEMENT_NAMES, @@ -56,7 +58,7 @@ def test_single_reference__error(self): self.assertRaises(PyXFormError) as err, ): case = context.format(token) - pr.validate_pyxform_reference_syntax( + validate_pyxform_reference_syntax( sheet_name="test", sheet_data=({"label": case},), element_names=ELEMENT_NAMES, @@ -84,7 +86,7 @@ def test_multiple_reference__ok(self): tok_desc=(tok_desc1, tok_desc2), ): case = context.format(token1, token2) - pr.validate_pyxform_reference_syntax( + validate_pyxform_reference_syntax( sheet_name="test", sheet_data=({"label": case},), element_names=ELEMENT_NAMES, @@ -112,7 +114,7 @@ def test_multiple_references__error(self): self.assertRaises(PyXFormError) as err, ): case = context.format(token1, token2) - pr.validate_pyxform_reference_syntax( + validate_pyxform_reference_syntax( sheet_name="test", sheet_data=({"label": case},), element_names=ELEMENT_NAMES, From dd68d0243699dd44eb5a99c9f8a4fa2cbb1c99f1 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Fri, 16 Oct 2026 17:44:24 +1100 Subject: [PATCH 7/8] chg: collapse unnecessary if blocks in transformer, add tests - at some point in grammar development these were significant, but now if the rule is matched then it's OK to return the processed string - added two test cases to expand on existing adjacent tests with a similar structure, to clarify that these cases don't raise an error. --- pyxform/parsing/expression_delimited.py | 22 ++++++--------- tests/parsing/test_expression_delimited.py | 31 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/pyxform/parsing/expression_delimited.py b/pyxform/parsing/expression_delimited.py index fd74c7e0..fad20170 100644 --- a/pyxform/parsing/expression_delimited.py +++ b/pyxform/parsing/expression_delimited.py @@ -65,24 +65,18 @@ def start(self, children: list["Token"]) -> str: def text(self, children: list["Token"]) -> str: """Join all children strings, resolve variable references.""" - original = self._flatten(children=children) - resolved = resolve_variables( - xml_text=original, survey=self.survey, context=self.context, wrap=True + return resolve_variables( + xml_text=self._flatten(children=children), + survey=self.survey, + context=self.context, + wrap=True, ) - if original != resolved: - return resolved - else: - return original def escaped(self, children: list["Token"]) -> str: r"""Drop the leading backslash: `'\{{' -> '{{'`, `'\}}' -> '}}'`.""" - original = children[0] - resolved = {r"\{{": "{{", r"\}}": "}}"}[original] - if original != resolved: - self.expression_modified = True - return resolved - else: - return original + resolved = {r"\{{": "{{", r"\}}": "}}"}[children[0]] + self.expression_modified = True + return resolved def invalid_nesting(self, children: list["Token"]) -> str: """Raise an error if a nested expression is found.""" diff --git a/tests/parsing/test_expression_delimited.py b/tests/parsing/test_expression_delimited.py index 579eb8a4..8f2ec110 100644 --- a/tests/parsing/test_expression_delimited.py +++ b/tests/parsing/test_expression_delimited.py @@ -872,6 +872,20 @@ def test_triple_delimiter_tokens(self): ], ) + def test_triple_opening_delimiter(self): + """Should ignore extra delimiter token and accept outer opening delimiter.""" + # EDB01 EDB08 EDB14 + inp = "Triple open: {{{ ${q1} }}" + expected = """Triple open: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Triple open: ", " "}), + (xpq.body_input_label_output_value("q1"), {"{ /test_name/q1 "}), + ], + ) + def test_triple_closing_delimiter(self): """Should ignore extra delimiter token and accept outer closing delimiter.""" # EDB01 EDB08 EDB14 @@ -886,6 +900,23 @@ def test_triple_closing_delimiter(self): ], ) + def test_extra_opening_delimiter_near_expression_start(self): + """Should ignore single quote near accepted opening delimiter.""" + # EDB01 EDB08 EDB14 + inp = "Starting brace: {{ concat('{', ${q1}) }}" + expected = """Starting brace: """ + self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + xml__xpath_exact=[ + (xpq.body_input_label_text("q1"), {" Starting brace: ", " "}), + ( + xpq.body_input_label_output_value("q1"), + {" concat('{', /test_name/q1 ) "}, + ), + ], + ) + def test_extra_closing_delimiter_near_expression_end(self): """Should ignore single quote near accepted closing delimiter.""" # EDB01 EDB08 EDB14 From c8b770140ec1a831d081f76f00a6dbc953715a80 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Fri, 16 Oct 2026 18:56:09 +1100 Subject: [PATCH 8/8] chg: raise error for empty/blank expression, tidy error messages + tests - for the empty/blank case, assume that a user forgot to add an expression there and raise an error. There doesn't seem to be a use case for an empty output node. - since the internal errors raised by expression_delimited.py end up being shown to the user, make the phrasing a bit friendlier. - add expected exception check for each error test parsing assertion, they might differ from assertPyxformXform e.g. if the error is caught and wrapped in an error that says which row/column has a problem. --- pyxform/errors.py | 23 ++++-- pyxform/parsing/expression_delimited.py | 6 +- tests/parsing/test_expression_delimited.py | 88 ++++++++++++++++------ 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/pyxform/errors.py b/pyxform/errors.py index 514ac88b..fbd0c7d7 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -247,14 +247,27 @@ class ErrorCode(Enum): ), ) INTERNAL_003 = Detail( - name="Internal error - expression parsing failed", - msg="Internal error: expression parsing failed. Input string: {s}", + name="Internal error - delimited expression parsing failed, grammar error", + msg=( + "Delimited expression parsing failed. " + "Check the expression syntax, or try using delimiter escapes. " + "Input string: {s}" + ), ) INTERNAL_004 = Detail( - name="Internal error - expression parsing failed", + name="Internal error - delimited expression parsing failed, nested expression", + msg=( + "Delimited expression parsing failed. " + "Expressions must not be nested inside another expression. " + "Input string: {s}" + ), + ) + INTERNAL_005 = Detail( + name="Internal error - delimited expression parsing failed, empty expression", msg=( - "Internal error: expression parsing failed. " - "Nested expressions are not allowed. Input string: {s}" + "Delimited expression parsing failed. " + "Expressions must not be empty or only contain whitespace characters. " + "Input string: {s}" ), ) LABEL_001: Detail = Detail( diff --git a/pyxform/parsing/expression_delimited.py b/pyxform/parsing/expression_delimited.py index fad20170..0d376621 100644 --- a/pyxform/parsing/expression_delimited.py +++ b/pyxform/parsing/expression_delimited.py @@ -6,6 +6,7 @@ from pyxform.parsing.instance_expression import ( replace_with_output as replace_with_output_old, ) +from pyxform.parsing.utils import maybe_strip from pyxform.parsing.variable_reference import ( is_pyxform_reference_candidate, resolve_variables, @@ -84,8 +85,11 @@ def invalid_nesting(self, children: list["Token"]) -> str: def expression(self, children: list["Token"]) -> str: """Join/wrap children in XML: ``, resolve variable references.""" + flat = self._flatten(children=children) + if len(maybe_strip(flat)) == 0: + raise PyXFormError(code=ErrorCode.INTERNAL_005) resolved = resolve_variables( - xml_text=self._flatten(children=children), + xml_text=flat, survey=self.survey, context=self.context, ) diff --git a/tests/parsing/test_expression_delimited.py b/tests/parsing/test_expression_delimited.py index 8f2ec110..5503b47c 100644 --- a/tests/parsing/test_expression_delimited.py +++ b/tests/parsing/test_expression_delimited.py @@ -7,6 +7,7 @@ - Validation - EDV01: an opening expression delimiter must have a matching closing delimiter. - EDV02: expressions must not be nested. + - EDV03: expressions must not be empty (or contain only whitespace characters). - Behaviour - EDB01: delimited expression contents are wrapped in an XML element. - EDB02: whitespace in expression contents is preserved. @@ -65,62 +66,115 @@ def test_unclosed_template_with_trailing_content(self): """Unclosed template with trailing content.""" # EDV01 inp = "Total cost: {{ ${q1} * ${q2}" - with self.assertRaises(PyXFormError): + expected = ErrorCode.INTERNAL_003.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) self.assertPyxformXform( md=MD_SURVEY.format(label=inp), errored=True, - error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + error__contains=[expected], ) def test_valid_template_followed_by_unclosed_template(self): """Valid template followed by unclosed template.""" # EDV01 inp = "User {{ ${q1} }} has pending {{ ${q2}" - with self.assertRaises(PyXFormError): + expected = ErrorCode.INTERNAL_003.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) self.assertPyxformXform( md=MD_SURVEY.format(label=inp), errored=True, - error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + error__contains=[expected], ) def test_unclosed_expression_following_escaped_delimiter(self): """Unclosed template following escaped delimiter.""" # EDV01 inp = r"\{{ escaped \}} but {{ unclosed template" - with self.assertRaises(PyXFormError): + expected = ErrorCode.INTERNAL_003.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) self.assertPyxformXform( md=MD_SURVEY.format(label=inp), errored=True, - error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + error__contains=[expected], ) def test_repeated_delimiters(self): """Doubled up delimiters inside expression.""" # EDV01 inp = "Value: {{{{ ${q1} }}}}" - with self.assertRaises(PyXFormError): + expected = ErrorCode.INTERNAL_003.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) self.assertPyxformXform( md=MD_SURVEY.format(label=inp), errored=True, - error__contains=[ErrorCode.INTERNAL_003.value.format(s=inp)], + error__contains=[expected], ) def test_nested_delimiters_inside_expression(self): """Nested delimiters inside expression.""" # EDV02 inp = "Value: {{ {{ ${q1} }} }}" - with self.assertRaises(PyXFormError): + expected = ErrorCode.INTERNAL_004.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) self.assertPyxformXform( md=MD_SURVEY.format(label=inp), errored=True, - error__contains=[ErrorCode.INTERNAL_004.value.format(s=inp)], + error__contains=[expected], ) + def test_empty_expression(self): + """Empty expression.""" + # EDV03 + inp = "Value: {{}}" + expected = ErrorCode.INTERNAL_005.value.format(s=inp) + with self.assertRaises(PyXFormError) as err: + replace_with_output(inp, Q1, SURVEY) + self.assertEqual(str(err.exception), expected) + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[expected], + ) + + def test_whitespace_expression(self): + """Expression containing only whitespace.""" + # EDV03 + + # Use concat func since the extra curly braces for python formatting is confusing. + def make_case(i): + return "Value: {{" + i + "}}" + + cases = (" ", "\t", "\r\n", " ") + for case in cases: + inp = make_case(case) + with self.subTest(inp): + with self.assertRaises(PyXFormError) as err: + replace_with_output(inp, Q1, SURVEY) + self.assertEqual( + str(err.exception), + ErrorCode.INTERNAL_005.value.format(s=inp), + ) + + # Newlines confuse the markdown parser. Multiple spaces are collapsed in xls2json. + for case in cases[0:1]: + inp = make_case(case) + with self.subTest(inp): + self.assertPyxformXform( + md=MD_SURVEY.format(label=inp), + errored=True, + error__contains=[ErrorCode.INTERNAL_005.value.format(s=inp)], + ) + class TestOutputText(PyxformTestCase): """Should return unchanged text when no expression template is found.""" @@ -264,20 +318,6 @@ def test_adjacent_templates(self): ], ) - def test_empty_template(self): - """Should accept an empty template.""" - # EDB01 - inp = "Empty: {{}}" - expected = 'Empty: ' - self.assertEqual(expected, replace_with_output(inp, Q1, SURVEY)) - self.assertPyxformXform( - md=MD_SURVEY.format(label=inp), - xml__xpath_exact=[ - (xpq.body_input_label_text("q1"), {" Empty: ", " "}), - (xpq.body_input_label_output_value("q1"), {""}), - ], - ) - def test_no_inner_spaces(self): """Should accept expressions with no spaces inside the delimiters.""" # EDB01