From 3f685e03916ee2e7823433bfd311d6259c8593c0 Mon Sep 17 00:00:00 2001 From: Miguel Ortega Date: Tue, 22 Sep 2026 16:58:01 +0200 Subject: [PATCH 1/2] fix(querying): implement real TMF630 .regex support via NGSI-LD ~= REGEX now maps to NGSI-LD's native ~= instead of ==, fixes the always-false regex bug in subscription filters, and optimizes the ^(?!X$).*$ not-equal idiom into a native != (configurable, default on) for brokers without regex lookahead support. --- .../configuration/GeneralProperties.java | 10 ++++ .../tmforum/common/querying/Operator.java | 4 +- .../tmforum/common/querying/QueryParser.java | 44 +++++++++++++- .../querying/SubscriptionQueryResolver.java | 5 ++ .../common/querying/QueryParserTest.java | 60 +++++++++++++++++++ .../SubscriptionQueryResolverTest.java | 7 ++- 6 files changed, 125 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/fiware/tmforum/common/configuration/GeneralProperties.java b/common/src/main/java/org/fiware/tmforum/common/configuration/GeneralProperties.java index 8f84aa187..0470928a7 100644 --- a/common/src/main/java/org/fiware/tmforum/common/configuration/GeneralProperties.java +++ b/common/src/main/java/org/fiware/tmforum/common/configuration/GeneralProperties.java @@ -86,4 +86,14 @@ public class GeneralProperties { * (or existing data) that expects the plain, unescaped value. */ private boolean escapeForbiddenCharacters = true; + + /** + * When a {@code .regex} filter's value matches the "not equal" idiom + * {@code ^(?!X$).*$}, translate it into NGSI-LD's native {@code !=} comparison + * instead of forwarding {@code ~=} (pattern match) to the broker. Not every + * broker's regex engine supports negative lookahead (e.g. Scorpio's + * PostgreSQL-backed POSIX regex doesn't), so this optimization keeps the + * common "not equal" case portable. Default true. + */ + private boolean optimizeRegexNotEquals = true; } diff --git a/common/src/main/java/org/fiware/tmforum/common/querying/Operator.java b/common/src/main/java/org/fiware/tmforum/common/querying/Operator.java index 6c9bb7a84..ed9e8c165 100644 --- a/common/src/main/java/org/fiware/tmforum/common/querying/Operator.java +++ b/common/src/main/java/org/fiware/tmforum/common/querying/Operator.java @@ -8,8 +8,8 @@ public enum Operator { LESS_THAN(new TMForumOperator(".lt", "<"), "<"), LESS_THAN_EQUALS(new TMForumOperator(".lte", "<="), "<="), EQUALS(new TMForumOperator(".eq", "="), "=="), - // NGSI-LD allows regex for every string, so no specific operator exists, it's just an equals - REGEX(new TMForumOperator(".regex", "*="), "=="); + // NGSI-LD's native pattern-match operator (ETSI GS CIM 009, Simple Query Language) + REGEX(new TMForumOperator(".regex", "*="), "~="); @Getter private final TMForumOperator tmForumOperator; diff --git a/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java b/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java index c9fcdcb0a..2c387a7bf 100644 --- a/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java +++ b/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java @@ -22,6 +22,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -108,6 +109,18 @@ private static List translateJsonLdReservedTokens(List pathParts private static final Pattern NUMBER_LITERAL_PATTERN = Pattern.compile("-?(0|[1-9]\\d*)(\\.\\d+)?([eE][-+]?\\d+)?"); + // NGSI-LD's native "not equal" operator, used to rewrite the `.regex` "not equal" idiom below + private static final String NGSI_LD_NOT_EQUALS = "!="; + + /** + * Idiom used to express "not equal to X" via a negative-lookahead regex: {@code ^(?!X$).*$}. When + * {@link GeneralProperties#isOptimizeRegexNotEquals()} is enabled, a {@code .regex} filter matching + * this idiom is rewritten to NGSI-LD's native {@code !=} instead of forwarding a broker-side pattern + * match, since not every broker's regex engine supports lookahead (e.g. Scorpio's PostgreSQL-backed + * POSIX regex doesn't). + */ + private static final Pattern NOT_EQUALS_REGEX_IDIOM = Pattern.compile("^\\^\\(\\?!(.+?)\\$\\)\\.\\*\\$$"); + // NGSI-LD's orderBy suffixes a field with ";desc" to sort descending; omitting it means ascending private static final String ORDER_BY_DESCENDING_SUFFIX = ";desc"; @@ -234,7 +247,9 @@ public QueryParams toNgsiLdQuery(Class queryClass, String queryString) { return null; } - return toQueryString(getQueryPart(attribute, qp, isRelationship(queryClass, attribute)), attribute.type()); + QueryPart queryPart = applyRegexNotEqualsOptimization( + getQueryPart(attribute, qp, isRelationship(queryClass, attribute))); + return toQueryString(queryPart, attribute.type()); }) .filter(Objects::nonNull); @@ -341,6 +356,25 @@ private QueryPart getQueryPart(NgsiLdAttribute attribute, QueryPart qp, boolean qp.value()); } + /** + * When {@code queryPart} came from a {@code .regex} filter whose value matches the "not equal" + * idiom {@code ^(?!X$).*$}, rewrites it to NGSI-LD's native {@code !=} comparison instead of the + * pattern-match operator {@code ~=}. Controlled by {@link GeneralProperties#isOptimizeRegexNotEquals()}. + * + * @param queryPart the query part to (maybe) rewrite + * @return the rewritten query part, or the original one unchanged if it doesn't apply + */ + private QueryPart applyRegexNotEqualsOptimization(QueryPart queryPart) { + if (!queryPart.operator().equals(REGEX.getNgsiLdOperator()) || !generalProperties.isOptimizeRegexNotEquals()) { + return queryPart; + } + Matcher matcher = NOT_EQUALS_REGEX_IDIOM.matcher(queryPart.value()); + if (!matcher.matches()) { + return queryPart; + } + return new QueryPart(queryPart.attribute(), NGSI_LD_NOT_EQUALS, matcher.group(1)); + } + private String mapPathPart(String part) { if (generalProperties.getUseDotSeperator()) { return "." + part; @@ -364,6 +398,10 @@ private static void validateOperator(QueryPart queryPart, QueryAttributeType que throw new QueryException(String.format("%s cannot be compared with %s, it is a boolean attribute.", queryPart.attribute(), queryPart.operator())); } + if (queryPart.operator().equals(REGEX.getNgsiLdOperator()) && queryAttributeType != QueryAttributeType.STRING) { + throw new QueryException(String.format("%s cannot be compared with %s, regex only applies to string attributes.", + queryPart.attribute(), queryPart.operator())); + } } /** @@ -456,7 +494,9 @@ private String toQueryString(QueryPart queryPart, QueryAttributeType queryAttrib validateOperator(queryPart, queryAttributeType); - if (queryPart.value().contains(TMFORUM_OR_VALUE)) { + // a regex pattern may legitimately contain a "," - it must not be split into OR'd values + boolean isRegex = queryPart.operator().equals(REGEX.getNgsiLdOperator()); + if (!isRegex && queryPart.value().contains(TMFORUM_OR_VALUE)) { String theQuery = ""; List encodedValues = new ArrayList<>(Arrays.stream(queryPart.value().split(TMFORUM_OR_VALUE)) .map(v -> encodeValue(queryPart.attribute(), v, queryAttributeType)) diff --git a/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java b/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java index 140a7b84f..be89d49b9 100644 --- a/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java +++ b/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Stream; import static org.fiware.tmforum.common.querying.Operator.*; @@ -201,6 +202,10 @@ private boolean matches(QueryPart qp, FieldData fieldData) { } else { result = fieldData.fieldValue.equals(qp.value()); } + } else if (Objects.equals(qp.operator(), REGEX.getTmForumOperator().operator())) { + // evaluated in-process with java.util.regex, which supports lookahead natively - + // no broker regex-engine limitation applies here. + result = fieldData.fieldValue != null && Pattern.matches(qp.value(), fieldData.fieldValue.toString()); } return result; } diff --git a/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java b/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java index 9394e2ec6..11841849c 100644 --- a/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java +++ b/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java @@ -363,6 +363,66 @@ public void testValidTypedValuesAreTranslated(String message, String tmForumQuer assertEquals(ngsiLdQuery, qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message); } + /** + * NGSI-LD's native pattern-match operator is {@code ~=} (ETSI GS CIM 009, Simple Query Language). + * A comma inside the pattern is not a TMForum OR-list separator, so it must survive unsplit. + * + * @param message description of the concrete case + * @param tmForumQuery the query to translate + * @param ngsiLdQuery the expected translation + */ + @ParameterizedTest + @MethodSource("regexQueries") + public void testRegexQueries(String message, String tmForumQuery, QueryParams ngsiLdQuery) { + QueryParser qp = new QueryParser(new GeneralProperties()); + assertEquals(ngsiLdQuery, qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message); + } + + private static Stream regexQueries() { + return Stream.of( + Arguments.of("A plain regex is forwarded as a broker-side pattern match.", "color.regex=bl.e", + new QueryParams(null, null, "color~=\"bl.e\"")), + Arguments.of("The not-equal idiom is optimized into a native != comparison.", + "relatedParty.role.regex=^(?!seller$).*$", + new QueryParams(null, null, "relatedParty.role!=\"seller\"")), + Arguments.of("A comma inside a regex pattern must not be split into OR'd values.", + "color.regex=a,b", new QueryParams(null, null, "color~=\"a,b\""))); + } + + /** + * The not-equal idiom optimization can be disabled, in which case the idiom is forwarded as a + * genuine pattern match - which relies on the broker's regex engine supporting lookahead. + */ + @Test + public void testRegexNotEqualsOptimizationCanBeDisabled() { + GeneralProperties properties = new GeneralProperties(); + properties.setOptimizeRegexNotEquals(false); + QueryParser qp = new QueryParser(properties); + assertEquals(new QueryParams(null, null, "relatedParty.role~=\"^(?!seller$).*$\""), + qp.toNgsiLdQuery(MyPojo.class, "relatedParty.role.regex=^(?!seller$).*$"), + "With the optimization disabled, the idiom must be forwarded as a pattern match."); + } + + /** + * Regex only makes sense for string attributes - the pattern-match operator does not apply to + * booleans or numbers, whose NGSI-LD literals are unquoted. + * + * @param message description of the concrete case + * @param tmForumQuery the query to translate + */ + @ParameterizedTest + @MethodSource("regexOnNonStringQueries") + public void testRegexIsRejectedForNonStringTypes(String message, String tmForumQuery) { + QueryParser qp = new QueryParser(new GeneralProperties()); + assertThrows(QueryException.class, () -> qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message); + } + + private static Stream regexOnNonStringQueries() { + return Stream.of( + Arguments.of("Regex cannot be applied to a boolean attribute.", "active.regex=true"), + Arguments.of("Regex cannot be applied to a number attribute.", "temperature.regex=5")); + } + private static Stream validTypedValueQueries() { return Stream.of( Arguments.of("An integer should be accepted.", "temperature=20", diff --git a/common/src/test/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolverTest.java b/common/src/test/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolverTest.java index 0e99e5cd6..52d988dc1 100644 --- a/common/src/test/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolverTest.java +++ b/common/src/test/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolverTest.java @@ -38,7 +38,12 @@ private static Stream queriesForCreationEvent() { Arguments.of(MyPojoBuilder.build().createdAt(Instant.parse("2023-05-01T00:00:00.000Z")), "myPojo.createdAt<=2023-06-01T00:00:00.000Z", "myPojo", true), Arguments.of(MyPojoBuilder.build().createdAt(Instant.parse("2023-05-01T00:00:00.000Z")), - "myPojo.createdAt=2023-05-01T00:00:00.000Z", "myPojo", true) + "myPojo.createdAt=2023-05-01T00:00:00.000Z", "myPojo", true), + // regex is evaluated in-process with java.util.regex, which supports lookahead natively + Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=R.d", "myPojo", true), + Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=Bl.e", "myPojo", false), + Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=^(?!seller$).*$", "myPojo", true), + Arguments.of(MyPojoBuilder.build().color("seller"), "myPojo.color.regex=^(?!seller$).*$", "myPojo", false) ); } From 1da6d0e2fc72b6f2f064f38635767d6fceb85471 Mon Sep 17 00:00:00 2001 From: Miguel Ortega Date: Tue, 22 Sep 2026 16:59:22 +0200 Subject: [PATCH 2/2] fix(querying): escape operator symbol before split to avoid regex crash --- .../org/fiware/tmforum/common/querying/QueryParser.java | 9 ++++++++- .../common/querying/SubscriptionQueryResolver.java | 4 +++- .../fiware/tmforum/common/querying/QueryParserTest.java | 5 ++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java b/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java index 2c387a7bf..1471fd5e3 100644 --- a/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java +++ b/common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java @@ -525,12 +525,19 @@ private String toQueryString(QueryPart queryPart, QueryAttributeType queryAttrib return theQuery; } + // a regex pattern is opaque - it must not go through encodeStringValue's own OR/AND splitting + if (isRegex) { + return String.format("%s%s\"%s\"", queryPart.attribute(), queryPart.operator(), queryPart.value()); + } + return String.format("%s%s%s", queryPart.attribute(), queryPart.operator(), encodeValue(queryPart.attribute(), queryPart.value(), queryAttributeType)); } private QueryPart paramsToQueryPart(String parameter, Operator operator) { - String[] parameterParts = parameter.split(operator.getTmForumOperator().operator()); + // the operator symbol is split on literally - some symbols (e.g. REGEX's "*=") are not + // valid regexes on their own and would otherwise throw a PatternSyntaxException + String[] parameterParts = parameter.split(Pattern.quote(operator.getTmForumOperator().operator())); if (parameterParts.length != 2) { throw new QueryException(String.format("%s is not a valid %s parameter.", parameter, diff --git a/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java b/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java index be89d49b9..903af6318 100644 --- a/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java +++ b/common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java @@ -134,7 +134,9 @@ private boolean doesAttributeBelongToPayload(String attributePath, String payloa } private QueryPart paramsToQueryPart(String parameter, Operator operator) { - String[] parameterParts = parameter.split(operator.getTmForumOperator().operator()); + // the operator symbol is split on literally - some symbols (e.g. REGEX's "*=") are not + // valid regexes on their own and would otherwise throw a PatternSyntaxException + String[] parameterParts = parameter.split(Pattern.quote(operator.getTmForumOperator().operator())); if (parameterParts.length != 2) { throw new QueryException(String.format("%s is not a valid %s parameter.", parameter, diff --git a/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java b/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java index 11841849c..4559234eb 100644 --- a/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java +++ b/common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java @@ -386,7 +386,10 @@ private static Stream regexQueries() { "relatedParty.role.regex=^(?!seller$).*$", new QueryParams(null, null, "relatedParty.role!=\"seller\"")), Arguments.of("A comma inside a regex pattern must not be split into OR'd values.", - "color.regex=a,b", new QueryParams(null, null, "color~=\"a,b\""))); + "color.regex=a,b", new QueryParams(null, null, "color~=\"a,b\"")), + // the symbolic form ("*=") must not crash - it is not a valid regex on its own + Arguments.of("The symbolic operator form should work as well.", "color*=bl.e", + new QueryParams(null, null, "color~=\"bl.e\""))); } /**