Feat/regex query

common/src/main/java/org/fiware/tmforum/common/configuration/GeneralProperties.java

@@ -86,4 +86,14 @@ public class GeneralProperties {
 	 * (or existing data) that expects the plain, unescaped value.
 	 */
 	private boolean escapeForbiddenCharacters = true;
+
+	/**
+	 * When a {@code .regex} filter's value matches the "not equal" idiom
+	 * {@code ^(?!X$).*$}, translate it into NGSI-LD's native {@code !=} comparison
+	 * instead of forwarding {@code ~=} (pattern match) to the broker. Not every
+	 * broker's regex engine supports negative lookahead (e.g. Scorpio's
+	 * PostgreSQL-backed POSIX regex doesn't), so this optimization keeps the
+	 * common "not equal" case portable. Default true.
+	 */
+	private boolean optimizeRegexNotEquals = true;
 }

common/src/main/java/org/fiware/tmforum/common/querying/Operator.java

@@ -8,8 +8,8 @@ public enum Operator {
 	LESS_THAN(new TMForumOperator(".lt", "<"), "<"),
 	LESS_THAN_EQUALS(new TMForumOperator(".lte", "<="), "<="),
 	EQUALS(new TMForumOperator(".eq", "="), "=="),
-	// NGSI-LD allows regex for every string, so no specific operator exists, it's just an equals
-	REGEX(new TMForumOperator(".regex", "*="), "==");
+	// NGSI-LD's native pattern-match operator (ETSI GS CIM 009, Simple Query Language)
+	REGEX(new TMForumOperator(".regex", "*="), "~=");
 
 	@Getter
 	private final TMForumOperator tmForumOperator;

common/src/main/java/org/fiware/tmforum/common/querying/QueryParser.java

@@ -22,6 +22,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -108,6 +109,18 @@ private static List<String> translateJsonLdReservedTokens(List<String> pathParts
     private static final Pattern NUMBER_LITERAL_PATTERN =
             Pattern.compile("-?(0|[1-9]\\d*)(\\.\\d+)?([eE][-+]?\\d+)?");
 
+    // NGSI-LD's native "not equal" operator, used to rewrite the `.regex` "not equal" idiom below
+    private static final String NGSI_LD_NOT_EQUALS = "!=";
+
+    /**
+     * Idiom used to express "not equal to X" via a negative-lookahead regex: {@code ^(?!X$).*$}. When
+     * {@link GeneralProperties#isOptimizeRegexNotEquals()} is enabled, a {@code .regex} filter matching
+     * this idiom is rewritten to NGSI-LD's native {@code !=} instead of forwarding a broker-side pattern
+     * match, since not every broker's regex engine supports lookahead (e.g. Scorpio's PostgreSQL-backed
+     * POSIX regex doesn't).
+     */
+    private static final Pattern NOT_EQUALS_REGEX_IDIOM = Pattern.compile("^\\^\\(\\?!(.+?)\\$\\)\\.\\*\\$$");
+
     // NGSI-LD's orderBy suffixes a field with ";desc" to sort descending; omitting it means ascending
     private static final String ORDER_BY_DESCENDING_SUFFIX = ";desc";
 
@@ -234,7 +247,9 @@ public QueryParams toNgsiLdQuery(Class<?> queryClass, String queryString) {
                         return null;
                     }
 
-                    return toQueryString(getQueryPart(attribute, qp, isRelationship(queryClass, attribute)), attribute.type());
+                    QueryPart queryPart = applyRegexNotEqualsOptimization(
+                            getQueryPart(attribute, qp, isRelationship(queryClass, attribute)));
+                    return toQueryString(queryPart, attribute.type());
                 })
                 .filter(Objects::nonNull);
 
@@ -341,6 +356,25 @@ private QueryPart getQueryPart(NgsiLdAttribute attribute, QueryPart qp, boolean
                 qp.value());
     }
 
+    /**
+     * When {@code queryPart} came from a {@code .regex} filter whose value matches the "not equal"
+     * idiom {@code ^(?!X$).*$}, rewrites it to NGSI-LD's native {@code !=} comparison instead of the
+     * pattern-match operator {@code ~=}. Controlled by {@link GeneralProperties#isOptimizeRegexNotEquals()}.
+     *
+     * @param queryPart the query part to (maybe) rewrite
+     * @return the rewritten query part, or the original one unchanged if it doesn't apply
+     */
+    private QueryPart applyRegexNotEqualsOptimization(QueryPart queryPart) {
+        if (!queryPart.operator().equals(REGEX.getNgsiLdOperator()) || !generalProperties.isOptimizeRegexNotEquals()) {
+            return queryPart;
+        }
+        Matcher matcher = NOT_EQUALS_REGEX_IDIOM.matcher(queryPart.value());
+        if (!matcher.matches()) {
+            return queryPart;
+        }
+        return new QueryPart(queryPart.attribute(), NGSI_LD_NOT_EQUALS, matcher.group(1));
+    }
+
     private String mapPathPart(String part) {
         if (generalProperties.getUseDotSeperator()) {
             return "." + part;
@@ -364,6 +398,10 @@ private static void validateOperator(QueryPart queryPart, QueryAttributeType que
             throw new QueryException(String.format("%s cannot be compared with %s, it is a boolean attribute.",
                     queryPart.attribute(), queryPart.operator()));
         }
+        if (queryPart.operator().equals(REGEX.getNgsiLdOperator()) && queryAttributeType != QueryAttributeType.STRING) {
+            throw new QueryException(String.format("%s cannot be compared with %s, regex only applies to string attributes.",
+                    queryPart.attribute(), queryPart.operator()));
+        }
     }
 
     /**
@@ -456,7 +494,9 @@ private String toQueryString(QueryPart queryPart, QueryAttributeType queryAttrib
 
         validateOperator(queryPart, queryAttributeType);
 
-        if (queryPart.value().contains(TMFORUM_OR_VALUE)) {
+        // a regex pattern may legitimately contain a "," - it must not be split into OR'd values
+        boolean isRegex = queryPart.operator().equals(REGEX.getNgsiLdOperator());
+        if (!isRegex && queryPart.value().contains(TMFORUM_OR_VALUE)) {
             String theQuery = "";
             List<String> encodedValues = new ArrayList<>(Arrays.stream(queryPart.value().split(TMFORUM_OR_VALUE))
                     .map(v -> encodeValue(queryPart.attribute(), v, queryAttributeType))
@@ -485,12 +525,19 @@ private String toQueryString(QueryPart queryPart, QueryAttributeType queryAttrib
             return theQuery;
         }
 
+        // a regex pattern is opaque - it must not go through encodeStringValue's own OR/AND splitting
+        if (isRegex) {
+            return String.format("%s%s\"%s\"", queryPart.attribute(), queryPart.operator(), queryPart.value());
+        }
+
         return String.format("%s%s%s", queryPart.attribute(), queryPart.operator(),
                 encodeValue(queryPart.attribute(), queryPart.value(), queryAttributeType));
     }
 
     private QueryPart paramsToQueryPart(String parameter, Operator operator) {
-        String[] parameterParts = parameter.split(operator.getTmForumOperator().operator());
+        // the operator symbol is split on literally - some symbols (e.g. REGEX's "*=") are not
+        // valid regexes on their own and would otherwise throw a PatternSyntaxException
+        String[] parameterParts = parameter.split(Pattern.quote(operator.getTmForumOperator().operator()));
         if (parameterParts.length != 2) {
             throw new QueryException(String.format("%s is not a valid %s parameter.",
                     parameter,

common/src/main/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolver.java

@@ -9,6 +9,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
 import static org.fiware.tmforum.common.querying.Operator.*;
@@ -133,7 +134,9 @@ private boolean doesAttributeBelongToPayload(String attributePath, String payloa
     }
 
     private QueryPart paramsToQueryPart(String parameter, Operator operator) {
-        String[] parameterParts = parameter.split(operator.getTmForumOperator().operator());
+        // the operator symbol is split on literally - some symbols (e.g. REGEX's "*=") are not
+        // valid regexes on their own and would otherwise throw a PatternSyntaxException
+        String[] parameterParts = parameter.split(Pattern.quote(operator.getTmForumOperator().operator()));
         if (parameterParts.length != 2) {
             throw new QueryException(String.format("%s is not a valid %s parameter.",
                     parameter,
@@ -201,6 +204,10 @@ private boolean matches(QueryPart qp, FieldData fieldData) {
             } else {
                 result = fieldData.fieldValue.equals(qp.value());
             }
+        } else if (Objects.equals(qp.operator(), REGEX.getTmForumOperator().operator())) {
+            // evaluated in-process with java.util.regex, which supports lookahead natively -
+            // no broker regex-engine limitation applies here.
+            result = fieldData.fieldValue != null && Pattern.matches(qp.value(), fieldData.fieldValue.toString());
         }
         return result;
     }

common/src/test/java/org/fiware/tmforum/common/querying/QueryParserTest.java

@@ -363,6 +363,69 @@ public void testValidTypedValuesAreTranslated(String message, String tmForumQuer
 		assertEquals(ngsiLdQuery, qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message);
 	}
 
+	/**
+	 * NGSI-LD's native pattern-match operator is {@code ~=} (ETSI GS CIM 009, Simple Query Language).
+	 * A comma inside the pattern is not a TMForum OR-list separator, so it must survive unsplit.
+	 *
+	 * @param message      description of the concrete case
+	 * @param tmForumQuery the query to translate
+	 * @param ngsiLdQuery  the expected translation
+	 */
+	@ParameterizedTest
+	@MethodSource("regexQueries")
+	public void testRegexQueries(String message, String tmForumQuery, QueryParams ngsiLdQuery) {
+		QueryParser qp = new QueryParser(new GeneralProperties());
+		assertEquals(ngsiLdQuery, qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message);
+	}
+
+	private static Stream<Arguments> regexQueries() {
+		return Stream.of(
+				Arguments.of("A plain regex is forwarded as a broker-side pattern match.", "color.regex=bl.e",
+						new QueryParams(null, null, "color~=\"bl.e\"")),
+				Arguments.of("The not-equal idiom is optimized into a native != comparison.",
+						"relatedParty.role.regex=^(?!seller$).*$",
+						new QueryParams(null, null, "relatedParty.role!=\"seller\"")),
+				Arguments.of("A comma inside a regex pattern must not be split into OR'd values.",
+						"color.regex=a,b", new QueryParams(null, null, "color~=\"a,b\"")),
+				// the symbolic form ("*=") must not crash - it is not a valid regex on its own
+				Arguments.of("The symbolic operator form should work as well.", "color*=bl.e",
+						new QueryParams(null, null, "color~=\"bl.e\"")));
+	}
+
+	/**
+	 * The not-equal idiom optimization can be disabled, in which case the idiom is forwarded as a
+	 * genuine pattern match - which relies on the broker's regex engine supporting lookahead.
+	 */
+	@Test
+	public void testRegexNotEqualsOptimizationCanBeDisabled() {
+		GeneralProperties properties = new GeneralProperties();
+		properties.setOptimizeRegexNotEquals(false);
+		QueryParser qp = new QueryParser(properties);
+		assertEquals(new QueryParams(null, null, "relatedParty.role~=\"^(?!seller$).*$\""),
+				qp.toNgsiLdQuery(MyPojo.class, "relatedParty.role.regex=^(?!seller$).*$"),
+				"With the optimization disabled, the idiom must be forwarded as a pattern match.");
+	}
+
+	/**
+	 * Regex only makes sense for string attributes - the pattern-match operator does not apply to
+	 * booleans or numbers, whose NGSI-LD literals are unquoted.
+	 *
+	 * @param message      description of the concrete case
+	 * @param tmForumQuery the query to translate
+	 */
+	@ParameterizedTest
+	@MethodSource("regexOnNonStringQueries")
+	public void testRegexIsRejectedForNonStringTypes(String message, String tmForumQuery) {
+		QueryParser qp = new QueryParser(new GeneralProperties());
+		assertThrows(QueryException.class, () -> qp.toNgsiLdQuery(MyPojo.class, tmForumQuery), message);
+	}
+
+	private static Stream<Arguments> regexOnNonStringQueries() {
+		return Stream.of(
+				Arguments.of("Regex cannot be applied to a boolean attribute.", "active.regex=true"),
+				Arguments.of("Regex cannot be applied to a number attribute.", "temperature.regex=5"));
+	}
+
 	private static Stream<Arguments> validTypedValueQueries() {
 		return Stream.of(
 				Arguments.of("An integer should be accepted.", "temperature=20",

common/src/test/java/org/fiware/tmforum/common/querying/SubscriptionQueryResolverTest.java

@@ -38,7 +38,12 @@ private static Stream<Arguments> queriesForCreationEvent() {
                 Arguments.of(MyPojoBuilder.build().createdAt(Instant.parse("2023-05-01T00:00:00.000Z")),
                         "myPojo.createdAt<=2023-06-01T00:00:00.000Z", "myPojo", true),
                 Arguments.of(MyPojoBuilder.build().createdAt(Instant.parse("2023-05-01T00:00:00.000Z")),
-                        "myPojo.createdAt=2023-05-01T00:00:00.000Z", "myPojo", true)
+                        "myPojo.createdAt=2023-05-01T00:00:00.000Z", "myPojo", true),
+                // regex is evaluated in-process with java.util.regex, which supports lookahead natively
+                Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=R.d", "myPojo", true),
+                Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=Bl.e", "myPojo", false),
+                Arguments.of(MyPojoBuilder.build().color("Red"), "myPojo.color.regex=^(?!seller$).*$", "myPojo", true),
+                Arguments.of(MyPojoBuilder.build().color("seller"), "myPojo.color.regex=^(?!seller$).*$", "myPojo", false)
         );
     }