diff --git a/docs/fulltext2-backport-4.2dev-manifest.md b/docs/fulltext2-backport-4.2dev-manifest.md new file mode 100644 index 0000000000000..8cc6df7f811ad --- /dev/null +++ b/docs/fulltext2-backport-4.2dev-manifest.md @@ -0,0 +1,29 @@ +# FULLTEXT2 4.2-dev backport manifest + +This branch is a bounded backport from the refs captured on 2026-08-26. The +source and target refs are intentionally fixed for this run; later commits on +`upstream/main` are not followed. + +| source | scope | local commit | +| --- | --- | --- | +| `1d7b6311ce91df0968da435bb405f3d68137c595` (#25904) | FULLTEXT2 engine, parser/catalog contracts, plugin/CDC, planner/DDL and tests | `966acb2a92bb740e57b0b7a181df2d21956e1d3`, `0eb9a921ece274c956447640b41ad2a255079e8d`, `403a99fb65304e75b82e75793448f0690a871d0e`, `bfa83f42737ca6304c0af3f6913e70ccc961464` | +| `ec591dc9d4ec6b3beb81b804dac11aceb9b66441` (#27165) | delayed phrase-position decode after document alignment | `2b142144c3c24a2100d7d313e952717025d56aeb` | +| `3da66f2ca28d62f5e657c46d90b7cde9475486d0` (#27171), selective FT2 portion | wrapped FT2 score expressions, argument-aware score resolution, AND-reachable score ranges, and conservative float32 bounds; classic/view/vector changes from the source commit are excluded | `835e046faa9242dbe288bd72364a475e2132bbed` | +| `6dccc954a1a32c12a2114328c752f15cf97ef2ba` (#27461) | cold-load reason/latency observation and cache-load lifecycle | `e4abf5e90d0b5e09590571ca78447e50806527c` | +| `b8aa3f7f7351aacaaa37da27891d8f8d85e2e44d` (#27462) | immutable base/tail reuse across cache generations | `e4abf5e90d0b5e09590571ca78447e50806527c` | +| `e07a7a792749db4228c1acf96d72827161747e60` (#27598) | FULLTEXT2 BVT readiness polling in place of fixed sleeps | `e3a7440e2ac32bac780669f33c656960e22cff22` | +| current validation and review repairs | restore execution-level wrapped-score coverage, repair reverse score-bound membership and ISCP error propagation, and remove fixed-sleep/whitespace gate failures | `35f910dd543e1d3a952e4bd1a1b9852558ee2e27`, `fbea073c3b55836317c7aeb49d090ed2167f004c`, `06ab54d812ffcae1b4910e74f2f42a85ae9dfe0b`, `4cce042c8d3ad2cbada59ddd24e287785c297c1a` | + +## Ref closure + +- Source reference: `upstream/main` = `8e616a693458652cc26b3c4be45c6602dff298d4` +- Target reference: `upstream/4.2-dev` = `d2393868a7aaa6343518d80849fe4696aa3577e9` +- Branch: `backport/fulltext2-main-8e616a6-4.2dev-d239386` +- The repair/BVT commits above are ancestors of the manifest update; the exact + post-update branch head is recorded in the PR and the final remote reread. +- The target branch's existing parser/PERFORM and pipe changes were preserved; + the generated MySQL parser was regenerated from the merged 4.2 grammar. + +The diff does not use classic FULLTEXT as validation evidence. No local image, +direct `4.2-dev` push, Kubernetes mutation, or GitOps file mutation is part of +this backport. diff --git a/go.mod b/go.mod index 21ce81d1b0609..f19a526e0b99b 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 github.com/aws/smithy-go v1.22.1 github.com/axiomhq/hyperloglog v0.2.6 + github.com/blevesearch/vellum v1.2.0 github.com/buger/jsonparser v1.1.1 github.com/bytedance/sonic v1.15.2 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 @@ -83,7 +84,7 @@ require ( github.com/segmentio/encoding v0.4.0 github.com/shirou/gopsutil/v3 v3.23.12 github.com/smartystreets/goconvey v1.7.2 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.10.2 github.com/spkg/bom v1.0.0 github.com/stretchr/testify v1.11.1 github.com/syncthing/notify v0.0.0-20250528144937-c7027d4f7465 @@ -138,6 +139,8 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect + github.com/blevesearch/mmap-go v1.2.0 // indirect + github.com/bufbuild/protocompile v0.6.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect @@ -231,7 +234,7 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/smartystreets/assertions v1.13.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 // indirect github.com/tinylib/msgp v1.6.1 // indirect diff --git a/go.sum b/go.sum index 7fc7f591d36df..880e46baeb43c 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= +github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= +github.com/blevesearch/vellum v1.2.0 h1:xkDiOEsHc2t3Cp0NsNZZ36pvc130sCzcGKOPMzXe+e0= +github.com/blevesearch/vellum v1.2.0/go.mod h1:uEcfBJz7mAOf0Kvq6qoEKQQkLODBF46SINYNkZNae4k= +github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= +github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= @@ -204,7 +210,7 @@ github.com/cpegeric/pdftotext-go v0.0.0-20241112123704-49cb86a3790e/go.mod h1:zt github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -801,13 +807,13 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spkg/bom v1.0.0 h1:S939THe0ukL5WcTGiGqkgtaW5JW+O6ITaIlpJXTYY64= github.com/spkg/bom v1.0.0/go.mod h1:lAz2VbTuYNcvs7iaFF8WW0ufXrHShJ7ck1fYFFbVXJs= diff --git a/pkg/catalog/fulltext2_params_test.go b/pkg/catalog/fulltext2_params_test.go new file mode 100644 index 0000000000000..501fdf81bc1f0 --- /dev/null +++ b/pkg/catalog/fulltext2_params_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package catalog + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIndexParamsToStringListFulltext2Options pins that the shared SHOW CREATE / +// checkpoint-restore option renderer emits the fulltext2 build options — +// position_free plus the capacities — so a rebuilt index keeps them instead of +// silently reverting to a positional / default-capacity one. +func TestIndexParamsToStringListFulltext2Options(t *testing.T) { + params, err := IndexParamsMapToJsonString(map[string]string{ + "parser": "ngram", + IndexAlgoParamPositionFree: "true", + IndexAlgoParamMaxIndexCapacity: "1000", + IndexAlgoParamMaxPostingsCapacity: "8000", + }) + require.NoError(t, err) + + s, err := IndexParamsToStringList(params) + require.NoError(t, err) + require.Contains(t, s, IndexAlgoParamPositionFree+" = true") + require.Contains(t, s, IndexAlgoParamMaxIndexCapacity+" = 1000") + require.Contains(t, s, IndexAlgoParamMaxPostingsCapacity+" = 8000") + + // position_free=false ⇒ positional ⇒ NOT rendered (only recorded/rendered when true). + params2, err := IndexParamsMapToJsonString(map[string]string{IndexAlgoParamPositionFree: "false"}) + require.NoError(t, err) + s2, err := IndexParamsToStringList(params2) + require.NoError(t, err) + require.NotContains(t, s2, IndexAlgoParamPositionFree) +} + +// TestIndexParamsToStringListAsyncCadence pins that the AUTO_UPDATE compaction cadence — +// DAY / HOUR / SECOND — round-trips through the shared SHOW CREATE / checkpoint-restore +// renderer. SECOND in particular was dropped, so an async index (fulltext2 / ivfflat / +// cagra / ivfpq — all persist it) created with SECOND=5 silently lost its cadence on +// SHOW CREATE / checkpoint restore and reverted to the idxcron default interval. +func TestIndexParamsToStringListAsyncCadence(t *testing.T) { + params, err := IndexParamsMapToJsonString(map[string]string{ + AutoUpdate: "true", + Day: "1", + Hour: "2", + Second: "5", + }) + require.NoError(t, err) + + s, err := IndexParamsToStringList(params) + require.NoError(t, err) + require.Contains(t, s, Day+" = 1") + require.Contains(t, s, Hour+" = 2") + require.Contains(t, s, Second+" = 5") +} diff --git a/pkg/catalog/secondary_index_utils.go b/pkg/catalog/secondary_index_utils.go index 0ce74fa2148cb..03d92739e8905 100644 --- a/pkg/catalog/secondary_index_utils.go +++ b/pkg/catalog/secondary_index_utils.go @@ -29,15 +29,16 @@ import ( // Index Algorithm names const ( - MoIndexDefaultAlgo = tree.INDEX_TYPE_INVALID // used by UniqueIndex or default SecondaryIndex - MoIndexBTreeAlgo = tree.INDEX_TYPE_BTREE // used for Mocking MySQL behaviour. - MoIndexRTreeAlgo = tree.INDEX_TYPE_RTREE // used for Spatial Index on GEOMETRY columns - MoIndexIvfFlatAlgo = tree.INDEX_TYPE_IVFFLAT // used for IVF flat index on Vector/Array columns - MOIndexMasterAlgo = tree.INDEX_TYPE_MASTER // used for Master Index on VARCHAR columns - MOIndexFullTextAlgo = tree.INDEX_TYPE_FULLTEXT // used for Fulltext Index on VARCHAR columns - MoIndexHnswAlgo = tree.INDEX_TYPE_HNSW // used for HNSW Index on Vector/Array columns - MoIndexCagraAlgo = tree.INDEX_TYPE_CAGRA // used for CAGRA Index on Vector/Array columns - MoIndexIvfpqAlgo = tree.INDEX_TYPE_IVFPQ // used for IVFPQ Index on Vector/Array columns + MoIndexDefaultAlgo = tree.INDEX_TYPE_INVALID // used by UniqueIndex or default SecondaryIndex + MoIndexBTreeAlgo = tree.INDEX_TYPE_BTREE // used for Mocking MySQL behaviour. + MoIndexRTreeAlgo = tree.INDEX_TYPE_RTREE // used for Spatial Index on GEOMETRY columns + MoIndexIvfFlatAlgo = tree.INDEX_TYPE_IVFFLAT // used for IVF flat index on Vector/Array columns + MOIndexMasterAlgo = tree.INDEX_TYPE_MASTER // used for Master Index on VARCHAR columns + MOIndexFullTextAlgo = tree.INDEX_TYPE_FULLTEXT // used for Fulltext Index on VARCHAR columns + MoIndexFullText2Algo = tree.INDEX_TYPE_FULLTEXT2 // CREATE FULLTEXT2 INDEX: WAND positional engine on TEXT/VARCHAR columns + MoIndexHnswAlgo = tree.INDEX_TYPE_HNSW // used for HNSW Index on Vector/Array columns + MoIndexCagraAlgo = tree.INDEX_TYPE_CAGRA // used for CAGRA Index on Vector/Array columns + MoIndexIvfpqAlgo = tree.INDEX_TYPE_IVFPQ // used for IVFPQ Index on Vector/Array columns ) // ToLower is used for before comparing AlgoType and IndexAlgoParamOpType. Reason why they are strings @@ -81,6 +82,14 @@ func IsFullTextIndexAlgo(algo string) bool { return _algo == MOIndexFullTextAlgo.ToString() } +// IsFullText2IndexAlgo reports the distinct WAND fulltext engine used by +// CREATE FULLTEXT2 INDEX. It deliberately remains separate from classic +// FULLTEXT so planner and plugin routing cannot depend on index enumeration order. +func IsFullText2IndexAlgo(algo string) bool { + _algo := ToLower(algo) + return _algo == MoIndexFullText2Algo.ToString() +} + func IsHnswIndexAlgo(algo string) bool { _algo := ToLower(algo) return _algo == MoIndexHnswAlgo.ToString() @@ -107,6 +116,7 @@ const ( AutoUpdate = "auto_update" Day = "day" Hour = "hour" + Second = "second" DistributionMode = "distribution_mode" Quantization = "quantization" BitsPerCode = "bits_per_code" @@ -128,6 +138,14 @@ const ( IndexAlgoParamQuantizerTrainLimit = "quantizer_train_limit" IndexAlgoParamPrefixLengths = "prefix_lengths" + // IndexAlgoParamPrefixLengthsV2 stores a JSON object when a quoted column + // name contains a delimiter used by the legacy prefix encoding. + IndexAlgoParamPrefixLengthsV2 = "prefix_lengths_v2" + IndexAlgoParamMaxPostingsCapacity = "max_postings_capacity" + IndexAlgoParamPositionFree = "position_free" + // IndexAlgoParamVersion identifies the index engine; v2 is the WAND + // FULLTEXT2 engine and absent/1 is the classic FULLTEXT engine. + IndexAlgoParamVersion = "version" ) func ParseIncludeColumnsValue(raw string) ([]string, error) { @@ -230,6 +248,10 @@ func IndexParamsToStringList(indexParams string) (string, error) { res += fmt.Sprintf(" %s = %s ", Hour, val) } + if val, ok := result[Second]; ok { + res += fmt.Sprintf(" %s = %s ", Second, val) + } + if val, ok := result[Quantization]; ok { res += fmt.Sprintf(" %s '%s' ", Quantization, val) } @@ -266,9 +288,19 @@ func IndexParamsToStringList(indexParams string) (string, error) { res += fmt.Sprintf(" %s = %s ", IndexAlgoParamMaxIndexCapacity, val) } + if val, ok := result[IndexAlgoParamMaxPostingsCapacity]; ok { + res += fmt.Sprintf(" %s = %s ", IndexAlgoParamMaxPostingsCapacity, val) + } + if val, ok := result[IndexAlgoParamPositionFree]; ok && val == "true" { + res += fmt.Sprintf(" %s = %s ", IndexAlgoParamPositionFree, val) + } + if val, ok := result[IndexAlgoParamQuantizerTrainLimit]; ok { res += fmt.Sprintf(" %s = %s ", IndexAlgoParamQuantizerTrainLimit, val) } + if val, ok := result[IndexAlgoParamVersion]; ok { + res += fmt.Sprintf(" %s = %s ", IndexAlgoParamVersion, val) + } return res, nil } @@ -298,8 +330,16 @@ func IndexParamsMapToJsonString(res map[string]string) (string, error) { } func AddIndexPrefixLengthsToParams(indexParams string, keyParts []*tree.KeyPart) (string, error) { - prefixLengths := IndexPrefixLengthsToString(keyParts) - if prefixLengths == "" { + if len(keyParts) == 0 { + return indexParams, nil + } + prefixLengths := make(map[string]int, len(keyParts)) + for _, keyPart := range keyParts { + if keyPart != nil && keyPart.ColName != nil && keyPart.Length > 0 { + prefixLengths[keyPart.ColName.ColName()] = keyPart.Length + } + } + if len(prefixLengths) == 0 { return indexParams, nil } @@ -311,10 +351,38 @@ func AddIndexPrefixLengthsToParams(indexParams string, keyParts []*tree.KeyPart) } params = existing } - params[IndexAlgoParamPrefixLengths] = prefixLengths + if err := SetIndexPrefixLengthsInParamMap(params, prefixLengths); err != nil { + return "", err + } return IndexParamsMapToJsonString(params) } +// SetIndexPrefixLengthsInParamMap keeps the legacy representation for ordinary +// names and switches to a lossless JSON value when a quoted identifier contains +// a legacy delimiter. +func SetIndexPrefixLengthsInParamMap(params map[string]string, prefixLengths map[string]int) error { + if params == nil { + return moerr.NewInvalidInputNoCtx("nil index params map") + } + delete(params, IndexAlgoParamPrefixLengths) + delete(params, IndexAlgoParamPrefixLengthsV2) + if len(prefixLengths) == 0 { + return nil + } + for part := range prefixLengths { + if strings.ContainsAny(part, ":,") { + encoded, err := json.Marshal(prefixLengths) + if err != nil { + return err + } + params[IndexAlgoParamPrefixLengthsV2] = string(encoded) + return nil + } + } + params[IndexAlgoParamPrefixLengths] = indexPrefixLengthsMapToLegacyString(prefixLengths) + return nil +} + func IndexPrefixLengthsToString(keyParts []*tree.KeyPart) string { if len(keyParts) == 0 { return "" @@ -331,6 +399,13 @@ func IndexPrefixLengthsToString(keyParts []*tree.KeyPart) string { return "" } + return indexPrefixLengthsMapToLegacyString(prefixLengths) +} + +func indexPrefixLengthsMapToLegacyString(prefixLengths map[string]int) string { + if len(prefixLengths) == 0 { + return "" + } parts := make([]string, 0, len(prefixLengths)) for part := range prefixLengths { parts = append(parts, part) @@ -361,6 +436,22 @@ func IndexPrefixLengthsFromParamsWithError(indexParams string) (map[string]int, return nil, err } + if encoded, ok := params[IndexAlgoParamPrefixLengthsV2]; ok { + if encoded == "" { + return nil, nil + } + prefixLengths := make(map[string]int) + if err := json.Unmarshal([]byte(encoded), &prefixLengths); err != nil { + return nil, moerr.NewInvalidInputNoCtxf("invalid v2 index prefix lengths %q", encoded) + } + for part, length := range prefixLengths { + if part == "" || length <= 0 { + return nil, moerr.NewInvalidInputNoCtxf("invalid v2 index prefix length %q", encoded) + } + } + return prefixLengths, nil + } + encoded := params[IndexAlgoParamPrefixLengths] if encoded == "" { return nil, nil @@ -496,6 +587,33 @@ func IsIndexAsync(indexAlgoParams string) (bool, error) { return false, nil } +// IndexParamAsync is the descriptive name used by the index-plugin framework. +// Keep IsIndexAsync above as a compatibility wrapper for existing callers. +func IndexParamAsync(indexAlgoParams string) (bool, error) { + return IsIndexAsync(indexAlgoParams) +} + +// IndexAlgoParamVersionOf returns the engine version recorded in algo_params. +// Missing or malformed values are legacy/classic version 1. +func IndexAlgoParamVersionOf(indexAlgoParams string) int64 { + if len(indexAlgoParams) == 0 { + return 1 + } + val, err := sonic.Get([]byte(indexAlgoParams), IndexAlgoParamVersion) + if err != nil { + return 1 + } + s, err := val.StrictString() + if err != nil { + return 1 + } + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 1 + } + return n +} + //------------------------[END] IndexAlgoParams------------------------ // ------------------------[START] Aliaser------------------------ diff --git a/pkg/catalog/secondary_index_utils_test.go b/pkg/catalog/secondary_index_utils_test.go index 00339aaa4ce11..08705686de187 100644 --- a/pkg/catalog/secondary_index_utils_test.go +++ b/pkg/catalog/secondary_index_utils_test.go @@ -58,7 +58,7 @@ func newPrefixKeyPart(colName string, length int) *tree.KeyPart { } } -func TestIsIndexAsync(t *testing.T) { +func TestIndexParamAsync(t *testing.T) { var ( json string err error @@ -66,22 +66,22 @@ func TestIsIndexAsync(t *testing.T) { ) json = "" - ok, err = IsIndexAsync(json) + ok, err = IndexParamAsync(json) require.Nil(t, err) require.Equal(t, ok, false) json = "{}" - ok, err = IsIndexAsync(json) + ok, err = IndexParamAsync(json) require.Nil(t, err) require.Equal(t, ok, false) json = `{"async": "true"}` - ok, err = IsIndexAsync(json) + ok, err = IndexParamAsync(json) require.Nil(t, err) require.Equal(t, ok, true) json = `{"async": 1}` - _, err = IsIndexAsync(json) + _, err = IndexParamAsync(json) require.NotNil(t, err) } diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index 7154dd0128c33..0d236948e6511 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -421,6 +421,39 @@ const ( FullTextIndex_TabCol_Id = "doc_id" FullTextIndex_TabCol_Position = "pos" + /************ 3c. FULLTEXT v2 (VERSION=2) Index **************/ + + // Fulltext v2 (CREATE FULLTEXT INDEX ... VERSION=2, the WAND positional engine) + // uses the same chunked storage + metadata layout as bm25 — segments are built + // and CDC-maintained, no postings table — as opposed to classic v1's single + // (word,doc_id,pos) postings table. The index stays algo="fulltext"; the version + // param selects the engine and these hidden tables. Values are <= 11 chars + // (IndexAlgoTableType varchar(11) limit), so "ftv2_*" not "fulltext2_*". + FullText2Index_TblType_Metadata = "ftv2_meta" + FullText2Index_TblType_Storage = "ftv2_index" + + FullText2Index_TblCol_Storage_Index_Id = "index_id" + FullText2Index_TblCol_Storage_Chunk_Id = "chunk_id" + FullText2Index_TblCol_Storage_Data = "data" + FullText2Index_TblCol_Storage_Tag = "tag" + + FullText2Index_TblCol_Metadata_Index_Id = "index_id" + FullText2Index_TblCol_Metadata_Timestamp = "timestamp" + FullText2Index_TblCol_Metadata_Checksum = "checksum" + FullText2Index_TblCol_Metadata_Filesize = "filesize" + FullText2Index_TblCol_Metadata_Recency = "recency" + FullText2Index_TblCol_Metadata_Nrow = "nrow" + + // fulltext2_search TVF RESERVED output-column names. Unlike FullTextIndex_TabCol_Id + // ("doc_id", a PHYSICAL classic-index column), these are plan-level output ALIASES of + // the fulltext2_search TVF (its storage is segments, not a doc_id column). The covered + // fast path emits INCLUDE columns as sibling outputs and the runtime classifies the + // output batch BY NAME, so the pk/relevance outputs must use names no user INCLUDE + // column can equal — hence the reserved "__mo_ft_" prefix. Referenced by the coldef + // builders (tablefunc.go, apply_indices_fulltext2.go) and the classifier (fulltext2_search.go). + FullText2Search_OutCol_DocId = "__mo_ft_doc_id" + FullText2Search_OutCol_Score = "__mo_ft_score" + /************ 4. HNSW Index *************/ // HNSW Table Types diff --git a/pkg/cnservice/server_task.go b/pkg/cnservice/server_task.go index 5c04fe17a87c9..f4a8b89535d0b 100644 --- a/pkg/cnservice/server_task.go +++ b/pkg/cnservice/server_task.go @@ -352,6 +352,7 @@ func (s *service) registerExecutorsLocked() { s.task.runner.Attach, s.cfg.UUID, common.ISCPAllocator, + s.fileService, ), ) diff --git a/pkg/frontend/variables.go b/pkg/frontend/variables.go index 04ec4c8fda6a0..2d3b44f5bdc8d 100644 --- a/pkg/frontend/variables.go +++ b/pkg/frontend/variables.go @@ -31,6 +31,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fulltext" + "github.com/matrixorigin/matrixone/pkg/fulltext2" "github.com/matrixorigin/matrixone/pkg/util/gpumode" ) @@ -3816,6 +3817,14 @@ var gSysVarsDefs = map[string]SystemVariable{ Type: InitSystemVariableBoolType("experimental_fulltext_index"), Default: int8(0), }, + "experimental_fulltext2_index": { + Name: "experimental_fulltext2_index", + Scope: ScopeBoth, + Dynamic: true, + SetVarHintApplies: false, + Type: InitSystemVariableBoolType("experimental_fulltext2_index"), + Default: int8(0), + }, "ft_relevancy_algorithm": { Name: fulltext.FulltextRelevancyAlgo, Scope: ScopeBoth, @@ -3824,6 +3833,14 @@ var gSysVarsDefs = map[string]SystemVariable{ Type: InitSystemVariableStringType(fulltext.FulltextRelevancyAlgo), Default: fulltext.FulltextRelevancyAlgo_tfidf, }, + "ft2_relevancy_algorithm": { + Name: fulltext2.Fulltext2RelevancyAlgo, + Scope: ScopeBoth, + Dynamic: true, + SetVarHintApplies: false, + Type: InitSystemVariableStringType(fulltext2.Fulltext2RelevancyAlgo), + Default: fulltext2.Fulltext2RelevancyAlgo_bm25, + }, "experimental_hnsw_index": { Name: "experimental_hnsw_index", Scope: ScopeBoth, diff --git a/pkg/fulltext/plugin/compile/compile.go b/pkg/fulltext/plugin/compile/compile.go index 859db6676f692..f980fdcc497cd 100644 --- a/pkg/fulltext/plugin/compile/compile.go +++ b/pkg/fulltext/plugin/compile/compile.go @@ -27,6 +27,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -80,7 +81,7 @@ func (Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map[s return nil } - async, err := catalog.IsIndexAsync(indexDef.IndexAlgoParams) + async, err := indexplugin.IsAsync(indexDef.IndexAlgo, indexDef.IndexAlgoParams) if err != nil { return err } @@ -108,7 +109,7 @@ func (Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map[s } // HandleReindex — fulltext does not support ALTER … REINDEX. -func (Hooks) HandleReindex(_ compileplugin.CompileContext, _ map[string]*plan.IndexDef, _ bool) error { +func (Hooks) HandleReindex(_ compileplugin.CompileContext, _ map[string]*plan.IndexDef, _ bool, _ bool) error { return moerr.NewNotSupportedNoCtx("ALTER ... REINDEX is not supported for fulltext indexes") } diff --git a/pkg/fulltext/plugin/runtime/runtime.go b/pkg/fulltext/plugin/runtime/runtime.go index b83beb8c482aa..9bdaef3675bd7 100644 --- a/pkg/fulltext/plugin/runtime/runtime.go +++ b/pkg/fulltext/plugin/runtime/runtime.go @@ -84,6 +84,7 @@ func (CatalogHooks) ExperimentalFlag() string { return "" } // SupportedVectorTypes: fulltext has no vector column. func (CatalogHooks) SupportedVectorTypes() []types.T { return nil } +func (CatalogHooks) IsVectorIndex() bool { return false } // fulltext-family, not ANN // SupportedPrimaryKeyTypes: fulltext imposes no PK-type constraint. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } diff --git a/pkg/fulltext2/README.md b/pkg/fulltext2/README.md new file mode 100644 index 0000000000000..2d39ec1dd2d9c --- /dev/null +++ b/pkg/fulltext2/README.md @@ -0,0 +1,595 @@ +# `pkg/fulltext2` — WAND-based positional fulltext engine + +`fulltext2` is a full-text index engine registered as the distinct index algorithm `"fulltext2"` +and created with `CREATE FULLTEXT2 INDEX` (experimental, gated by the +`experimental_fulltext2_index` session var). It is a **completely separate engine** from classic +v1 fulltext (generated-SQL + Go `Eval`): a from-scratch Block-Max WAND positional engine with its +own registered plugin (`pkg/fulltext2/plugin`), segment format, and query pipeline — **not** a +version of the `fulltext` plugin. + +> There is no `WITH (VERSION = 2)` syntax. fulltext2 is its own algorithm keyed by +> `catalog.MoIndexFullText2Algo`; you select it with `CREATE FULLTEXT2 INDEX`, not by versioning a +> classic `FULLTEXT` index. + +fulltext2 also **absorbs the position-free ranked-retrieval role** once served by the former +standalone `bm25` engine (now removed): a `POSITION_FREE` fulltext2 index queried `IN BM25 MODE` +is the bag-of-words path. + +It rides the shared **index-plugin framework** — the plugin hooks, the ISCP CDC consumer, +`VectorIndexCache`, idxcron `MERGE`/`REBUILD`, and the payload-agnostic CDC chunk framing — plus +common leaf serialization patterns (tar members, little-endian buffers, PK encode/decode). + +One engine serves two build modes, chosen at CREATE: + +| | positional (default) | `POSITION_FREE = TRUE` (bag-of-words) | +|---|---|---| +| **term dict** | **sorted** FST of the actual indexed terms (`word*`) | same sorted FST | +| **posting payload** | **+ positions** (phrase, ngram reassembly) | df / tf only (no positions, ~57% smaller) | +| **query model** | NL phrase · BOOLEAN tree · BM25 bag | `IN BM25 MODE` bag-of-words **only** | +| **scoring** | **TF-IDF *and* BM25** | **TF-IDF *and* BM25** | + +A `POSITION_FREE` index rejects NL / BOOLEAN phrase queries up front (it has no positions) — only +`MATCH(…) AGAINST(… IN BM25 MODE)` is valid against it. + +--- + +## Table of contents + +1. [Architecture at a glance](#1-architecture-at-a-glance) +2. [The segment format](#2-the-segment-format) +3. [Core in-memory structures](#3-core-in-memory-structures) +4. [The query model](#4-the-query-model) +5. [Algorithm: Block-Max WAND (top-k)](#5-algorithm-block-max-wand-top-k) +6. [Algorithm: phrase & boolean evaluation](#6-algorithm-phrase--boolean-evaluation) +7. [Scoring: TF-IDF and BM25](#7-scoring-tf-idf-and-bm25) +8. [Multi-segment: liveness, deletes, recency](#8-multi-segment-liveness-deletes-recency) +9. [INCLUDE columns: in-index prefilter & coverage](#9-include-columns-in-index-prefilter--coverage) +10. [Build & lifecycle: sync build, CDC, MERGE/REBUILD](#10-build--lifecycle-sync-build-cdc-mergerebuild) +11. [Memory model: mmap, lazy decode, bounded build](#11-memory-model-mmap-lazy-decode-bounded-build) +12. [File map](#12-file-map) + +--- + +## 1. Architecture at a glance + +An index is a set of **segments** plus a small overlay of deletes: + +``` +CREATE INDEX ──► sync build (from source rows) ──► tag=0 BASE segments + │ +INSERT/UPDATE/DELETE ──► CDC (ISCP) ──► tag=1 TAIL segments (+ delete frames) + │ +ALTER … REINDEX ──► MERGE (fold tail into base) / REBUILD (from source) + +query ──► Index{ segments:[base…, tail…], deletes } ──► per-segment WAND/phrase ──► global top-k +``` + +- **Base segments** (`tag=0`) are written by the synchronous build (`fulltext2_create` TVF) and + by REBUILD; they are stored as chunk rows and materialized on the LOCAL (SSD) fileservice under + `__fulltext2` and **mmap'd read-only**. +- **Tail segments** (`tag=1`) are CDC deltas appended incrementally; they are smaller and loaded + into the Go heap. +- A query builds an `Index` over *all* base + tail segments, runs the retrieval algorithm on each + segment against **global** corpus statistics, and merges into one top-k. Per-segment **liveness** + hides docs superseded by a newer copy or a tombstone. + +The engine answers three query shapes, all through one `Index`: + +| Mode | SQL | Semantics | +|---|---|---| +| **NL (natural language)** | `MATCH(col) AGAINST('quick brown')` | exact **ordered phrase** (positional) | +| **BOOLEAN** | `… AGAINST('+quick -fox "brown dog" quic*' IN BOOLEAN MODE)` | operator tree (`+`/`-`/`~`/`>`/`<`/`()`/`"…"`/`*`) | +| **BM25 / bag-of-words** | `… IN BM25 MODE` | pure disjunction of tokens (position-free) | + +--- + +## 2. The segment format + +A segment is the unit of storage and retrieval. Every segment — base or tail — has the same +logical shape (`doc.go`, `serialize.go`): + +``` +segment + ├─ term dict : term(string) → byte offset of its posting directory (SORTED, dictionary-free) + ├─ postings : per term, doc-sorted: docID/tf blocks + block-max meta + POSITIONS + ├─ docmap : ord → { pk, docLen [, INCLUDE values] }; N = doc count + └─ avgDocLen : Σ docLen / N (computed at LOAD across all loaded segments) +``` + +**On disk** it is a **tar archive** of five members (`serialize.go`): + +| Member | Contents | Residency at load | +|---|---|---| +| `docmap` | `pkType`, `N`, `ord→pk` (length-prefixed), `ord→docLen`, optional `ord→INCLUDE values` (§9) | viewed (mmap); pk / include decoded on demand | +| `termdict` | the **vellum FST**: `term → offset of its directory entry` | resident (compact, minimized) | +| `postings` | the **ranking directory**: per-term `df`, per-block max-TF / skip meta | resident (small) | +| `blocks` | per-term `docID/tf` blocks (delta + varint) | **mmap; block-decoded on demand** | +| `positions` | per-term compressed positions (phrase-only) | **mmap; decoded on demand** | + +Two deliberate properties: + +- **Dictionary-free.** The term dict holds exactly the terms that were indexed (gojieba words or + ngram bigrams), with *no jieba-dict dependency* in the backbone. One sorted structure gives both + exact term lookup **and** prefix enumeration (`word*`). On disk it is a **vellum FST** — a + compact minimized automaton; because it is byte-oriented and UTF-8 byte order equals Unicode + code-point order, CJK terms need no special handling (Go's string sort already produces the FST's + required ascending byte order). See `termdict.go`. +- **Scorer-agnostic block-max.** The block-max section stores the **raw** per-block max-TF and + min-doc-length (`deriveTermStats`), never a baked impact score. So *one* segment serves both + TF-IDF and BM25; the active scorer derives its max-impact bound at query time. + +Blocks are `BlockSize = 128` docs. Block-level bounds (`blockLastDoc`, `blockMaxTf`, +`blockMinDocLn`) let WAND locate and skip whole 128-doc regions without decoding them. + +--- + +## 3. Core in-memory structures + +### `Index` (`index.go`) + +The queryable handle over a set of segments. + +```go +type Index struct { + segments []*Segment + deletes map[any]int64 // normalizeKey(pk) → recency at/after which older copies are dead + globalN int64 // total live docs across segments + globalAvgDocLen float64 // global average doc length (for BM25 length normalization) + liveOrd [][]bool // liveOrd[si][ord] = is that doc live? nil ⇒ whole segment live +} +``` + +- `NewIndex(segments, deletes)` builds it; `resolve()` (lazy, once) computes `globalN`, + `globalAvgDocLen`, and the per-segment `liveOrd` bitmaps. +- **Global stats matter**: a term's IDF and the corpus `avgDocLen` are computed *across all + segments*, not per-segment — otherwise appending a CDC tail would shift every score. WAND uses + these global stats so a multi-segment index ranks identically to a single rebuilt one. +- `liveOrd` is the **sole resident liveness structure**. A fully-live (append-only) segment keeps + `liveOrd[si] == nil` — the "all live" fast path — so an append-only index costs *zero* resident + liveness heap. The transient `pk→loc` map that `resolve()` builds to derive it is local and + discarded (it was the dominant load-time heap floor). + +### `Segment` (`segment.go`) + +A segment has **two representations** that never coexist: + +- **Build-side** (in-memory, freshly built): `pks []any`, `terms map[string]*termPostings`, + `sortedTerms []string`. Postings hold docIDs/tfs/positions as plain Go slices. +- **Loaded-side** (deserialized for query): `pks == nil`. Instead: + - `dict *termDict` — the vellum FST (`term → directory-entry offset`). + - `ranking / blocks / positions []byte` — **views into the mmap/blob**; a base segment's are + backed by `mmapData` (page-cache, reclaimable, shared by all concurrent queries, no copy). + - `pkOffsets []int32` + `pkRaw []byte` — the docmap bytes; `pk(ord)` decodes a pk on demand + (instead of materializing `N` boxed `any` pks, ~24 B each). + +A loaded segment **expands nothing at load**: `LookupLoaded(term)` decodes only the touched term's +directory entry on demand; WAND then decodes only the *blocks* its walk lands on. Resident heap is +`O(current query)`, not `O(vocabulary)`. `Free()` releases the mmap under the cache's eviction +write-lock. + +`Recency` orders segments when the same pk lands in several (UPDATE / reinsert / stale base copy): +only the highest-`Recency` copy is live. + +### `termPostings` (`segment.go`) + +One term's posting list. + +```go +type termPostings struct { + // build-side (nil on a loaded segment): + docIDs []int64 // ascending doc ords + tfs []uint8 // capped term frequency (≤ MaxCappedTf) + positions [][]int32 // per-doc token positions (byte offsets) + + ndoc int // df (document frequency) + + // loaded-side: block-compressed views into the mmap + blockData []byte; blockOff []int64 // docID/tf blocks (delta+varint), one per 128 docs + posRaw []byte; blockPosOff []int64 // compressed positions, block-seekable + + // scorer-agnostic score-UB inputs (raw): + maxTf uint8; minDocLen int32 // whole-list bounds + blockLastDoc []int64; blockMaxTf []uint8; blockMinDocLn []int32 // per-block bounds +} +``` + +The docID/tf blocks are the largest section (~46 % at load) and positions ~48 %; **neither is +expanded at load** — WAND touches only the blocks its block-max walk visits; positions are read +only for phrase verification / MERGE. + +### `wandIter` (`wand.go`) + +A per-term posting **cursor** for the WAND walk, carrying its `maxImpact` bound and a small +per-cursor decoded-block cache (`bDocs`/`bTfs`, cap 128). Key methods: `doc()` (cached current +ord), `skipTo(d)` (locate block via resident `blockLastDoc`, binary-search within), `blockMax(d)` +(block-level score UB), `blockEndAt(d)`. See §5. + +### `Builder` / `TailBuilder` (`build.go`, `tailbuild.go`) + +- `Builder` accumulates `(word, position, pk)` occurrences via `Add`, then `FinishSegments` / + `Finish` assembles a `Segment`. It seals at `min(max_index_capacity docs, max_postings_capacity + postings)` so per-segment build memory is bounded (see §10). +- `TailBuilder` is the streaming CDC builder: it tokenizes insert rows into capacity-capped + segments, **spilling each sealed segment to a temp file** as it fills, so the sinker's peak + memory is one open segment, not the whole CDC stream. Deletes are spilled likewise. + +--- + +## 4. The query model + +`SearchQuery(pattern, boolean, parser, algo, k, filter)` (`query.go`) is the dispatch: + +- **BOOLEAN mode** → `buildBooleanQuery` parses the pattern into a `BoolQuery` (an operator tree) + and calls `SearchBoolean`. +- **NL mode** → `phraseSlots` tokenizes into positioned slots and calls `SearchPhrase` — an exact + **ordered phrase** (matching classic fulltext NL semantics, including CJK). +- **BM25 mode** → `SearchBagOfWords` tokenizes the whole pattern into a pure disjunction of tokens + (each a SHOULD term) and runs the position-free WAND — bag-of-words retrieval that works on a + `POSITION_FREE` index. + +The boolean operator tree is built from `clause` nodes (`boolean.go`): + +```go +type clause struct { + kind clauseKind // clauseTerm | clausePhrase | clausePrefix | clauseAnd | clauseOr | clauseNot | group + terms []string // leaf term(s) / prefix + phrase []phraseSlot // positional (byte-offset) slots for a "…" phrase + children []clause // group + weight float32 // impact multiplier: 1.0 default; +/-/~//() change it +} +``` + +Operators: `+term` (MUST), `-term` (MUST NOT), `~term` (weight down), `>`/`<` (weight up/down), +`(…)` (group), `"…"` (exact phrase), `word*` (prefix → OR of all terms with that prefix, enumerated +from the FST). A bare multi-word run in boolean mode is an implicit contiguous phrase. + +`phraseSlot{ term, off }` carries a term and its byte offset within the query, so phrase adjacency +in the query matches adjacency in the source text. + +--- + +## 5. Algorithm: Block-Max WAND (top-k) + +**WAND** (Weak-AND, Broder 2003) + **Block-Max WAND** (Ding & Suel 2011) answer disjunctive +top-k — a pure OR of terms — returning *the exact same top-k as a full scan* while skipping most +documents. Implemented in `searchWAND` (`wand.go`). + +**Idea.** Each term carries a `maxImpact` (its largest possible weighted contribution). The top-k +min-heap's k-th best score is a moving threshold **θ**. Any document whose summed term upper bounds +cannot reach θ can never make the top-k, so it is never scored. + +**Per-iteration loop** (`searchWAND`): + +``` +loop: + 1. insertion-sort cursors by current doc ascending (nearly-sorted between iters ⇒ O(n)) + 2. pivot = first cursor where Σ maxImpact[0..i] ≥ θ (term-level WAND) + extend pivot over all cursors also sitting on pivotDoc + 3. blockSum = Σ blockMax(pivotDoc)[0..pivot] (Block-Max refinement, tighter UB) + if blockSum ≤ θ: skipTo(block end) — skip the whole 128-doc region, score nothing + 4. elif iters[0].doc() == pivotDoc: score pivotDoc, push to heap, advance; update θ + 5. else: skipTo(pivotDoc) — align a lagging cursor +``` + +**Multi-term query: how the cursors combine.** One `wandIter` per SHOULD term +(`buildWandIters`); a document's score is the **sum of the contributions of whichever query +terms it contains** — disjunctive, so a doc need not hold every term. Only cursors *sitting on +the same doc* contribute to that doc's score (step 4 sums `it.tf()`-based contributions for +every `it` at `pivotDoc`). The pivot is what makes it fast: because cursors are sorted by +current doc and `maxImpact` is accumulated in that order, any doc below `pivotDoc` could only +contain the terms of cursors `0..pivot-1`, whose max-impacts sum to `< θ` — so it provably +can't make the top-k and is skipped without ever being read. + +Worked example — query `A B C`, want top-1, θ currently 3.0, per-term +`maxImpact{A:4, B:1, C:5}`, cursors at `A@5, B@2, C@8`: + +``` +1. sort by doc → [B@2, A@5, C@8] +2. pivot scan → acc=B(1)=1 < 3 ; +A(4)=5 ≥ 3 ⇒ pivot=A, pivotDoc=5 + ⇒ doc 2 is skipped: B alone (max 1) can never beat θ=3 +3. blockSum{B,A}@5 = 4.2 > 3 ⇒ don't block-skip +4. iters[0].doc()==5 ? lead is B@2 ≠ 5 ⇒ NOT aligned ⇒ skipTo(5) on B, re-loop + (once cursors align on a doc, it is scored = Σ contributions of the terms it has; + that raises θ, which prunes even more of the next iteration) +``` + +The heavier a term (larger `maxImpact`), the earlier the pivot lands on it and the more docs +below it are pruned; a rare, high-idf term therefore does most of the skipping. + +Design notes worth knowing: + +- **Insertion sort, not `sort.Slice`** (step 1): the cursor array is nearly sorted between + iterations (only the skipped cursor moved), so insertion sort is O(n); `sort.Slice` boxed the + slice into an `interface{}` and heap-allocated the `less` closure every call — that alloc churn + dominated query CPU. +- **Cached `doc()`**: `wandIter.cur` is recomputed only on cursor move (`refresh`), because it is + read many times per pivot iteration (sort, pivot scan, blockMax, alignment). +- **Block-skip is free**: it only removes docs whose score UB ≤ θ — which the `score > θ` + insertion rule would reject anyway. Same top-k **set** and **scores** as the full scan; only the + work differs. +- **WHERE prefilter**: `allow` (a `Membership`) admits a doc to the heap only if its ord passes, + so `LIMIT` bounds the *filtered* set. Block-skip stays valid — `blockSum` bounds every doc in the + region regardless of the filter. +- **`newTopKHeap` bounds k** to the segment's live-doc count, so an absurd pushed `LIMIT 5e8` + can't eagerly allocate GB-sized heap buffers and OOM the CN. + +**Tie caveat**: documents with an *exactly equal* score have unspecified order (WAND sums +contributions in cursor order, the full scan in clause-map order — they can differ in the last +float ULP). The top-k *set* above the boundary and the score *multiset* are identical. + +The no-LIMIT sibling `streamWAND` (`stream.go`) reuses the cursors but walks *every* matching doc +in ord order (no θ, no heap), emitting in bounded batches — used when the upstream `ORDER BY score` +does the ranking. + +--- + +## 6. Algorithm: phrase & boolean evaluation + +WAND (§5) is **only** the pure-OR path. `SearchBoolean` dispatches on query shape: + +``` +pure OR of single-term SHOULDs → searchWAND (§5: θ/pivot/block-skip) +any MUST / MUST-NOT / phrase clause → searchBooleanFull (dense accumulator, NO skip) +NL / "…" phrase → SearchPhrase (conjunctive block-cursor) +``` + +The three engines treat their per-term cursors in fundamentally different ways — desynchronized +(OR), no cursors at all (boolean), or forced-to-converge (phrase). One worked example each below; +all share a corpus where the postings are: + +``` +term "quick" → docs {2, 5, 8, 40} term "brown" → docs {5, 8, 9} +term "fox" → docs {8, 60} (doc 8 = "quick brown fox", doc 5 = "quick brown …") +``` + +### Exact phrase (NL mode) — `SearchPhrase` → `matchPhraseCursor` (`search.go`) + +Conjunctive, anchored on the **rarest** slot, then a byte-offset positional verify. Cursors are +forced to *converge* on the anchor's current doc — the opposite of WAND. + +`AGAINST('brown fox')` → slots `[{brown, off=0}, {fox, off=6}]`: + +``` +1. rarest slot = "fox" (df 2) ⇒ drive its docs {8, 60} +2. doc 8: skipTo(8) the "brown" cursor → present. Positional check: + fox @ pos p ⇒ phrase-start = p-6 ; is "brown" @ start+0 ? yes ⇒ MATCH + doc 60: skipTo(60) the "brown" cursor → absent ⇒ drop, no position decode +3. score only doc 8 (the one verified doc) +``` + +So `AGAINST('brown fox')` matches doc 8 ("brown fox") but **not** a doc containing "fox … brown" +out of order — the offset check (`phraseSlot.off`) is what makes it a phrase, not a bag. Positions +are decoded **only** for docs that survive the doc-level intersection; `boundedTopK` keeps the +top-k without materializing all matches. (This is also the only mode that reads positions.) + +### Boolean AND / MUST / MUST-NOT — `searchBooleanFull` (`boolean.go`) + +No impact cursors, no skipping: every clause's **whole** posting list is materialized into a dense +`O(N)` per-doc `score[]` array; AND is an intersection by **hit-count**, MUST-NOT an exclusion +bitset. + +`AGAINST('+quick +brown -fox' IN BOOLEAN MODE)` → must `{quick, brown}`, mustNot `{fox}`: + +``` +1. mustNot "fox" → set bits {8, 60} in the exclusion bitset +2. MUST pass, count hits per doc (need == 2): + quick {2,5,8,40}: mustHit[2,5,8,40]=1 ; score[..]+= + brown {5,8,9} : mustHit[5]=2, mustHit[8]=2, mustHit[9]=1 ; score[..]+= +3. admit docs with mustHit == 2 → {5, 8} + doc 8 excluded by mustNot("fox") ⇒ dropped + ⇒ candidate = {5}, pushed to the top-k heap by its summed score +``` + +Every MUST/SHOULD/MUST-NOT term is read in full (`materializeDocIDs` decodes all its blocks) — +cost `O(Σ df)` to decode + `O(N)` dense memory, no block-skip. WAND's impact bound is useless here +because MUST/MUST-NOT need *membership* of every doc, which an impact upper bound cannot tell you. +`~term` (ADJUST) adds a (typically negative) contribution but does **not** exclude — a doc matching +only a `~`-term still ranks, just low (MySQL parity). + +### Boolean OR — routes to WAND + +`AGAINST('quick brown fox' IN BOOLEAN MODE)` with no `+`/`-`/`"…"` is a pure disjunction, so +`disjunctiveTerms` returns true and it runs the §5 WAND engine — identical to the OR example there. + +`word*` prefix clauses enumerate matching terms from the FST and union their posting lists; a group +`(…)` recurses. Both fall back to the dense `searchBooleanFull` path (they need a per-doc MAX over +their expansion). + +### Mode comparison + +| Mode | SQL | Engine | Cursor behavior | Skips docs? | Reads positions? | +|---|---|---|---|---|---| +| **OR** | `AGAINST('a b' IN BM25 MODE)` / pure-OR `IN BOOLEAN MODE` | `searchWAND` | desynchronized, impact-driven `skipTo` | **yes** (θ + block-max) | no | +| **AND / NOT** | `+a +b -c IN BOOLEAN MODE` | `searchBooleanFull` | none — full `materializeDocIDs` + dense arrays | no (full read) | no | +| **phrase** | `AGAINST('a b')` (default NL) / `"a b"` | `matchPhraseCursor` | conjunctive, converge on rarest-slot doc | yes (skip to anchor) | **yes** (offset check) | + +All three top-k paths use `vectorindex.FastMaxHeap` (SoA, keyed by ord, distance = −score) — zero +per-candidate allocation, ties unspecified (the top-k *set* and score *multiset* are stable; the +order among exactly-equal scores is not). + +--- + +## 7. Scoring: TF-IDF and BM25 + +`ScoreAlgo` selects the scorer; both are supported from the *same* segment because block-max is +raw (§2). + +- **TF-IDF**: `weight · tf · idf`. +- **BM25**: `weight · idf · bm25Factor(tf, docLen, avgDocLen)` — the saturating term-frequency + factor with document-length normalization against the **global** `avgDocLen`. + +Key quantities: + +- `idf` is computed from **global** df/N (across all segments), so a CDC tail doesn't shift scores. +- `termMaxImpact` / `blockMax` derive the WAND upper bounds in the *same* scorer, so the block-skip + bound is always ≥ the real score — WAND returns the identical ranking to a full scan. +- `tf` is capped (`MaxCappedTf`) to one byte; fulltext2 keeps *real* tf (unlike a position-free + impact-only index), so phrase/NL scoring is faithful. + +--- + +## 8. Multi-segment: liveness, deletes, recency + +An index is append-mostly: base segments plus CDC tail segments plus delete frames. Correctness +comes from three overlays computed in `Index.resolve()`: + +- **Recency**: when the same pk appears in several segments (UPDATE, reinsert, a stale base copy), + only the **highest-Recency** copy is live. Tail chunk_id > base recency, so a later append always + wins. +- **Deletes**: `deletes[pk] = recency` — any copy of `pk` older than that recency is dead + (tombstone). CDC delete frames carry pk-only. +- **`liveOrd[si]`**: a per-segment ord-indexed liveness bitmap derived once; `nil` means "whole + segment live" (append-only fast path). Every liveness check — phrase `isLive`, boolean/stream + `livenessMembership`, and compaction `ReconstructLiveDocs` — is an O(1) bitmap index, allocation + free. + +A skipped/dead doc is never scored and never emitted; `globalN` counts only live docs. + +--- + +## 9. INCLUDE columns: in-index prefilter & coverage + +A fulltext2 index can carry **INCLUDE columns** — the actual per-document values of chosen scalar +columns, stored *inside* the segments (and the CDC tail) alongside the pk. They let a filtered +top-k query run entirely inside `fulltext2_search`, dropping the two JOINs a plain fulltext query +needs (a prefilter second-scan for the WHERE, and a base-table join-back for the SELECT): + +```sql +CREATE FULLTEXT2 INDEX ftidx ON docs (body) + WITH PARSER json + INCLUDE (status, priority); + +-- filter on an include column is evaluated INSIDE the WAND walk; status/priority are +-- served straight from the index — no base-table JOIN, no prefilter second-scan: +SELECT id, status, priority +FROM docs +WHERE status = 'active' AND priority > 3 + AND MATCH(body) AGAINST('search terms'); +``` + +**Supported types.** Exactly what the segment pk codec already round-trips +(`CatalogHooks.SupportedIncludeColumnTypes`, `pkcodec.go`): the integer family +`int8/16/32/64`, `uint8/16/32/64` (+ `bit`), and `varchar`/`char`. The value is stored as its +**actual value, not a hash** — so both prefiltering *and* covering projection work, including +range and prefix/`LIKE 'x%'` predicates that a hash could not answer. `float`/`date`/`decimal`/ +`text` are deferred (the codec has no float case; widening the set is a codec extension, not a +plan change). The set + column names are pinned into `IndexAlgoParams` at CREATE and read back on +every build / CDC / query path. + +**Storage.** Include values live in the segment **docmap**, ord-aligned next to `pk` and `docLen` +(`serialize.go` `encodeDocmap`/`decodeDocmap`): a `[nCols][per-col type]` header, then per-doc, +per-col values encoded with the same codec as pks — integers dense fixed-width, strings as +`[u32 len][content]` varlena. A NULL include value carries a per-element null flag plus a +well-formed placeholder so the byte cursor stays aligned. On load the section is an mmap view +(`Segment.includeRaw` + per-varlena-col offset tables); lazy accessors decode on demand: +`Segment.includeVal(ord, colIdx)` (for the predicate evaluator) and the box-free +`Segment.appendIncludeTo(buf, colIdx, ord)` (for covering TVF output — the mirror of `appendPkTo`). + +**Prefilter** (`include_predicate.go`). A WHERE predicate on include columns is pushed into the +TVF as JSON and parsed into an `includePredMembership` implementing the existing +`Membership.Contains(ord)`: it decodes `seg.includeVal(ord, colIdx)` and tests the op +(`= < > BETWEEN IN`, string comparison, and **prefix** / `LIKE 'x%'`) with 3-valued NULL logic. +It is `andAllow`'d with the liveness/docfilter memberships, so it runs **inside** the Block-Max +WAND walk *before* top-k admission — a pushed `LIMIT` therefore bounds the already-filtered set +(no over-fetch). Block-skip stays valid: `blockSum` bounds every doc in a region regardless of the +filter. A pk predicate is peeled only when the pk type is one the evaluator can compare +(`fulltext2PeelablePkColName`); otherwise it stays a residual on the base scan and the JOIN is +retained — correct, just unoptimized. + +**Coverage** (`buildFulltext2SearchNodeCovered`, `search_cache.go` `SearchInto`/streaming). When +the index's INCLUDE columns cover the query's SELECT, the `fulltext2_search` TVF's output coldefs +are `[__mo_ft_doc_id, __mo_ft_score, ]`, so the projection +reads pk/score/include directly from the TVF with **no base-table JOIN**. The pk/score outputs use +the reserved names `catalog.FullText2Search_OutCol_DocId` / `_Score` (`__mo_ft_doc_id` / +`__mo_ft_score`) precisely so an INCLUDE column named `doc_id` or `score` cannot collide with them +(the runtime classifies the output batch by name). Results carry include values box-free through +`vectorindex.ColumnBuffer` / `SearchOutput` on both the LIMIT (`SearchInto`) and no-LIMIT +(streaming `Emit`) paths. + +**Value source.** Build (CREATE/REBUILD) selects the INCLUDE columns after the text columns and +passes them as trailing `fulltext2_create` args (the tokenizer stops *before* them so they are not +indexed as text); CDC (ISCP `Fulltext2SqlWriter`) resolves each include column's position from +`Name2ColIndex` and threads its value through `cdc.Insert/Upsert` into `SetDoc`, which replaces +include values in lock-step with terms so an UPSERT supersedes them. + +Cost tradeoff: the docmap grows by the include values per doc (full string values bloat the +segment — the same tradeoff fulltext2 already accepts for varchar pks; prefer small scalar +include columns). + +--- + +## 10. Build & lifecycle: sync build, CDC, MERGE/REBUILD + +Index CREATE → sync build → CDC maintenance → periodic compaction. All bases/tails are chunk rows +in two hidden tables (storage + metadata), framed with the shared CDC chunk format. + +- **Sync build** (`fulltext2_create` TVF): CROSS APPLY over the source table, tokenize each row + (datalink → plain text, json → values, ngram/gojieba parser), `Add` to a streaming `Builder`, + seal + persist tag=0 base sub-segments as they fill. `CREATE FULLTEXT2 INDEX` registers this + + an always-async CDC task. +- **CDC** (`cdc.go`, `tailbuild.go`, `sink.go`, ISCP consumer): INSERT/UPSERT/DELETE flow into the + tag=1 tail. `TailBuilder` tokenizes inserts into capped, spilled tail segments; deletes into + spilled tombstone frames; on flush it appends them (delete-frames first, then insert segments) at + the next chunk_id in one txn, advancing the watermark. +- **MERGE** (`compact.go`, `fulltext2_compact` TVF): fold the tag=1 tail into the tag=0 base and + reclaim dead space — load base+tail+deletes, `ReconstructLiveDocs` (from postings, no re-tokenize) + into a fresh capacity-bounded base, atomically replace all prior bases + the whole tail. +- **REBUILD**: discard the tail and rebuild the base from source (`buildFromSource`). +- idxcron schedules MERGE (or REBUILD once dead-doc % is high) when the tail grows past a chunk + threshold. + +--- + +## 11. Memory model: mmap, lazy decode, bounded build + +Full-text indexes are large; the engine is careful on both the query and build sides. + +**Query side:** + +- Base segments are **mmap'd read-only** on the fast LOCAL (SSD) `__fulltext2` fileservice dir — + page-cache-backed (reclaimable, *not* Go heap), shared by all concurrent queries, no copy. +- A loaded segment expands **nothing** at load: FST resident (compact), directory entries decoded + per touched term, docID/tf blocks decoded per block the WAND walk visits, positions decoded only + for phrase verification. Resident heap is `O(current query)`, not `O(vocabulary)`. +- pks are decoded on demand from the docmap bytes (no `N` boxed `any`). +- `liveOrd == nil` for an append-only segment ⇒ zero resident liveness heap. +- A base-load budget guard fails fast (clear error) rather than letting a huge base OOM-kill the CN. + +**Build side** — per-segment build memory ≈ **Σ postings** (term occurrences held in +`Builder.docs`), *not* doc count (a doc can hold 1 token or thousands). So every streaming build +path (create TVF, CDC `TailBuilder`, MERGE/REBUILD `CompactSegments`) seals a segment on +`ReachedSegmentCap(b, docCap, postingCap)` — **whichever of `max_index_capacity` (docs) or +`max_postings_capacity` (postings, default 8M ≈ ~512 MB build peak) fires first**. This bounds +build memory regardless of document shape. + +--- + +## 12. File map + +| File | Responsibility | +|---|---| +| `doc.go` | package overview + segment-format summary | +| `index.go` | `Index` type, `resolve()`, `SearchPhrase`/`SearchBoolean`/`SearchText` entry points, liveness | +| `segment.go` | `Segment` (build-side vs loaded-side), `termPostings`, `deriveTermStats`, `BlockSize`, INCLUDE accessors (`includeVal`, `appendIncludeTo`) | +| `wand.go` | `wandIter`, Block-Max WAND (`searchWAND`), `buildWandIters`, top-k heap | +| `stream.go` | no-LIMIT streaming WAND (`streamWAND`, `StreamQuery`, `StreamBagOfWords`) | +| `query.go` | `SearchQuery` dispatch, `SearchBagOfWords`, boolean-query build, phrase slots | +| `boolean.go` | boolean operator tree (`clause`, `clauseKind`), boolean evaluation | +| `search.go` | phrase evaluation, per-segment scoring glue, `boundedTopK` | +| `membership.go` | WHERE-prefilter + liveness `Membership` adapters | +| `include_predicate.go` | in-index INCLUDE-column predicate → `Membership` (`= < > BETWEEN IN`, prefix/`LIKE`, 3-valued NULL) | +| `pkcodec.go` | typed pk / INCLUDE value encode/decode (integer family dense fixed-width; varchar/char varlena `[len][value]`) | +| `termdict.go` | vellum FST term dict: build, load, exact `get`, `prefixIter`, `forEachTerm` | +| `build.go` | `Builder` (Add/Finish/FinishSegments), `TokenizedDoc`, `ReachedSegmentCap`, seal caps | +| `tailbuild.go` | streaming CDC `TailBuilder` (spill-as-you-fill) | +| `cdc.go` | CDC event decode (`Cdc`), pk typing | +| `deletes.go` | delete-record framing / tombstones | +| `frames.go` | CDC chunk framing (`FrameSegment`, `UnframeTail`) | +| `sink.go` | ISCP sink adapter glue | +| `compact.go` | `CompactSegments` (MERGE): reconstruct live docs → fresh bounded base | +| `serialize.go` | tar-archive segment encode/decode (docmap incl. INCLUDE section / termdict/postings/blocks/positions) | +| `storage.go` | chunk-row persistence, `LoadAllBases`/tail, `__fulltext2` spill, load budgets, `TableConfig` | +| `search_cache.go` | `VectorIndexCache` integration (per-CN cached loaded index) | +| `mmap_unix.go` / `mmap_other.go` | read-only mmap + munmap (platform) | + +For the plugin wiring (hooks, grammar, CDC registration, idxcron), see `pkg/fulltext2/plugin/…`. diff --git a/pkg/fulltext2/base_pool.go b/pkg/fulltext2/base_pool.go new file mode 100644 index 0000000000000..01b7e623f5f2d --- /dev/null +++ b/pkg/fulltext2/base_pool.go @@ -0,0 +1,498 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "fmt" + "sync" + "time" +) + +const ( + // The pool is deliberately bounded independently of VectorIndexCache. A cache + // entry can expire while an immutable base is still reusable, but that reuse + // must not turn into a process-lifetime cache for every index ever queried. + basePoolMaxEntries = 1024 + basePoolMaxBytes = int64(8 << 30) + basePoolIdleTTL = 15 * time.Minute +) + +// baseKey identifies the immutable bytes of one tag=0 segment. Recency is +// deliberately excluded: a metadata rewrite may change recency while the +// segment bytes remain identical, and each generation gets its own Segment +// header with the new recency. +type baseKey struct { + index string + id string + checksum string + filesize int64 +} + +type baseEntry struct { + ready chan struct{} + lease *segmentLease + err error + loading bool + retired bool + published bool + owners map[uint64]struct{} + lastUsed time.Time +} + +// segmentLease owns one mmap and hands out shallow Segment views. The views +// have independent mutable headers (notably AvgDocLen and Recency) while all +// serialized slices/FST pointers remain shared and read-only. +type segmentLease struct { + mu sync.Mutex + template *Segment + refs int // pool reference plus one reference per returned view + pooled bool + retired bool +} + +func newSegmentLease(template *Segment) *segmentLease { + return &segmentLease{template: template, refs: 1, pooled: true} +} + +func (l *segmentLease) acquire(recency int64) *Segment { + l.mu.Lock() + defer l.mu.Unlock() + if l.template == nil { + return nil + } + l.refs++ + view := *l.template + if recency == 0 { + recency = l.template.Recency + } + view.Recency = recency + view.lease = l + return &view +} + +func (l *segmentLease) release() { + template := func() *Segment { + l.mu.Lock() + defer l.mu.Unlock() + if l.refs <= 0 { + return nil + } + l.refs-- + if l.refs != 0 { + return nil + } + template := l.template + l.template = nil + return template + }() + if template != nil { + template.freeOwned() + } +} + +// retire removes the pool's reference. Active generations keep the mapping +// alive until their Segment views are freed. +func (l *segmentLease) retire() { + template := func() *Segment { + l.mu.Lock() + defer l.mu.Unlock() + if !l.pooled { + return nil + } + l.pooled = false + l.retired = true + l.refs-- // drop the pool reference + if l.refs != 0 { + return nil + } + template := l.template + l.template = nil + return template + }() + if template != nil { + template.freeOwned() + } +} + +type immutableBasePool struct { + mu sync.Mutex + entries map[baseKey]*baseEntry + totalBytes int64 + maxEntries int + maxBytes int64 + idleTTL time.Duration +} + +var loadedBasePool = newImmutableBasePool() + +func newImmutableBasePool() *immutableBasePool { + return &immutableBasePool{ + entries: make(map[baseKey]*baseEntry), + maxEntries: basePoolMaxEntries, + maxBytes: basePoolMaxBytes, + idleTTL: basePoolIdleTTL, + } +} + +func (p *immutableBasePool) limits() (int, int64, time.Duration) { + maxEntries, maxBytes, idleTTL := p.maxEntries, p.maxBytes, p.idleTTL + if maxEntries <= 0 { + maxEntries = basePoolMaxEntries + } + if maxBytes <= 0 { + maxBytes = basePoolMaxBytes + } + if idleTTL <= 0 { + idleTTL = basePoolIdleTTL + } + return maxEntries, maxBytes, idleTTL +} + +func baseEntryBytes(key baseKey) int64 { + if key.filesize <= 0 { + return 0 + } + return key.filesize +} + +// evictLocked retires pool ownership for idle or over-cap entries. It must be +// called with p.mu held; the returned leases are retired after unlocking so +// Segment.Free never runs while the pool map is locked. +func (p *immutableBasePool) evictLocked(now time.Time) (retire []*segmentLease) { + _, _, idleTTL := p.limits() + remove := func(key baseKey, entry *baseEntry) { + delete(p.entries, key) + p.totalBytes -= baseEntryBytes(key) + if p.totalBytes < 0 { + p.totalBytes = 0 + } + if entry.lease != nil { + retire = append(retire, entry.lease) + } + } + for key, entry := range p.entries { + if entry.loading || entry.lease == nil || entry.lastUsed.IsZero() { + continue + } + if now.Sub(entry.lastUsed) >= idleTTL { + remove(key, entry) + } + } + maxEntries, maxBytes, _ := p.limits() + for len(p.entries) > maxEntries || p.totalBytes > maxBytes { + var oldestKey baseKey + var oldest *baseEntry + for key, entry := range p.entries { + if entry.loading || entry.lease == nil { + continue + } + if oldest == nil || entry.lastUsed.Before(oldest.lastUsed) { + oldestKey, oldest = key, entry + } + } + if oldest == nil { + break // a loading entry will be accounted for when its load completes + } + remove(oldestKey, oldest) + } + return retire +} + +// retireLoadingLocked marks an in-flight entry so its loader will not repool +// the result when it completes. The entry stays in the map until the loader +// closes ready; this keeps existing waiters from being stranded on a channel +// that can never be signaled. +func (p *immutableBasePool) retireLoadingLocked(entry *baseEntry) { + if entry != nil && entry.loading { + entry.retired = true + } +} + +func retireBaseLeases(leases []*segmentLease) { + for _, lease := range leases { + lease.retire() + } +} + +func (p *immutableBasePool) evict(now time.Time) { + retire := func() []*segmentLease { + p.mu.Lock() + defer p.mu.Unlock() + return p.evictLocked(now) + }() + retireBaseLeases(retire) +} + +func (p *immutableBasePool) acquire(ctx context.Context, key baseKey, load func() (*Segment, error), recency int64) (*Segment, error) { + return p.acquireOwned(ctx, key, load, recency, 0) +} + +// acquireOwned is the load-path variant. Each load attempt claims the pooled +// entry it creates or reuses, allowing an obsolete failed load to roll back +// only its own unpublished claims. +func (p *immutableBasePool) acquireOwned(ctx context.Context, key baseKey, load func() (*Segment, error), recency int64, owner uint64) (*Segment, error) { + if ctx == nil { + ctx = context.Background() + } + p.evict(time.Now()) + for { + if err := context.Cause(ctx); err != nil { + return nil, err + } + e, ready, view, found := func() (*baseEntry, chan struct{}, *Segment, bool) { + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.entries[key]; ok { + if e.loading { + if owner != 0 { + if e.owners == nil { + e.owners = make(map[uint64]struct{}, 1) + } + e.owners[owner] = struct{}{} + } + return nil, e.ready, nil, true + } + if owner != 0 { + if e.owners == nil { + e.owners = make(map[uint64]struct{}, 1) + } + e.owners[owner] = struct{}{} + } + e.lastUsed = time.Now() + return nil, nil, e.lease.acquire(recency), true + } + owners := make(map[uint64]struct{}, 1) + if owner != 0 { + owners[owner] = struct{}{} + } + e := &baseEntry{ready: make(chan struct{}), loading: true, owners: owners, lastUsed: time.Now()} + p.entries[key] = e + return e, nil, nil, false + }() + if found { + if ready != nil { + select { + case <-ready: + case <-ctx.Done(): + return nil, context.Cause(ctx) + } + continue + } + if view == nil { + continue + } + return view, nil + } + + template, err := func() (template *Segment, err error) { + defer func() { + if r := recover(); r != nil { + err = baseLeaseError(fmt.Sprintf("fulltext2 base load panic: %v", r)) + } + }() + return load() + }() + retired, directTemplate, resultErr := func() (bool, *Segment, error) { + p.mu.Lock() + defer p.mu.Unlock() + e.loading = false + e.err = err + if e.retired { + delete(p.entries, key) + e.err = errBaseLeaseGone + close(e.ready) + if err != nil { + return true, template, err + } + if template == nil { + return true, nil, errBaseLeaseGone + } + return true, template, nil + } + if err == nil { + if template == nil { + err = errBaseLeaseGone + e.err = err + } else { + e.lease = newSegmentLease(template) + e.lastUsed = time.Now() + p.totalBytes += baseEntryBytes(key) + } + } + close(e.ready) + if err != nil { + delete(p.entries, key) + } + return false, nil, err + }() + if retired { + if directTemplate != nil && resultErr != nil { + directTemplate.Free() + } + if resultErr != nil { + return nil, resultErr + } + if directTemplate == nil { + return nil, errBaseLeaseGone + } + if ctxErr := context.Cause(ctx); ctxErr != nil { + directTemplate.Free() + return nil, ctxErr + } + // The invalidation retired the pool entry while this query was + // loading it. Let this in-flight query finish with direct ownership; + // the stale closure result is never repooled or retried here. + return directTemplate, nil + } + if resultErr != nil { + if template != nil { + template.Free() + } + return nil, resultErr + } + if err := context.Cause(ctx); err != nil { + return nil, err + } + view = func() *Segment { + p.mu.Lock() + defer p.mu.Unlock() + e.lastUsed = time.Now() + return e.lease.acquire(recency) + }() + if view == nil { + continue + } + p.evict(time.Now()) + return view, nil + } +} + +// commit keeps only the base keys used by the successfully assembled +// generation. A retired lease remains alive for old readers but cannot be +// acquired by a later generation. +func (p *immutableBasePool) commit(index string, used map[baseKey]struct{}) { + p.commitOwned(index, used, 0) +} + +func (p *immutableBasePool) commitOwned(index string, used map[baseKey]struct{}, owner uint64) { + p.commitOwnedIfCurrent(index, used, owner, nil) +} + +func (p *immutableBasePool) commitOwnedIfCurrent(index string, used map[baseKey]struct{}, owner uint64, current func() bool) bool { + published := true + retire := func() []*segmentLease { + p.mu.Lock() + defer p.mu.Unlock() + if current != nil && !current() { + published = false + return nil + } + retire := make([]*segmentLease, 0, len(p.entries)) + for key, entry := range p.entries { + if key.index != index { + continue + } + if entry.loading { + p.retireLoadingLocked(entry) + continue + } + if _, ok := used[key]; ok { + entry.owners = nil + entry.published = true + entry.lastUsed = time.Now() + continue + } + delete(p.entries, key) + p.totalBytes -= baseEntryBytes(key) + retire = append(retire, entry.lease) + } + return append(retire, p.evictLocked(time.Now())...) + }() + retireBaseLeases(retire) + return published +} + +// rollback removes only entries still claimed by owner. Reused entries retain +// claims from concurrent loads until one of them publishes or rolls back. +func (p *immutableBasePool) rollback(owner uint64) { + if owner == 0 { + return + } + retire := func() []*segmentLease { + p.mu.Lock() + defer p.mu.Unlock() + retire := make([]*segmentLease, 0) + for key, entry := range p.entries { + if _, ok := entry.owners[owner]; !ok { + continue + } + delete(entry.owners, owner) + if len(entry.owners) != 0 || entry.published { + if entry.published && len(entry.owners) == 0 { + entry.owners = nil + } + continue + } + if entry.loading { + entry.retired = true + continue + } + delete(p.entries, key) + p.totalBytes -= baseEntryBytes(key) + if p.totalBytes < 0 { + p.totalBytes = 0 + } + if entry.lease != nil { + retire = append(retire, entry.lease) + } + } + return retire + }() + retireBaseLeases(retire) +} + +func (p *immutableBasePool) clearIndex(index string) { + p.commit(index, nil) +} + +func (p *immutableBasePool) clearAll() { + retire := func() []*segmentLease { + p.mu.Lock() + defer p.mu.Unlock() + retire := make([]*segmentLease, 0, len(p.entries)) + for key, entry := range p.entries { + if entry.loading { + p.retireLoadingLocked(entry) + continue + } + delete(p.entries, key) + if entry.lease != nil { + retire = append(retire, entry.lease) + } + } + p.totalBytes = 0 + return retire + }() + retireBaseLeases(retire) +} + +// errBaseLeaseGone is only reachable if a pool entry is retired concurrently +// with an acquire. The caller can retry the metadata/load sequence safely. +var errBaseLeaseGone = baseLeaseError("fulltext2 base lease retired") + +type baseLeaseError string + +func (e baseLeaseError) Error() string { return string(e) } diff --git a/pkg/fulltext2/base_pool_test.go b/pkg/fulltext2/base_pool_test.go new file mode 100644 index 0000000000000..26fc76bd70a0a --- /dev/null +++ b/pkg/fulltext2/base_pool_test.go @@ -0,0 +1,466 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSegmentLeaseFreeIsIdempotent(t *testing.T) { + template := NewSegment("base", 0) + lease := newSegmentLease(template) + view := lease.acquire(17) + require.NotNil(t, view) + require.Equal(t, int64(17), view.Recency) + + view.Free() + view.Free() + require.NotNil(t, lease.template, "the pool reference still owns the template") + lease.retire() + require.Nil(t, lease.template) + lease.retire() +} + +func TestSegmentLeaseAcquireAfterRetire(t *testing.T) { + lease := newSegmentLease(NewSegment("base", 0)) + lease.retire() + require.Nil(t, lease.acquire(0)) +} + +func TestImmutableBasePoolSingleflightAndRetire(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + var loads atomic.Int32 + var wg sync.WaitGroup + errs := make(chan error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + view, err := p.acquire(context.Background(), key, func() (*Segment, error) { + loads.Add(1) + return NewSegment("base-0", 0), nil + }, 3) + if err != nil { + errs <- err + return + } + view.Free() + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.Equal(t, int32(1), loads.Load()) + + p.commit(key.index, nil) + require.Empty(t, p.entries) +} + +func TestImmutableBasePoolWaiterCancellationDoesNotCancelLoader(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + started := make(chan struct{}) + release := make(chan struct{}) + loaderDone := make(chan error, 1) + go func() { + view, err := p.acquire(context.Background(), key, func() (*Segment, error) { + close(started) + <-release + return NewSegment("base-0", 0), nil + }, 0) + if view != nil { + view.Free() + } + loaderDone <- err + }() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + waiterDone := make(chan error, 1) + go func() { + _, err := p.acquire(ctx, key, func() (*Segment, error) { + return nil, errors.New("canceled waiter became the loader") + }, 0) + waiterDone <- err + }() + select { + case err := <-waiterDone: + t.Fatalf("waiter returned before cancellation: %v", err) + case <-time.After(20 * time.Millisecond): + } + cancel() + select { + case err := <-waiterDone: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("canceled waiter did not return") + } + + close(release) + require.NoError(t, <-loaderDone) + p.clearAll() +} + +type doneSignalContext struct { + context.Context + doneCalled chan struct{} + once sync.Once +} + +func (c *doneSignalContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.doneCalled) }) + return c.Context.Done() +} + +func TestImmutableBasePoolWaitingOwnerSurvivesObsoleteRollback(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + loaderStarted := make(chan struct{}) + releaseLoader := make(chan struct{}) + loaderDone := make(chan error, 1) + var loads atomic.Int32 + go func() { + view, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + if loads.Add(1) == 1 { + close(loaderStarted) + <-releaseLoader + } + return NewSegment(key.id, 0), nil + }, 0, 11) + if view != nil { + view.Free() + } + loaderDone <- err + }() + <-loaderStarted + + waiting := &doneSignalContext{Context: context.Background(), doneCalled: make(chan struct{})} + replacementDone := make(chan error, 1) + go func() { + view, err := p.acquireOwned(waiting, key, func() (*Segment, error) { + loads.Add(1) + return NewSegment("replacement", 0), nil + }, 0, 22) + if view != nil { + view.Free() + } + replacementDone <- err + }() + select { + case <-waiting.doneCalled: + case <-time.After(time.Second): + t.Fatal("replacement did not reach the in-flight wait") + } + + p.mu.Lock() + _, claimed := p.entries[key].owners[22] + p.mu.Unlock() + require.True(t, claimed, "a waiting generation must claim the in-flight entry") + + p.rollback(11) + close(releaseLoader) + require.NoError(t, <-loaderDone) + require.NoError(t, <-replacementDone) + require.Equal(t, int32(1), loads.Load(), "the replacement must reuse the completed base") + require.Contains(t, p.entries, key) + + p.rollback(22) + p.clearAll() +} + +func TestImmutableBasePoolCanceledWaiterReleasesProvisionalOwner(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + loaderStarted := make(chan struct{}) + releaseLoader := make(chan struct{}) + loaderDone := make(chan error, 1) + go func() { + view, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + close(loaderStarted) + <-releaseLoader + return NewSegment(key.id, 0), nil + }, 0, 11) + if view != nil { + view.Free() + } + loaderDone <- err + }() + <-loaderStarted + + ctx, cancel := context.WithCancel(context.Background()) + waiting := &doneSignalContext{Context: ctx, doneCalled: make(chan struct{})} + waiterDone := make(chan error, 1) + go func() { + _, err := p.acquireOwned(waiting, key, func() (*Segment, error) { + return nil, errors.New("canceled waiter became the loader") + }, 0, 22) + waiterDone <- err + }() + select { + case <-waiting.doneCalled: + case <-time.After(time.Second): + t.Fatal("waiter did not reach the in-flight wait") + } + + p.mu.Lock() + _, claimed := p.entries[key].owners[22] + p.mu.Unlock() + require.True(t, claimed, "the canceled waiter must have a provisional claim to release") + + cancel() + select { + case err := <-waiterDone: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("canceled waiter did not return") + } + p.rollback(22) + p.mu.Lock() + _, claimed = p.entries[key].owners[22] + _, loaderClaimed := p.entries[key].owners[11] + p.mu.Unlock() + require.False(t, claimed) + require.True(t, loaderClaimed) + + p.rollback(11) + close(releaseLoader) + require.NoError(t, <-loaderDone) + p.clearAll() +} + +func TestImmutableBasePoolBoundsEntriesAndEvictsIdle(t *testing.T) { + p := &immutableBasePool{ + entries: make(map[baseKey]*baseEntry), + maxEntries: 1, + maxBytes: 1 << 30, + idleTTL: time.Hour, + } + for i := 0; i < 2; i++ { + key := baseKey{index: "db" + string(rune('0'+i)), id: "base", checksum: "sum", filesize: 1} + view, err := p.acquire(context.Background(), key, func() (*Segment, error) { + return NewSegment(key.id, 0), nil + }, 0) + require.NoError(t, err) + view.Free() + } + require.LessOrEqual(t, len(p.entries), 1) + + p.idleTTL = time.Nanosecond + p.evict(time.Now().Add(time.Second)) + require.Empty(t, p.entries) + p.clearAll() +} + +func TestImmutableBasePoolBoundsBytes(t *testing.T) { + p := &immutableBasePool{ + entries: make(map[baseKey]*baseEntry), + maxEntries: 8, + maxBytes: 2, + idleTTL: time.Hour, + } + for i := 0; i < 2; i++ { + key := baseKey{index: "db.store", id: string(rune('a' + i)), checksum: "sum", filesize: 2} + view, err := p.acquire(context.Background(), key, func() (*Segment, error) { + return NewSegment(key.id, 0), nil + }, 0) + require.NoError(t, err) + view.Free() + } + require.LessOrEqual(t, p.totalBytes, int64(2)) + require.LessOrEqual(t, len(p.entries), 1) + p.clearAll() +} + +func TestImmutableBasePoolLoadFailureDoesNotRetainEntry(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "broken", checksum: "sum", filesize: 1} + want := errors.New("load failed") + _, err := p.acquire(context.Background(), key, func() (*Segment, error) { + return nil, want + }, 0) + require.ErrorIs(t, err, want) + require.Empty(t, p.entries) + require.Zero(t, p.totalBytes) +} + +func TestImmutableBasePoolInvalidationDuringLoadDoesNotRepoolStaleLease(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + started := make(chan struct{}) + release := make(chan struct{}) + var loads atomic.Int32 + done := make(chan error, 1) + go func() { + view, err := p.acquire(context.Background(), key, func() (*Segment, error) { + if loads.Add(1) == 1 { + close(started) + <-release + } + return NewSegment(key.id, 0), nil + }, 0) + if view != nil { + view.Free() + } + done <- err + }() + <-started + + p.clearIndex(key.index) + close(release) + require.NoError(t, <-done) + require.Equal(t, int32(1), loads.Load(), "the invalidated in-flight result must not be retried in place") + require.Empty(t, p.entries, "the stale in-flight result is owned by the current query, not the pool") + p.clearAll() + require.Empty(t, p.entries) +} + +func TestImmutableBasePoolCommittedReplacementSurvivesObsoleteRollback(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + + first, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return NewSegment("base-0", 0), nil + }, 0, 11) + require.NoError(t, err) + first.Free() + + // The replacement reuses the same immutable bytes and commits them before + // the obsolete loader reports its later failure. Commit transfers ownership + // away from the obsolete attempt, so its rollback must not retire this entry. + replacement, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return nil, errors.New("replacement unexpectedly became the loader") + }, 0, 22) + require.NoError(t, err) + replacement.Free() + p.commitOwned(key.index, map[baseKey]struct{}{key: {}}, 22) + p.rollback(11) + + require.Contains(t, p.entries, key) + p.clearAll() +} + +func TestImmutableBasePoolReusedClaimSurvivesObsoleteRollback(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + + first, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return NewSegment("base-0", 0), nil + }, 0, 11) + require.NoError(t, err) + t.Cleanup(func() { + first.Free() + p.clearAll() + }) + + replacement, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return nil, errors.New("replacement unexpectedly became the loader") + }, 0, 22) + require.NoError(t, err) + t.Cleanup(replacement.Free) + + // The replacement has claimed the reused lease but has not published its + // generation yet. The obsolete loader must not retire that lease first. + p.rollback(11) + p.commitOwned(key.index, map[baseKey]struct{}{key: {}}, 22) + + require.Contains(t, p.entries, key) +} + +func TestImmutableBasePoolPublishedReuseFailureKeepsEntry(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + key := baseKey{index: "db.store", id: "base-0", checksum: "sum", filesize: 1} + + first, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return NewSegment("base-0", 0), nil + }, 0, 11) + require.NoError(t, err) + first.Free() + p.commitOwned(key.index, map[baseKey]struct{}{key: {}}, 11) + + replacement, err := p.acquireOwned(context.Background(), key, func() (*Segment, error) { + return nil, errors.New("published entry unexpectedly became the loader") + }, 0, 22) + require.NoError(t, err) + replacement.Free() + p.rollback(22) + + require.Contains(t, p.entries, key, "a failed reuse must not retire a published base") + p.clearAll() +} + +func TestImmutableBasePoolRollbackOwnerIsGlobalAcrossIndexes(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + firstGeneration := beginLoadGeneration("db.a") + secondGeneration := beginLoadGeneration("db.b") + require.NotEqual(t, firstGeneration.owner, secondGeneration.owner) + + firstKey := baseKey{index: "db.a", id: "base-0", checksum: "sum", filesize: 1} + secondKey := baseKey{index: "db.b", id: "base-0", checksum: "sum", filesize: 1} + first, err := p.acquireOwned(context.Background(), firstKey, func() (*Segment, error) { + return NewSegment("base-a", 0), nil + }, 0, firstGeneration.owner) + require.NoError(t, err) + first.Free() + second, err := p.acquireOwned(context.Background(), secondKey, func() (*Segment, error) { + return NewSegment("base-b", 0), nil + }, 0, secondGeneration.owner) + require.NoError(t, err) + second.Free() + + p.rollback(firstGeneration.owner) + require.NotContains(t, p.entries, firstKey) + require.Contains(t, p.entries, secondKey) + p.clearAll() + endLoadGeneration(firstGeneration) + endLoadGeneration(secondGeneration) +} + +func TestLoadReasonRegistryIsDatabaseQualified(t *testing.T) { + cleanup := setLoadObserver(func(LoadEvent) {}) + defer cleanup() + key1 := loadReasonKey("db1", "store") + key2 := loadReasonKey("db2", "store") + rememberLoadReason(key1, LoadMissCDCFlush) + rememberLoadReason(key2, LoadMissMerge) + reason, generation := peekLoadReason(key1) + require.Equal(t, LoadMissCDCFlush, reason) + consumeLoadReason(key1, generation) + reason, generation = peekLoadReason(key2) + require.Equal(t, LoadMissMerge, reason) + consumeLoadReason(key2, generation) + reason, generation = peekLoadReason(key1) + require.Empty(t, reason) + require.Zero(t, generation) +} + +func TestLoadReasonRegistryIsDisabledWithObserverOff(t *testing.T) { + cleanup := setLoadObserver(nil) + defer cleanup() + key := loadReasonKey("db", "store") + rememberLoadReason(key, LoadMissCDCFlush) + reason, generation := peekLoadReason(key) + require.Empty(t, reason) + require.Zero(t, generation) +} diff --git a/pkg/fulltext2/boolean.go b/pkg/fulltext2/boolean.go new file mode 100644 index 0000000000000..f87853df18c74 --- /dev/null +++ b/pkg/fulltext2/boolean.go @@ -0,0 +1,590 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math/bits" + + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" +) + +// docBitset is a dense per-doc-ord bitset (doc ords are dense in [0, N) per segment), +// used by the boolean evaluator instead of map[int64]struct{} — no rehash, ~1/64 the +// memory, and forEach yields ords ascending. +type docBitset []uint64 + +func newDocBitset(n int) docBitset { return make(docBitset, (n+63)/64) } +func (b docBitset) set(i int) { b[i>>6] |= uint64(1) << (uint(i) & 63) } +func (b docBitset) get(i int) bool { return b[i>>6]&(uint64(1)<<(uint(i)&63)) != 0 } + +// forEach calls f with each set bit index, ascending. +func (b docBitset) forEach(f func(int)) { + for w, word := range b { + base := w << 6 + for word != 0 { + f(base + bits.TrailingZeros64(word)) + word &= word - 1 + } + } +} + +// Boolean-mode search (MATCH … AGAINST('…' IN BOOLEAN MODE)). +// +// MatrixOne/MySQL boolean semantics, mapped onto the positional segment: +// - +clause → MUST (AND): the doc must contain it. +// - -clause → MUST NOT (the doc must not contain it). +// - >clause → SHOULD, impact ×1.1 (boost). +// - < ~ weight operators (§6). +const ( + weightGreater = 1.1 + weightLess = 0.9 + weightTilde = -1.0 +) + +type clause struct { + kind clauseKind + terms []string // leaf (term / prefix) + phrase []phraseSlot // clausePhrase: positional (byte-offset) slots + children []clause // group + weight float32 // impact multiplier (1.0 default; > < ~ change it) +} + +// BoolQuery is a parsed boolean-mode query. adjust holds the ~ clauses, which +// adjust score without driving inclusion. +type BoolQuery struct { + must []clause // + + mustNot []clause // - + should []clause // bare / > / < + adjust []clause // ~ +} + +// SearchBoolean evaluates a boolean query and returns up to k hits, score desc +// (ties by ascending doc ord). A doc matches when every MUST clause is present, +// no MUST-NOT clause is present, and — with no MUST — at least one SHOULD clause +// is present. Score sums the MUST + SHOULD contributions, then applies ~ ADJUST +// penalties to the matched docs. +// +// A pure disjunction of single terms (bare/>/< SHOULD terms, no MUST/MUST-NOT/~, +// no phrase/prefix/group) is routed to the WAND top-k (searchWAND), which +// early-terminates via term max-impact bounds and returns the identical ranking +// as the full scan. Everything else uses the full evaluator. +func (s *Segment) SearchBoolean(q BoolQuery, algo ScoreAlgo, k int, allow Membership, gs *globalStats) ([]Result, error) { + // A query with no clauses at all returns nothing. A ~-only query (adjust, no MUST/ + // SHOULD) still returns its matching rows ranked low — classic/MySQL "~ rates lower + // but does not exclude" — so it must NOT early-out here (searchBooleanFull's no-MUST + // candidate set unions the adjust terms). + if k <= 0 || s.N == 0 || (len(q.must) == 0 && len(q.should) == 0 && len(q.adjust) == 0) { + return nil, nil + } + if terms, ok := disjunctiveTerms(q); ok { + return s.searchWAND(terms, algo, k, allow, gs), nil + } + return s.searchBooleanFull(q, algo, k, allow, gs) +} + +// disjunctiveTerms reports whether q is a pure OR of single-term SHOULD clauses +// (the WAND-eligible shape) and returns those clauses. +func disjunctiveTerms(q BoolQuery) ([]clause, bool) { + if len(q.must) != 0 || len(q.mustNot) != 0 || len(q.adjust) != 0 || len(q.should) == 0 { + return nil, false + } + for _, c := range q.should { + if c.kind != clauseTerm { + return nil, false + } + } + return q.should, true +} + +// evalBoolean runs ONE segment's boolean evaluation against a DENSE per-doc-ord accumulator +// (score []float32 + doc bitsets) rather than a map[int64]float64 per clause plus a +// candidate map, and calls emit(ord, score) for every doc that satisfies the query (all +// MUST present, no MUST-NOT, and — with no MUST — at least one SHOULD/ADJUST) AND passes the +// WHERE/liveness prefilter. Doc ords are dense in [0, N) per segment, so the arrays are O(N) +// with no rehash — a low-selectivity 2-char CJK term matches most of the corpus, and the old +// maps then rehashed themselves to death and held ~30 bytes/doc × (#clauses+1). Peak Go heap +// is O(segment N) (the dense score array + candidate/must bitsets), bounded by the segment +// capacity and freed when the call returns — INDEPENDENT of k. Both the top-k +// searchBooleanFull and the no-LIMIT streaming path (Index.streamBoolean) build on this; +// they differ only in what emit does (bounded heap push vs. stream to the sink). +func (s *Segment) evalBoolean(q BoolQuery, algo ScoreAlgo, allow Membership, gs *globalStats, emit func(ord int, score float32)) error { + n := int(s.N) + if n == 0 { + return nil + } + avgDocLen := gs.avgdl(s) + score := make([]float32, n) + + // MUST-NOT: presence only, into an exclusion bitset. + mustNot := newDocBitset(n) + for _, c := range q.mustNot { + if err := s.evalClauseInto(c, algo, avgDocLen, gs, false, func(ord int64, _ float32) { + mustNot.set(int(ord)) + }); err != nil { + return err + } + } + + admit := func(ord int) { + if mustNot.get(ord) || !allowed(allow, int64(ord)) { // WHERE prefilter (nil = allow all) + return + } + emit(ord, score[ord]) + } + + if len(q.must) > 0 { + // A candidate must satisfy EVERY MUST clause: count hits per doc (intersection). + mustHit := make([]int32, n) + touched := newDocBitset(n) + for _, c := range q.must { + if err := s.evalClauseInto(c, algo, avgDocLen, gs, true, func(ord int64, sc float32) { + score[ord] += sc + mustHit[ord]++ + touched.set(int(ord)) + }); err != nil { + return err + } + } + // SHOULD/ADJUST add to the score of whatever docs they touch; only MUST-satisfying + // docs are collected, so contributions to non-candidates are simply never read. + for _, cs := range [][]clause{q.should, q.adjust} { + for _, c := range cs { + if err := s.evalClauseInto(c, algo, avgDocLen, gs, true, func(ord int64, sc float32) { + score[ord] += sc + }); err != nil { + return err + } + } + } + need := int32(len(q.must)) + touched.forEach(func(ord int) { + if mustHit[ord] == need { + admit(ord) + } + }) + return nil + } + + // No MUST: the candidate set is the union of SHOULD *and* ADJUST. A ~ (ADJUST) term + // lowers a row's score but does NOT exclude it (MySQL: "rated lower ... but not + // excluded, as with -"), so a row matching only a ~term is still a candidate, ranked + // low by its negative contribution — matching classic fulltext. + cand := newDocBitset(n) + for _, cs := range [][]clause{q.should, q.adjust} { + for _, c := range cs { + if err := s.evalClauseInto(c, algo, avgDocLen, gs, true, func(ord int64, sc float32) { + score[ord] += sc + cand.set(int(ord)) + }); err != nil { + return err + } + } + } + cand.forEach(admit) + return nil +} + +// searchBooleanFull returns the segment's top-k boolean hits by feeding evalBoolean's +// admitted candidates into a bounded top-K heap (score desc; equal scores by ascending ord). +func (s *Segment) searchBooleanFull(q BoolQuery, algo ScoreAlgo, k int, allow Membership, gs *globalStats) ([]Result, error) { + if s.N == 0 || k <= 0 { + return nil, nil + } + h := newTopKHeap(k, s.N) + if err := s.evalBoolean(q, algo, allow, gs, func(ord int, score float32) { + h.Push(int64(ord), -score) + }); err != nil { + return nil, err + } + return heapToResults(s, h), nil +} + +// evalClauseInto streams one clause's weighted (doc, score) contributions to emit, WITHOUT +// building a per-clause map. Term and phrase clauses touch each doc once, so they emit +// directly (weight folded in); prefix and group clauses need a per-doc MAX over their +// expansion/children, so they fall back to evalClause's map (rare, and bounded by the +// prefix's terms). scored=false emits presence only (score 0), for MUST-NOT. +func (s *Segment) evalClauseInto(c clause, algo ScoreAlgo, avgDocLen float64, gs *globalStats, scored bool, emit func(int64, float32)) error { + w := c.weight + switch c.kind { + case clauseTerm: + pl, ok := s.lookup(c.terms[0]) + if !ok { + return nil + } + docs := pl.materializeDocIDs() + if !scored { + for _, ord := range docs { + emit(ord, 0) + } + return nil + } + idf2 := gs.idfFor(s, c.terms[0], pl) + tfs := pl.materializeTfs() + for i, ord := range docs { + emit(ord, w*s.scoreTerm(algo, float64(tfs[i]), idf2, ord, avgDocLen)) + } + case clausePhrase: + var hits []docTf + if gs != nil { + hits = gs.phraseHits(s, c.phrase) // memoized; shared with gs.phraseDf + } else { + hits = s.matchPhrase(c.phrase) + } + if !scored { + for _, hh := range hits { + emit(hh.ord, 0) + } + return nil + } + idf2 := gs.phraseIdfFor(s, c.phrase, len(hits)) + for _, hh := range hits { + emit(hh.ord, w*s.scoreTerm(algo, float64(hh.tf), idf2, hh.ord, avgDocLen)) + } + default: // clausePrefix, clauseGroup: per-doc MAX needs a temp (evalClause applies weight) + m, err := s.evalClause(c, algo, avgDocLen, gs, scored) + if err != nil { + return err + } + for ord, sc := range m { + emit(ord, sc) + } + } + return nil +} + +// evalClause returns the weighted (doc ord → score) contribution of one clause. +// A present doc is always a key (even at score 0, so MUST/presence is exact); an +// absent term yields an empty map. The clause weight (> < ~) scales the result. +// Term / prefix / phrase clauses all score with the global corpus idf (gs) — a phrase +// clause via the cross-segment phrase df (gs.phraseDf), so ranking is consistent +// across a base + CDC tail. +// +// scored=false skips the per-doc score computation (idf + scoreTerm) and returns a +// presence-only map (all 0 values). MUST-NOT clauses use this: only the ord set is read +// there, so scoring every doc of a common negated term (e.g. `-the`) was wasted work. +func (s *Segment) evalClause(c clause, algo ScoreAlgo, avgDocLen float64, gs *globalStats, scored bool) (map[int64]float32, error) { + raw := make(map[int64]float32) + switch c.kind { + case clauseTerm: + if pl, ok := s.lookup(c.terms[0]); ok { + docs := pl.materializeDocIDs() + if !scored { + for _, ord := range docs { + raw[ord] = 0 + } + break + } + idf2 := gs.idfFor(s, c.terms[0], pl) + tfs := pl.materializeTfs() + for i, ord := range docs { + // tf-aware BM25 (real within-doc occurrence count). Classic fulltext scores + // boolean matches with tf≡1, but ONLY because its SQL retrieval collapses + // GROUP BY doc_id to avoid a JOIN row blow-up (sql.go:493-496) — a + // restriction fulltext2's in-memory scorer does not have. Keeping tf gives + // better ranking among matched docs (BM25 saturation, k1, tames repetition) + // and tighter WAND max-impact bounds. This is an intentional divergence. + raw[ord] = s.scoreTerm(algo, float64(tfs[i]), idf2, ord, avgDocLen) + } + } + case clausePhrase: + var hits []docTf + if gs != nil { + hits = gs.phraseHits(s, c.phrase) // memoized; shared with gs.phraseDf + } else { + hits = s.matchPhrase(c.phrase) + } + if !scored { + for _, h := range hits { + raw[h.ord] = 0 + } + break + } + idf2 := gs.phraseIdfFor(s, c.phrase, len(hits)) + for _, h := range hits { + // tf-aware: real phrase-occurrence count (see clauseTerm for why fulltext2 keeps + // tf where classic's SQL forced tf≡1). + raw[h.ord] = s.scoreTerm(algo, float64(h.tf), idf2, h.ord, avgDocLen) + } + case clausePrefix: + terms, err := s.prefixTerms(c.terms[0]) + if err != nil { + return nil, err + } + for _, t := range terms { // combined impact = MAX over expanded terms (§6) + pl, ok := s.lookup(t) + if !ok { + continue + } + docs := pl.materializeDocIDs() + if !scored { + for _, ord := range docs { + raw[ord] = 0 + } + continue + } + idf2 := gs.idfFor(s, t, pl) + tfs := pl.materializeTfs() + for i, ord := range docs { + sc := s.scoreTerm(algo, float64(tfs[i]), idf2, ord, avgDocLen) // tf-aware (see clauseTerm) + if cur, seen := raw[ord]; !seen || sc > cur { + raw[ord] = sc + } + } + } + case clauseGroup: + for _, ch := range c.children { // OR of children, score = MAX (§6) + m, err := s.evalClause(ch, algo, avgDocLen, gs, scored) + if err != nil { + return nil, err + } + for ord, sc := range m { + if cur, seen := raw[ord]; !seen || sc > cur { + raw[ord] = sc + } + } + } + } + if c.weight != 1.0 { + for ord := range raw { + raw[ord] *= c.weight + } + } + return raw, nil +} + +// prefixTerms expands a word* prefix to its matching terms, over the loaded FST +// or the build-side sorted key list. +func (s *Segment) prefixTerms(prefix string) ([]string, error) { + if s.dict != nil { + return s.dict.prefixTerms(prefix) + } + return s.PrefixRange(prefix), nil +} + +// SearchBooleanText parses query in boolean mode (tokenizing with tok, the index's +// tokenizer) and evaluates it — the convenience entry mirroring +// MATCH(col) AGAINST('query' IN BOOLEAN MODE). +func (s *Segment) SearchBooleanText(query []byte, tok tokenizer.Tokenizer, algo ScoreAlgo, k int) ([]Result, error) { + q, err := ParseBoolean(query, tok) + if err != nil { + return nil, err + } + return s.SearchBoolean(q, algo, k, nil, nil) // convenience entry: no prefilter, local stats +} + +// ---- parser ---- + +// bucket classifies a clause by its prefix operator. +type bucket int + +const ( + bShould bucket = iota + bMust + bMustNot + bAdjust +) + +func bucketOf(prefix byte) bucket { + switch prefix { + case '+': + return bMust + case '-': + return bMustNot + case '~': + return bAdjust + default: // '>' '<' 0 + return bShould + } +} + +func weightOf(prefix byte) float32 { + switch prefix { + case '>': + return weightGreater + case '<': + return weightLess + case '~': + return weightTilde + default: + return 1.0 + } +} + +// ParseBoolean parses a boolean-mode query string into clauses, tokenizing each +// leaf's text with tok. Surface: leading +/-/>/ < ~ or 0 + quoted bool + star bool + group bool + text string + children []rawClause +} + +func isPrefixChar(c byte) bool { + return c == '+' || c == '-' || c == '>' || c == '<' || c == '~' +} + +// scanBoolean splits a boolean query into raw clauses at the top level. +func scanBoolean(q string) []rawClause { + cs, _ := scanRange(q, 0) + return cs +} + +// scanRange scans clauses from index i until end-of-string or an unmatched ')' +// (a group close), returning the clauses and the index just past the stop point. +func scanRange(q string, i int) ([]rawClause, int) { + var out []rawClause + n := len(q) + for i < n { + for i < n && q[i] == ' ' { + i++ + } + if i >= n { + break + } + if q[i] == ')' { + return out, i + 1 // end of this group + } + var rc rawClause + if isPrefixChar(q[i]) { + rc.prefix = q[i] + i++ + } + switch { + case i < n && q[i] == '(': + rc.group = true + i++ + rc.children, i = scanRange(q, i) + case i < n && q[i] == '"': + rc.quoted = true + i++ + start := i + for i < n && q[i] != '"' { + i++ + } + rc.text = q[start:i] + if i < n { + i++ // closing quote + } + default: + start := i + for i < n && q[i] != ' ' && q[i] != ')' { + i++ + } + word := q[start:i] + if len(word) > 0 && word[len(word)-1] == '*' { + rc.star = true + word = word[:len(word)-1] + } + rc.text = word + } + if rc.group || rc.text != "" { + out = append(out, rc) + } + } + return out, i +} diff --git a/pkg/fulltext2/boolean_parity_test.go b/pkg/fulltext2/boolean_parity_test.go new file mode 100644 index 0000000000000..5c7fac03e72b8 --- /dev/null +++ b/pkg/fulltext2/boolean_parity_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Boolean-mode PARITY with classic fulltext (pkg/fulltext/fulltext_test.go): +// fulltext2 uses its own parser + WAND evaluator, but must produce the same +// MySQL boolean-mode membership as the classic engine for the same queries +// (+apple banana, +apple -banana, +apple ~banana, nested +(...), CJK +/-). +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// classicCorpus mirrors the apple/banana doc shape used by classic fulltext's +// TestFullTextPlusOr / TestFullTextMinus / TestFullTextTilda (doc 0 has both, +// doc 1 apple-only, docs 11/12 banana-only), plus a matrix/origin group set. +func classicCorpus(t *testing.T) *Segment { + docs := []Doc{ + {int64(0), []byte("apple banana")}, + {int64(1), []byte("apple apple apple")}, + {int64(11), []byte("banana banana banana")}, + {int64(12), []byte("banana banana banana banana")}, + {int64(20), []byte("matrix origin")}, + {int64(21), []byte("matrix one")}, + {int64(22), []byte("matrix two")}, + {int64(23), []byte("matrix alone")}, + {int64(24), []byte("origin standalone")}, + } + s, err := BuildSegmentFromDocs("classic", int32(types.T_int64), docs, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + return s +} + +// TestParityPlusOr: "+apple banana" — apple MUST, banana SHOULD (OR). Docs with +// apple match (0,1); banana-only docs (11,12) do NOT. (classic TestFullTextPlusOr) +func TestParityPlusOr(t *testing.T) { + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, classicCorpus(t), "+apple banana")) +} + +// TestParityMinus: "+apple -banana" — apple MUST, banana MUST-NOT. Only doc 1 +// (apple without banana). (classic TestFullTextMinus) +func TestParityMinus(t *testing.T) { + require.Equal(t, []any{int64(1)}, bquery(t, classicCorpus(t), "+apple -banana")) +} + +// TestParityTilde: "+apple ~banana" — apple MUST, banana a penalty (not an +// exclusion): docs 0 AND 1 match, 0 ranked below 1. (classic TestFullTextTilda) +func TestParityTilde(t *testing.T) { + got := bqueryOrdered(t, classicCorpus(t), "+apple ~banana") + require.ElementsMatch(t, []any{int64(0), int64(1)}, got) + require.Equal(t, int64(1), got[0], "the banana-free doc ranks first under ~") +} + +// TestParityNestedGroup: "+matrix +(origin (one two))" — matrix AND (origin OR +// one OR two). Nested groups flatten to an OR. (classic TestPatternBoolean shape) +func TestParityNestedGroup(t *testing.T) { + require.ElementsMatch(t, []any{int64(20), int64(21), int64(22)}, + bquery(t, classicCorpus(t), "+matrix +(origin (one two))")) +} + +// TestParityPlusPlus: "+apple +banana" — both MUST → only doc 0. +func TestParityPlusPlus(t *testing.T) { + require.Equal(t, []any{int64(0)}, bquery(t, classicCorpus(t), "+apple +banana")) +} diff --git a/pkg/fulltext2/boolean_stream_test.go b/pkg/fulltext2/boolean_stream_test.go new file mode 100644 index 0000000000000..8a293ba8e8a49 --- /dev/null +++ b/pkg/fulltext2/boolean_stream_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "errors" + "reflect" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// TestStreamBoolean pins the no-LIMIT boolean streaming path (Index.streamBoolean): a +// NON-disjunctive boolean query (MUST / MUST-NOT / mixed phrase) must stream EXACTLY the +// (pk, score) set the materialized SearchQuery(k=all) produces — one segment at a time, +// with no global top-K sized to globalN — must dedup a pk across the base + CDC tail +// (per-segment liveness), and must propagate an emit error. This is precisely the shape +// that used to fall back to SearchQuery(globalN) and materialize the whole live corpus +// before emitting; +alpha (matches every doc) exercises the low-selectivity case. +func TestStreamBoolean(t *testing.T) { + split := NewIndex([]*Segment{ + buildFtSeg(t, "a", 0, 0, 12), + buildFtSeg(t, "b", 1, 12, 24), + }, nil) + + cases := []struct { + pat string + nonEmpty bool + }{ + {"+alpha", true}, // MUST, matches EVERY doc (low selectivity) + {"+alpha beta", true}, // MUST + SHOULD + {"+alpha -delta", true}, // MUST + MUST-NOT + {"alpha -gamma", true}, // SHOULD + MUST-NOT (non-disjunctive via -) + {"+beta gamma", true}, // MUST + SHOULD + {`+alpha "beta gamma"`, false}, // MUST + phrase (mixed) — parity even if empty + } + + // Guard: every case is genuinely non-disjunctive, so StreamQuery routes it to + // streamBoolean (NOT streamWAND / streamPhrase). If a future refactor made one + // disjunctive, this test would silently exercise the wrong path — so pin it here. + for _, c := range cases { + q, err := buildBooleanQuery(c.pat, normalizeParser(ParserDefault)) + require.NoErrorf(t, err, "build %q", c.pat) + _, disj := disjunctiveTerms(q) + require.Falsef(t, disj, "%q must be non-disjunctive (exercise streamBoolean)", c.pat) + } + + collect := func(pattern string) map[int64]float64 { + m := map[int64]float64{} + err := split.StreamQuery([]byte(pattern), true, ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { + require.LessOrEqual(t, o.Keys.N, streamBatch) + for i, k := range int64ColumnBuffer(o.Keys) { + _, dup := m[k] + require.Falsef(t, dup, "pk %d emitted twice", k) // liveness dedup + m[k] = float64(o.Dists[i]) + } + return nil + }) + require.NoError(t, err) + return m + } + + for _, c := range cases { + want, err := split.SearchQuery([]byte(c.pat), true, ParserDefault, BM25, 1000, nil) + require.NoError(t, err) + got := collect(c.pat) + require.Equalf(t, len(want), len(got), "%q: same match count", c.pat) + for _, r := range want { + g, ok := got[r.Pk.(int64)] + require.Truef(t, ok, "%q: pk %v streamed", c.pat, r.Pk) + require.InDeltaf(t, r.Score, g, 1e-5, "%q: pk %v score", c.pat, r.Pk) + } + if c.nonEmpty { + require.NotEmptyf(t, got, "%q: expected a non-empty streamed result", c.pat) + } + } + + // emit-error path: the callback error propagates and stops the boolean walk. + sentinel := errors.New("consumer aborted") + err := split.StreamQuery([]byte("+alpha beta"), true, ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { return sentinel }) + require.ErrorIs(t, err, sentinel) +} + +// buildPhraseSeg builds a segment [lo,hi) where EVERY doc is "common foo bar" — so the +// term "common" and the contiguous phrase "foo bar" each match every doc (low selectivity). +func buildPhraseSeg(t *testing.T, id string, rec int64, lo, hi int) *Segment { + b := NewBuilder(id, int32(types.T_int64)) + for i := lo; i < hi; i++ { + feed(t, b, int64(i), "common", "foo", "bar") + } + s, err := b.Finish() + require.NoError(t, err) + s.Recency = rec + return s +} + +// TestStreamBooleanMixedPhraseLowSelectivity is the bounded-memory regression for the +// no-LIMIT streaming path: a mixed boolean query with a LOW-SELECTIVITY phrase over MANY +// segments must stream correctly WITHOUT retaining every segment's phrase hits. The old +// phraseHitsCache held one []docTf per (phrase, segment) for the whole query, so the first +// segment's scoring — which triggers the cross-segment phraseDf scan — materialized O(the +// global matching corpus) before a single row could be emitted (CN OOM). The phrase here +// matches ALL 120 docs across 3 segments; the streamed result must equal the materialized +// SearchQuery and be non-empty. (Correctness of count-and-discard phraseDf: the global +// phrase idf must still be identical to the retaining version — verified via score parity.) +func TestStreamBooleanMixedPhraseLowSelectivity(t *testing.T) { + idx := NewIndex([]*Segment{ + buildPhraseSeg(t, "a", 0, 0, 40), + buildPhraseSeg(t, "b", 1, 40, 80), + buildPhraseSeg(t, "c", 2, 80, 120), + }, nil) + + const pat = `+common "foo bar"` // MUST term + phrase (mixed, non-disjunctive → streamBoolean) + q, err := buildBooleanQuery(pat, normalizeParser(ParserDefault)) + require.NoError(t, err) + _, disj := disjunctiveTerms(q) + require.False(t, disj, "must route to streamBoolean") + + got := map[int64]float64{} + err = idx.StreamQuery([]byte(pat), true, ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { + require.LessOrEqual(t, o.Keys.N, streamBatch) + for i, k := range int64ColumnBuffer(o.Keys) { + _, dup := got[k] + require.Falsef(t, dup, "pk %d emitted twice", k) + got[k] = float64(o.Dists[i]) + } + return nil + }) + require.NoError(t, err) + + want, err := idx.SearchQuery([]byte(pat), true, ParserDefault, BM25, 1000, nil) + require.NoError(t, err) + require.NotEmpty(t, got) // non-empty, low selectivity + require.Equal(t, 120, len(got)) // the phrase matches every doc across all 3 segments + require.Equal(t, len(want), len(got)) // parity with the materialized path + for _, r := range want { + g, ok := got[r.Pk.(int64)] + require.Truef(t, ok, "pk %v streamed", r.Pk) + require.InDeltaf(t, r.Score, g, 1e-5, "pk %v score parity (global phrase idf unchanged)", r.Pk) + } +} + +// TestGlobalStatsNoPhraseHitRetention structurally guards the fix above: global phrase DF +// must be count-and-discard, so globalStats must NOT hold a field that retains per-segment +// matchPhrase slices ([]docTf). Reintroducing a cross-segment []docTf cache (the removed +// phraseHitsCache) would defeat the no-LIMIT streaming O(one segment) bound. +func TestGlobalStatsNoPhraseHitRetention(t *testing.T) { + typ := reflect.TypeOf(globalStats{}) + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + require.NotContainsf(t, f.Type.String(), "docTf", + "globalStats.%s (%s) retains phrase hit slices; global phrase DF must count-and-discard, not cache []docTf", + f.Name, f.Type) + } +} diff --git a/pkg/fulltext2/boolean_test.go b/pkg/fulltext2/boolean_test.go new file mode 100644 index 0000000000000..a5b3f38b20e0e --- /dev/null +++ b/pkg/fulltext2/boolean_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +func bquery(t *testing.T, s *Segment, q string) []any { + t.Helper() + rs, err := s.SearchBooleanText([]byte(q), tokenizer.NewSimpleTokenizer(), TfIdf, 10) + require.NoError(t, err) + return pkslice(rs) +} + +// TestBooleanMode covers the + - " " * and bare-OR surface over the shared +// fulltext corpus. +func TestBooleanMode(t *testing.T) { + s := fulltextCorpus(t) + + // MUST (AND) + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, s, "+quick +fox")) + // MUST + MUST-NOT + require.Equal(t, []any{int64(2)}, bquery(t, s, "+quick -jumps")) + // SHOULD (OR) minus MUST-NOT + require.ElementsMatch(t, []any{int64(2), int64(4)}, bquery(t, s, "fox -dog")) + // a required absent term → no matches + require.Empty(t, bquery(t, s, "+fox +cat")) + + // quoted phrase (bare) and required phrase + require.ElementsMatch(t, []any{int64(1), int64(3)}, bquery(t, s, `"lazy dog"`)) + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, s, `+"quick brown fox"`)) + + // bare OR of two separate words + require.ElementsMatch(t, []any{int64(3), int64(4)}, bquery(t, s, "sleeps high")) + + // prefix: qui* → quick + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, s, "qui*")) + // prefix in a MUST, combined with another MUST: (day|dog) AND fox → docs 1,5 + require.ElementsMatch(t, []any{int64(1), int64(5)}, bquery(t, s, "+d* +fox")) +} + +// TestBooleanNonContiguousPhrase: a phrase clause still requires adjacency. +func TestBooleanNonContiguousPhrase(t *testing.T) { + s := fulltextCorpus(t) + // "quick fox" as a phrase is never adjacent (brown between) → no matches, + // even though both terms exist. + require.Empty(t, bquery(t, s, `+"quick fox"`)) +} + +// TestBooleanOnLoadedSegment exercises the loaded (FST) path, including prefix +// expansion over the FST iterator. +func TestBooleanOnLoadedSegment(t *testing.T) { + data, err := fulltextCorpus(t).Serialize() + require.NoError(t, err) + loaded, err := Deserialize("seg", bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + require.Equal(t, []any{int64(2)}, bquery(t, loaded, "+quick -jumps")) + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, loaded, "qui*")) + require.ElementsMatch(t, []any{int64(1), int64(5)}, bquery(t, loaded, "+d* +fox")) +} + +// bqueryOrdered runs a boolean query and returns pks in ranked order. +func bqueryOrdered(t *testing.T, s *Segment, q string) []any { + t.Helper() + rs, err := s.SearchBooleanText([]byte(q), tokenizer.NewSimpleTokenizer(), TfIdf, 10) + require.NoError(t, err) + return pkslice(rs) +} + +// TestBooleanGroups: ( ) groups — OR of children (max sub-scorer), combinable +// with + and nestable. +func TestBooleanGroups(t *testing.T) { + s := fulltextCorpus(t) + // (quick OR sleeps) AND fox + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, s, "+(quick sleeps) +fox")) + // (lazy OR high) AND fox + require.ElementsMatch(t, []any{int64(1), int64(4)}, bquery(t, s, "+(lazy high) +fox")) + // bare group = OR + require.ElementsMatch(t, []any{int64(3), int64(4)}, bquery(t, s, "(sleeps high)")) + // nested group + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4), int64(5)}, bquery(t, s, "+((quick) fox)")) +} + +// TestBooleanWeights: > boosts and < demotes a SHOULD clause's rank without +// changing membership. +func TestBooleanWeights(t *testing.T) { + s := fulltextCorpus(t) + // membership is unchanged by a weight operator + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, bquery(t, s, ">quick")) + + // sleeps(doc3) and high(doc4) score equally → tied, order unspecified. + require.ElementsMatch(t, []any{int64(3), int64(4)}, bqueryOrdered(t, s, "sleeps high")) + // boosting high gives it a strictly higher score, so it ranks first: [4,3]. + require.Equal(t, []any{int64(4), int64(3)}, bqueryOrdered(t, s, "sleeps >high")) +} + +// TestBooleanTilde: ~ penalizes but does not exclude — jumps-bearing docs (1,4) +// sink below the others (2,5), all still present. +func TestBooleanTilde(t *testing.T) { + s := fulltextCorpus(t) + // Two equal-score groups: {2,5} (fox, unpenalized) rank above {1,4} (fox + ~jumps + // penalty); order WITHIN each tied group is unspecified. + got := bqueryOrdered(t, s, "fox ~jumps") + require.Len(t, got, 4) + require.ElementsMatch(t, []any{int64(2), int64(5)}, got[:2]) + require.ElementsMatch(t, []any{int64(1), int64(4)}, got[2:]) +} + +// TestBooleanGroupsLoaded exercises groups/weights on the loaded (FST) path. +func TestBooleanGroupsLoaded(t *testing.T) { + data, err := fulltextCorpus(t).Serialize() + require.NoError(t, err) + loaded, err := Deserialize("seg", bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + require.ElementsMatch(t, []any{int64(1), int64(4)}, bquery(t, loaded, "+(lazy high) +fox")) + // {2,5} rank above {1,4}; order within each tied group is unspecified (see TestBooleanTilde). + got := bqueryOrdered(t, loaded, "fox ~jumps") + require.Len(t, got, 4) + require.ElementsMatch(t, []any{int64(2), int64(5)}, got[:2]) + require.ElementsMatch(t, []any{int64(1), int64(4)}, got[2:]) +} + +// TestScanBoolean checks the top-level clause scanner directly. +func TestScanBoolean(t *testing.T) { + got := scanBoolean(`+quick -"lazy dog" fox*`) + require.Len(t, got, 3) + require.Equal(t, rawClause{prefix: '+', text: "quick"}, got[0]) + require.Equal(t, rawClause{prefix: '-', quoted: true, text: "lazy dog"}, got[1]) + require.Equal(t, rawClause{star: true, text: "fox"}, got[2]) +} + +// TestScanBooleanGroups: nested groups and weight prefixes scan correctly. +func TestScanBooleanGroups(t *testing.T) { + got := scanBoolean(`>quick +(lazy dog) ~jumps`) + require.Len(t, got, 3) + require.Equal(t, byte('>'), got[0].prefix) + require.Equal(t, "quick", got[0].text) + require.Equal(t, byte('+'), got[1].prefix) + require.True(t, got[1].group) + require.Len(t, got[1].children, 2) + require.Equal(t, "lazy", got[1].children[0].text) + require.Equal(t, "dog", got[1].children[1].text) + require.Equal(t, byte('~'), got[2].prefix) + require.Equal(t, "jumps", got[2].text) +} diff --git a/pkg/fulltext2/boolean_tree_test.go b/pkg/fulltext2/boolean_tree_test.go new file mode 100644 index 0000000000000..38d92d4d27c0c --- /dev/null +++ b/pkg/fulltext2/boolean_tree_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Boolean query-tree printer — ported verbatim from classic fulltext +// (pkg/fulltext/fulltext_test.go TestPatternBoolean / TestPatternPhrase). +// BooleanTreeString reuses fulltext.PatternToString, so the printed S-expression +// is identical to classic. +package fulltext2 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBooleanTreeString(t *testing.T) { + cases := []struct { + pattern string + expect string + }{ + {"Ma'trix Origin", "(text 0 ma'trix) (text 1 origin)"}, + {"Matrix Origin", "(text 0 matrix) (text 1 origin)"}, + {"+Matrix Origin", "(+ (text 0 matrix)) (text 1 origin)"}, + {"+Matrix -Origin", "(+ (text 0 matrix)) (- (text 1 origin))"}, + {"Matrix ~Origin", "(text 0 matrix) (~ (text 1 origin))"}, + {"Matrix +(One)", "(+ (group (< (text 0 origin)) (> (text 1 one)))) (text 2 matrix)"}, + {"+Matrix +Origin", "(join 0 (+ (text 0 matrix)) (+ (text 0 origin)))"}, + {`"Matrix origin"`, "(phrase (text 0 matrix) (text 1 origin))"}, + {"Matrix Origin*", "(text 0 matrix) (* 1 origin*)"}, + {"+Matrix +(Origin (One Two))", "(+ (text 0 matrix)) (+ (group (text 1 origin) (group (text 2 one) (text 3 two))))"}, + {"+读写汉字 -学中文", "(+ (text 0 读写汉字)) (- (text 1 学中文))"}, + {"+读书会 +提效 +社群 +案例 +运营", "(join 0 (+ (text 0 读书会)) (+ (text 0 提效)) (+ (text 0 社群)) (+ (text 0 案例)) (+ (text 0 运营)))"}, + } + for _, c := range cases { + got, err := BooleanTreeString(c.pattern) + require.NoError(t, err, c.pattern) + require.Equal(t, c.expect, got, "pattern %q", c.pattern) + } +} + +func TestBooleanTreeStringWithPosition(t *testing.T) { + cases := []struct { + pattern string + expect string + }{ + {`"Ma'trix Origin"`, "(phrase (text 0 0 ma'trix) (text 1 12 origin))"}, + {`"Matrix Origin"`, "(phrase (text 0 0 matrix) (text 1 7 origin))"}, + {`"Matrix"`, "(phrase (text 0 0 matrix))"}, + {`" Matrix "`, "(phrase (text 0 0 matrix))"}, + {`"Matrix Origin"`, "(phrase (text 0 0 matrix) (text 1 11 origin))"}, + {`" 你好嗎? Hello World 在一起 Happy 再见 "`, + "(phrase (text 0 0 你好嗎?) (text 1 11 hello) (text 2 17 world) (text 3 24 在一起) (text 4 35 happy) (text 5 42 再见))"}, + } + for _, c := range cases { + got, err := BooleanTreeStringWithPosition(c.pattern) + require.NoError(t, err, c.pattern) + require.Equal(t, c.expect, got, "pattern %q", c.pattern) + } +} + +// Ported from classic TestPatternNL: NL-mode ngram/word expansion with positions +// (short tokens → * prefix, CJK → 3-grams, mixed CJK/latin). +func TestNLTreeStringWithPosition(t *testing.T) { + cases := []struct { + pattern string + expect string + }{ + {"Ma'trix Origin", "(* 0 0 ma*) (text 1 3 trix) (text 2 8 origin)"}, + {"Matrix Origin", "(text 0 0 matrix) (text 1 7 origin)"}, + {"读写汉字 学中文", "(text 0 0 读写汉) (text 1 3 写汉字) (text 2 13 学中文)"}, + {"读写", "(* 0 0 读写*)"}, + {"肥胖的原因都是因为摄入脂肪多导致的吗", + "(text 0 0 肥胖的) (text 1 9 原因都) (text 2 18 是因为) (text 3 27 摄入脂) (text 4 36 肪多导) (text 5 45 致的吗)"}, + {"肥胖的原因都是因为摄入fat多导致的吗", + "(text 0 0 肥胖的) (text 1 9 原因都) (text 2 18 是因为) (text 3 24 为摄入) (text 4 33 fat) (text 5 36 多导致) (text 6 42 致的吗)"}, + } + for _, c := range cases { + got, err := NLTreeStringWithPosition(c.pattern) + require.NoError(t, err, c.pattern) + require.Equal(t, c.expect, got, "pattern %q", c.pattern) + } +} + +// Ported from classic TestPatternFail: malformed boolean queries error. +func TestBooleanTreeStringFail(t *testing.T) { + for _, p := range []string{ + "Matrix Origin( ", "(+Matrix Origin", "++Matrix -Origin", + "Matrix ~~Origin", "Matrix +(<(+Origin -apple) >One)", "+Matrix --Origin", + } { + _, err := BooleanTreeString(p) + require.Error(t, err, "pattern %q should fail", p) + } +} + +// Ported from classic TestPatternNLFail: malformed NL queries error. +func TestNLTreeStringFail(t *testing.T) { + for _, p := range []string{"+[[[", "+''"} { + _, err := NLTreeString(p) + require.Error(t, err, "pattern %q should fail", p) + } +} diff --git a/pkg/fulltext2/build.go b/pkg/fulltext2/build.go new file mode 100644 index 0000000000000..ad4ba0a4d6564 --- /dev/null +++ b/pkg/fulltext2/build.go @@ -0,0 +1,485 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" +) + +// Doc is one source row to index: its primary key and the text of the indexed +// column(s). +type Doc struct { + Pk any + Text []byte +} + +// tokenWord decodes the length-prefixed word out of a tokenizer.Token +// (TokenBytes[0] is the byte length; the word follows). The tokenizer already +// lowercases and truncates to MAX_TOKEN_SIZE. +func tokenWord(tk tokenizer.Token) string { + n := int(tk.TokenBytes[0]) + return string(tk.TokenBytes[1 : 1+n]) +} + +// buildOpts carries optional build settings shared by the build entry points. +type buildOpts struct { + positionFree bool // drop the positional payload (keep the FST + docID/tf postings) + includeTypes []int32 // types.T of each INCLUDE column (nil = no INCLUDE columns) +} + +// BuildOpt configures a segment build (functional-options, so existing callers are +// unaffected). +type BuildOpt func(*buildOpts) + +// WithPositionFree builds a POSITION-FREE segment: term frequencies are still derived +// (so BM25 ranking is unchanged), but the per-token positional payload — the largest +// section (~half the segment) — is omitted and the FST term dictionary is kept. Phrase +// / NL exact-match is therefore unavailable on such a segment (bag-of-words retrieval +// only). Mirrors the footprint of a position-free bm25 index while keeping the FST. +func WithPositionFree() BuildOpt { return func(o *buildOpts) { o.positionFree = true } } + +// WithIncludeTypes builds a segment that carries per-doc INCLUDE column values (the actual +// typed values, stored in the docmap for in-index prefilter + covering). types holds each +// INCLUDE column's types.T in column order; a doc's values are supplied via Builder.SetInclude +// / SetDoc (or TokenizedDoc.Include on the batch build path). Nil/empty ⇒ no INCLUDE columns. +func WithIncludeTypes(types []int32) BuildOpt { + return func(o *buildOpts) { o.includeTypes = types } +} + +func applyBuildOpts(opts []BuildOpt) buildOpts { + var bo buildOpts + for _, o := range opts { + o(&bo) + } + return bo +} + +// buildEntry is one (doc ord, positions-in-that-doc) occurrence of a term, collected +// in ascending doc order (SearchPhrase relies on the ordering). +type buildEntry struct { + ord int64 + positions []int32 +} + +// addPosting records one (term, byte-position) occurrence for document ord into global, +// keeping each term's buildEntry list ascending by ord. The caller MUST feed documents in +// ascending ord and a document's tokens contiguously (both build paths do), so the last +// entry for a term is always the current document's when the term repeats within it — a +// repeat then extends that entry's positions in place. This groups a document's repeated +// occurrences of a term into ONE buildEntry (tf = len(positions), df = len(entries)) without +// a per-document map, which profiled as ~21% of a large CREATE FULLTEXT2 build (the old +// per-doc `local := make(map[string][]int32)` + per-token map assign). +func addPosting(global map[string][]buildEntry, w string, ord int64, pos int32) { + entries := global[w] + if n := len(entries); n > 0 && entries[n-1].ord == ord { + // Same document, repeated term: extend the open entry in place. entries aliases the + // map's slice, so mutating entries[n-1] updates global[w][n-1] — no map write needed. + entries[n-1].positions = append(entries[n-1].positions, pos) + return + } + global[w] = append(entries, buildEntry{ord: ord, positions: []int32{pos}}) +} + +// assembleTerms turns the collected per-term (ord, positions) lists into termPostings. +// When positionFree, tf is still derived from the occurrence count but the positional +// payload is dropped (tp.positions left nil) — Serialize then writes an empty positions +// section while the FST/docID/tf sections are unchanged. +func assembleTerms(global map[string][]buildEntry, docLen []int32, positionFree bool) map[string]*termPostings { + terms := make(map[string]*termPostings, len(global)) + for w, entries := range global { + tp := &termPostings{ + docIDs: make([]int64, len(entries)), + tfs: make([]uint8, len(entries)), + } + if !positionFree { + tp.positions = make([][]int32, len(entries)) + } + for i, e := range entries { + tp.docIDs[i] = e.ord + tf := len(e.positions) + if tf > MaxCappedTf { + tf = MaxCappedTf + } + tp.tfs[i] = uint8(tf) + if !positionFree { + tp.positions[i] = e.positions + } + } + // Derive the raw term-level + Block-Max score-UB fields (mirrors the load + // path), so a build-side segment gets the same WAND block-skip bounds. + deriveTermStats(tp, docLen) + terms[w] = tp + } + return terms +} + +// BuildSegmentFromDocs tokenizes each doc with tok and builds an in-memory +// (build-side) segment: per-term posting lists with per-doc token positions, +// the docmap (pk + token-count length), and AvgDocLen. Docs are indexed by their +// slice position (the doc ord), so pks/docLen are ord-aligned. +// +// It is the minimal build path used to produce queryable segments (tests, and +// the seed for the eventual CDC/sync build sink). Positions are the tokenizer's +// TokenPos, so phrase adjacency in a query matches adjacency in the source text. +func BuildSegmentFromDocs(id string, pkType int32, docs []Doc, tok tokenizer.Tokenizer, opts ...BuildOpt) (*Segment, error) { + bo := applyBuildOpts(opts) + s := NewSegment(id, pkType) + s.pks = make([]any, len(docs)) + s.docLen = make([]int32, len(docs)) + + // term -> ascending list of (doc ord, positions-in-that-doc). Built in doc + // order, so each term's list is ascending by ord (SearchPhrase relies on it). + global := make(map[string][]buildEntry) + + for ord, d := range docs { + s.pks[ord] = d.Pk + var ntok int32 + for tk, err := range tok.Tokenize(d.Text) { + if err != nil { + return nil, err + } + addPosting(global, tokenWord(tk), int64(ord), tk.BytePos) + ntok++ + } + s.docLen[ord] = ntok + } + + terms := assembleTerms(global, s.docLen, bo.positionFree) + + s.N = int64(len(docs)) + s.setTerms(terms) + s.AvgDocLen = meanDocLen(s.docLen) + return s, nil +} + +// SearchText tokenizes query with tok (the same tokenizer used to build the +// segment) and runs an NL exact-phrase search — the convenience entry point that +// mirrors MATCH(col) AGAINST('query') in natural-language mode. +func (s *Segment) SearchText(query []byte, tok tokenizer.Tokenizer, algo ScoreAlgo, k int) ([]Result, error) { + return s.SearchPhrase(tokenizePhraseSlots(tok, query), algo, k), nil +} + +// TokenizedDoc is a document already tokenized into an ordered term slice (the +// term at index i has token position i). Used by the build path that tokenizes +// via the execution-side fulltext2_tokenize TVF (datalink/json/parsers resolved +// there) and only assembles the segment here. +type TokenizedDoc struct { + Pk any + Terms []string + Positions []int32 // byte position of Terms[i] (parallel to Terms) + // Include holds this doc's INCLUDE column values in column order (nil element = SQL + // NULL; a nil slice = all NULL). Only meaningful when the segment is built with + // WithIncludeTypes; ignored otherwise. + Include []any +} + +// BuildSegmentFromTokenized builds a segment from pre-tokenized docs (positions +// are term indices). It is the assembly half of BuildSegmentFromDocs without the +// tokenizer — the terms already came from fulltext2_tokenize at execution. +func BuildSegmentFromTokenized(id string, pkType int32, docs []TokenizedDoc, opts ...BuildOpt) (*Segment, error) { + bo := applyBuildOpts(opts) + s := NewSegment(id, pkType) + s.pks = make([]any, len(docs)) + s.docLen = make([]int32, len(docs)) + + global := make(map[string][]buildEntry) + + for ord, d := range docs { + s.pks[ord] = d.Pk + for i, w := range d.Terms { + pos := int32(i) // fall back to token index when Positions is unset + if i < len(d.Positions) { + pos = d.Positions[i] // byte position + } + addPosting(global, w, int64(ord), pos) + } + s.docLen[ord] = int32(len(d.Terms)) + } + + terms := assembleTerms(global, s.docLen, bo.positionFree) + + // INCLUDE column values: carried on each TokenizedDoc.Include, packed ord-aligned into + // the segment so encodeDocmap writes the docmap include section. A nil Include ⇒ all-NULL + // row (encodeDocmap fills NULLs for missing columns). + if len(bo.includeTypes) > 0 { + s.includeTypes = bo.includeTypes + s.includeVals = make([][]any, len(docs)) + for ord := range docs { + s.includeVals[ord] = docs[ord].Include + } + } + + s.N = int64(len(docs)) + s.setTerms(terms) + s.AvgDocLen = meanDocLen(s.docLen) + return s, nil +} + +// DefaultBuildCapacity floors a non-positive segment DOC capacity for the streaming +// build paths (base CREATE build and the CDC tail), so a segment is sealed and +// spilled every ~1M docs and peak build memory stays bounded to one segment even +// when max_index_capacity is unset. Splitting a logically "unbounded" base into +// 1M-doc sub-indexes is transparent to queries (the Index already spans segments). +const DefaultBuildCapacity int64 = 1000000 + +// DefaultPostingCapacity floors a non-positive segment POSTING capacity +// (max_postings_capacity). Per-segment build memory tracks the number of +// accumulated postings (one term occurrence ≈ a term-string ref + an int32 +// position ≈ 30 B of Go heap, ~2× at assembly), NOT the doc count — a doc can hold +// one token or tens of thousands, so a doc-only seal lets a long-document corpus +// (e.g. wiki articles ≈ 1k tokens each) blow past any memory bound. 8M postings ≈ +// a ~512 MB peak build heap per segment; a segment seals on whichever cap +// (docs OR postings) is reached first, so build memory is bounded regardless of +// doc shape. See ReachedSegmentCap. +const DefaultPostingCapacity int64 = 8_000_000 + +// DefaultDataBytesCapacity bounds the retained variable-width PAYLOAD heap — the copied +// varlena primary-key and INCLUDE-column bytes held in Builder.docs — which neither the doc +// cap nor the posting cap covers. A corpus of one-token rows (postings ≈ docs) with wide +// VARCHAR/BLOB pk or INCLUDE values reaches neither of those caps yet can retain up to +// DefaultBuildCapacity docs' worth of near-max varlena data (tens of GiB) before sealing. +// 512 MiB keeps that payload heap in the same ~512 MB per-segment envelope as the posting +// cap, so total build peak stays bounded regardless of doc shape. See ReachedSegmentCap. +const DefaultDataBytesCapacity int64 = 512 << 20 // 512 MiB + +// ReachedSegmentCap reports whether an open streaming Builder has hit any of its three +// per-segment build-memory bounds: the doc cap (max_index_capacity, bounds the docmap/liveOrd +// heap for a many-tiny-docs index), the posting cap (max_postings_capacity, bounds the +// positional posting heap for a few-long-docs index), or the payload-bytes cap (bounds the +// retained varlena pk + INCLUDE heap for a wide-varlena corpus). Non-positive doc/posting caps +// fall back to their defaults. Every streaming build path (CREATE TVF, CDC TailBuilder, +// MERGE/REBUILD compaction) seals on this single predicate so peak build memory is one bounded +// segment. +// +// The cap is checked AFTER a row is added, so a SINGLE row larger than a cap seals into its own +// segment (peak bounded to that one oversized row) rather than being rejected — a covering CREATE +// must still ingest a legitimately huge row. +func ReachedSegmentCap(b *Builder, docCap, postingCap int64) bool { + if docCap <= 0 { + docCap = DefaultBuildCapacity + } + if postingCap <= 0 { + postingCap = DefaultPostingCapacity + } + // Compare the int64 counters directly (not via the int-returning getters) so a cap above + // 2^31 can never wrap on a 32-bit build. + return int64(b.NumDocs()) >= docCap || b.postings >= postingCap || b.dataBytes >= DefaultDataBytesCapacity +} + +// Builder accumulates a token stream fed in (word, pk) order — the positional +// analogue of bm25's wand.Builder, with the SAME API (NewBuilder / Add / Finish / +// FinishSegments) so the two can share a core later. Each Add appends one token +// occurrence to its document (position = the doc's running token count), so the +// caller feeds a document's tokens contiguously and in order. +type Builder struct { + id string + pkType int32 + ordMap map[any]int64 // normalized pk -> ord + docs []TokenizedDoc + postings int64 // total term occurrences Add'd (one Add == one posting) + // dataBytes is the retained variable-width payload heap: the copied varlena pk bytes plus + // the retained INCLUDE value bytes across all open docs. It is the memory dimension the doc + // and posting caps miss (a one-token row with a huge VARCHAR pk/INCLUDE), so ReachedSegmentCap + // also seals on it. Kept in lock-step: docOrd adds a new doc's pk bytes; SetDoc/SetInclude + // add the new INCLUDE bytes and subtract the superseded ones on replace. + dataBytes int64 + opts []BuildOpt // carried into FinishSegments (e.g. WithPositionFree) +} + +// anyDataBytes is the retained-heap size of one boxed pk / INCLUDE value: the content length +// for a varlena value (the dimension that blows up), a small fixed estimate for a scalar, 0 for +// SQL NULL. Used to keep Builder.dataBytes tracking the varlena payload the caps must bound. +func anyDataBytes(v any) int64 { + switch x := v.(type) { + case nil: + return 0 + case string: + return int64(len(x)) + case []byte: + return int64(len(x)) + default: + return 8 // fixed-width scalar (int/uint/temporal/decimal) — bounded, small + } +} + +// includeDataBytes sums the retained payload of a doc's INCLUDE value slice. +func includeDataBytes(include []any) int64 { + var n int64 + for _, v := range include { + n += anyDataBytes(v) + } + return n +} + +// NewBuilder creates a Builder for an index id and source pk type (types.T). Pass +// WithPositionFree() to build position-free segments. +func NewBuilder(id string, pkType int32, opts ...BuildOpt) *Builder { + return &Builder{id: id, pkType: pkType, ordMap: make(map[any]int64), opts: opts} +} + +// docOrd returns the dense ord for a pk, assigning one on first sight (mirrors +// wand.Builder.docOrd). +func (b *Builder) docOrd(pk any) int64 { + key := builderKey(pk) + if o, ok := b.ordMap[key]; ok { + return o + } + o := int64(len(b.docs)) + b.ordMap[key] = o + b.docs = append(b.docs, TokenizedDoc{Pk: builderCopyPk(pk)}) + b.dataBytes += anyDataBytes(pk) // retained copied pk bytes (varlena payload the caps must bound) + return o +} + +// Add records one (word, byte-position, doc) token occurrence. Tokens of a document +// must be fed contiguously and in order; pos is the token's BYTE position in the doc +// (so a positional phrase query can match by byte offset, consistent with the base +// build's tk.BytePos and classic fulltext's byte-position phrase JOIN). +func (b *Builder) Add(word string, pos int32, pk any) error { + if word == "" { + return moerr.NewInternalErrorNoCtx("fulltext2 builder: empty word") + } + ord := b.docOrd(pk) + b.docs[ord].Terms = append(b.docs[ord].Terms, word) + b.docs[ord].Positions = append(b.docs[ord].Positions, pos) + b.postings++ + return nil +} + +// SetDoc REPLACES a pk's document terms with words (assigning a fresh ord on first +// sight), rather than appending like Add. It is the CDC upsert primitive: a later +// upsert of a pk already present in this open segment must supersede — not merge with — +// its earlier terms (Add would leave both the old and new terms searchable under the +// same live doc). A zero-word upsert produces a live 0-term doc: it carries the pk + +// this segment's Recency, so it SHADOWS any lower-Recency copy (base or an earlier tail +// segment) at query time while itself matching nothing — the way an upsert to empty / +// NULL text removes the old version from search. Positions/Terms are reset in place so +// repeated SetDoc of one pk does not grow the backing arrays unboundedly. +func (b *Builder) SetDoc(pk any, words []WordPos, include []any) { + ord := b.docOrd(pk) + d := &b.docs[ord] + b.postings -= int64(len(d.Terms)) // drop the prior version's postings before rewriting + d.Terms = d.Terms[:0] + d.Positions = d.Positions[:0] + for _, w := range words { + if w.Word == "" { + continue // never index an empty token (mirrors Add's guard) + } + d.Terms = append(d.Terms, w.Word) + d.Positions = append(d.Positions, w.Pos) + } + b.postings += int64(len(d.Terms)) + // An upsert supersedes the prior version's INCLUDE values in lock-step with its terms; + // track the payload-byte delta so dataBytes reflects only the live version. + b.dataBytes += includeDataBytes(include) - includeDataBytes(d.Include) + d.Include = include +} + +// SetInclude sets a doc's INCLUDE column values (assigning a fresh ord on first sight, +// like Add). Used by the Add-based build paths (base CREATE, MERGE reconstruction) which +// feed a doc's tokens via Add and then attach its per-row INCLUDE values once. include is +// retained as-is (column order; a nil element = SQL NULL). +func (b *Builder) SetInclude(pk any, include []any) { + ord := b.docOrd(pk) + b.dataBytes += includeDataBytes(include) - includeDataBytes(b.docs[ord].Include) + b.docs[ord].Include = include +} + +// NumDocs returns the number of distinct documents added so far. +func (b *Builder) NumDocs() int { return len(b.docs) } + +// NumDataBytes returns the retained variable-width payload heap (copied varlena pk + INCLUDE +// bytes) — the dimension DefaultDataBytesCapacity bounds in ReachedSegmentCap. +func (b *Builder) NumDataBytes() int64 { return b.dataBytes } + +// NumPostings returns the number of term occurrences Add'd so far (one Add == one +// posting). It is the memory-correlated dimension the streaming build paths seal +// on (max_postings_capacity) — see ReachedSegmentCap. +func (b *Builder) NumPostings() int { return int(b.postings) } + +// Finish produces a single-segment index (no capacity limit). +func (b *Builder) Finish() (*Segment, error) { + segs, err := b.FinishSegments(0) + if err != nil { + return nil, err + } + return segs[0], nil +} + +// FinishSegments finalizes the build into one or more segments, each holding at +// most capacity documents (capacity <= 0 => a single segment). Segment i is built +// under SubIndexId(b.id, i); the caller may override Segment.Id. Mirrors +// wand.Builder.FinishSegments (contiguous doc-ord range split). +func (b *Builder) FinishSegments(capacity int64) ([]*Segment, error) { + n := int64(len(b.docs)) + if n == 0 { + return nil, nil + } + if capacity <= 0 || n <= capacity { + seg, err := BuildSegmentFromTokenized(SubIndexId(b.id, 0), b.pkType, b.docs, b.opts...) + if err != nil { + return nil, err + } + return []*Segment{seg}, nil + } + var segs []*Segment + i := 0 + for lo := int64(0); lo < n; lo += capacity { + hi := lo + capacity + if hi > n { + hi = n + } + seg, err := BuildSegmentFromTokenized(SubIndexId(b.id, i), b.pkType, b.docs[lo:hi], b.opts...) + if err != nil { + return nil, err + } + segs = append(segs, seg) + i++ + } + return segs, nil +} + +// builderKey / builderCopyPk mirror wand.go's normalizeKey / copyPk so a []byte +// pk is a stable map key and safely retained. +func builderKey(pk any) any { + if b, ok := pk.([]byte); ok { + return string(b) + } + return pk +} + +func builderCopyPk(pk any) any { + if b, ok := pk.([]byte); ok { + c := make([]byte, len(b)) + copy(c, b) + return c + } + return pk +} + +// tokenizeToTerms flattens a tokenizer stream into the ordered term slice a +// phrase query needs. +func tokenizeToTerms(text []byte, tok tokenizer.Tokenizer) ([]string, error) { + // token count is unknown up front; estimate from input length to avoid regrowth. + terms := make([]string, 0, len(text)/4) + for tk, err := range tok.Tokenize(text) { + if err != nil { + return nil, err + } + terms = append(terms, tokenWord(tk)) + } + return terms, nil +} diff --git a/pkg/fulltext2/builder_test.go b/pkg/fulltext2/builder_test.go new file mode 100644 index 0000000000000..aac52ec7ac59d --- /dev/null +++ b/pkg/fulltext2/builder_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Builder API parity with bm25.wand.Builder: NewBuilder(id, pkType) / Add(word, pk) +// / Finish / FinishSegments(capacity), so the two engines can share a core later. +// fulltext2's builder additionally records token POSITIONS (Add order within a doc). +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// feed adds a doc's tokens in order (mirrors the fulltext2_create per-token Add loop). +// These unit tests use the token index as a synthetic byte position — self-consistent +// with phr() below, which builds consecutive-offset phrase slots. (Real byte positions +// are exercised by the text-based / CJK tests via BuildSegmentFromDocsParser + SearchQuery.) +func feed(t *testing.T, b *Builder, pk any, words ...string) { + pos := int32(0) + for _, w := range words { + require.NoError(t, b.Add(w, pos, pk)) + pos += int32(len(w)) + 1 // byte position of single-space-joined words + } +} + +// phr builds phrase slots for a contiguous term list at the byte offsets a +// single-space-joined query text would tokenize to — matching feed's positions and the +// production query path (tokenizing "a b" → a@0, b@len(a)+1). +func phr(terms ...string) []phraseSlot { + s := make([]phraseSlot, len(terms)) + pos := int32(0) + for i, t := range terms { + s[i] = phraseSlot{term: t, off: pos} + pos += int32(len(t)) + 1 + } + return s +} + +// TestBuilderSingleSegment: Finish() builds one positional segment; phrase and +// term queries resolve against it. +func TestBuilderSingleSegment(t *testing.T) { + b := NewBuilder("idx", int32(types.T_int64)) + feed(t, b, int64(0), "quick", "brown", "fox") + feed(t, b, int64(1), "brown", "fox", "jumps") + feed(t, b, int64(2), "quick", "red", "fox") + require.Equal(t, 3, b.NumDocs()) + + seg, err := b.Finish() + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + // exact phrase "brown fox" is contiguous in docs 0 and 1, not 2. + require.ElementsMatch(t, []any{int64(0), int64(1)}, resultIDs(idx.SearchPhrase(phr("brown", "fox"), BM25, 10, nil))) + // single term. + require.ElementsMatch(t, []any{int64(0), int64(2)}, resultIDs(idx.SearchPhrase(phr("quick"), BM25, 10, nil))) +} + +// TestBuilderCapacitySplit: FinishSegments(capacity) splits by contiguous doc-ord +// range into ceil(n/capacity) segments; the merged Index still finds every doc. +func TestBuilderCapacitySplit(t *testing.T) { + b := NewBuilder("idx", int32(types.T_int64)) + for i := 0; i < 10; i++ { + feed(t, b, int64(i), "common", "brown") + } + segs, err := b.FinishSegments(3) // 10 docs / 3 => 4 segments + require.NoError(t, err) + require.Len(t, segs, 4) + require.Equal(t, int64(3), segs[0].N) + require.Equal(t, int64(1), segs[3].N) // remainder + + idx := NewIndex(segs, nil) + got := resultIDs(idx.SearchPhrase(phr("brown"), BM25, 100, nil)) + require.Len(t, got, 10) // all docs across the 4 bases +} + +// TestBuilderEmpty: no Add → no segments. +func TestBuilderEmpty(t *testing.T) { + b := NewBuilder("idx", int32(types.T_int64)) + segs, err := b.FinishSegments(0) + require.NoError(t, err) + require.Nil(t, segs) +} + +// TestBuilderNumPostings: NumPostings counts term occurrences (one per Add), not docs +// — the memory-correlated seal dimension. Three 2-token docs => 6 postings, 3 docs. +func TestBuilderNumPostings(t *testing.T) { + b := NewBuilder("idx", int32(types.T_int64)) + feed(t, b, int64(0), "a", "b") + feed(t, b, int64(1), "c", "d") + feed(t, b, int64(2), "e", "f") + require.Equal(t, 3, b.NumDocs()) + require.Equal(t, 6, b.NumPostings()) +} + +// TestReachedSegmentCap: a segment seals on whichever cap (docs OR postings) is hit +// first, and non-positive caps fall back to the package defaults. +func TestReachedSegmentCap(t *testing.T) { + // Doc cap fires first: 2 single-token docs reach docCap=2 before postingCap=100. + b := NewBuilder("d", int32(types.T_int64)) + feed(t, b, int64(0), "x") + require.False(t, ReachedSegmentCap(b, 2, 100)) // 1 doc / 1 posting + feed(t, b, int64(1), "y") + require.True(t, ReachedSegmentCap(b, 2, 100)) // 2 docs => doc cap + + // Posting cap fires first: one long doc crosses postingCap=3 while docCap=100 idle. + // This is the long-document case a doc-only seal would miss. + b2 := NewBuilder("p", int32(types.T_int64)) + feed(t, b2, int64(0), "a", "b", "c") // 1 doc, 3 postings + require.True(t, ReachedSegmentCap(b2, 100, 3)) + require.False(t, ReachedSegmentCap(b2, 100, 4)) // 3 postings < 4 + + // Non-positive caps fall back to defaults (both large) — a tiny builder never seals. + require.False(t, ReachedSegmentCap(b2, 0, 0)) + require.Less(t, int64(b2.NumDocs()), DefaultBuildCapacity) + require.Less(t, int64(b2.NumPostings()), DefaultPostingCapacity) +} + +// TestReachedSegmentCapDataBytes: one-token rows with wide varlena pk/INCLUDE reach neither +// the doc nor posting cap, so the payload-bytes cap must bound them. Also checks the byte +// accounting (varlena pk + INCLUDE, NULL = 0, replace-on-upsert delta). +func TestReachedSegmentCapDataBytes(t *testing.T) { + // varchar pk (10 B) + one INCLUDE value (5 B) on a single-token doc. + b := NewBuilder("d", int32(types.T_varchar)) + require.NoError(t, b.Add("tok", 0, "PPPPPPPPPP")) // pk = 10 bytes + b.SetInclude("PPPPPPPPPP", []any{"IIIII"}) // +5 include bytes + require.EqualValues(t, 15, b.NumDataBytes()) + + // upsert replace supersedes the INCLUDE bytes in lock-step (5 -> 2), pk unchanged. + b.SetDoc("PPPPPPPPPP", []WordPos{{Word: "tok2", Pos: 0}}, []any{"XX"}) + require.EqualValues(t, 12, b.NumDataBytes()) // 10 pk + 2 include + + // 1 doc / 1 posting: doc & posting caps idle, but the byte cap seals when dataBytes hits it. + require.False(t, ReachedSegmentCap(b, 0, 0)) + b.dataBytes = DefaultDataBytesCapacity + require.True(t, ReachedSegmentCap(b, 0, 0)) + + // fixed pk (8 B est.) + INCLUDE with a NULL (0) and a 2-byte value. + b2 := NewBuilder("n", int32(types.T_int64)) + require.NoError(t, b2.Add("t", 0, int64(1))) + b2.SetInclude(int64(1), []any{nil, "ab"}) + require.EqualValues(t, 10, b2.NumDataBytes()) // 8 pk + 0 + 2 +} + +// TestBuilderPostingSplitStreaming: the streaming seal pattern the build paths use — +// ReachedSegmentCap on a posting cap seals a long-document corpus into multiple +// segments even though the doc cap is never reached; the merged Index finds every doc. +func TestBuilderPostingSplitStreaming(t *testing.T) { + const postingCap = int64(4) + var segs []*Segment + cur := NewBuilder("s", int32(types.T_int64)) + seal := func() { + if cur.NumDocs() == 0 { + return + } + seg, err := cur.Finish() + require.NoError(t, err) + segs = append(segs, seg) + cur = NewBuilder("s", int32(types.T_int64)) + } + // 6 docs × 2 tokens = 12 postings; docCap huge, postingCap=4 => seal every 2 docs. + for i := 0; i < 6; i++ { + feed(t, cur, int64(i), "common", "brown") + if ReachedSegmentCap(cur, 1_000_000, postingCap) { + seal() + } + } + seal() + require.Len(t, segs, 3) // 12 postings / 4 => 3 segments + idx := NewIndex(segs, nil) + require.Len(t, resultIDs(idx.SearchPhrase(phr("brown"), BM25, 100, nil)), 6) +} diff --git a/pkg/fulltext2/cdc.go b/pkg/fulltext2/cdc.go new file mode 100644 index 0000000000000..e080a3558e6f9 --- /dev/null +++ b/pkg/fulltext2/cdc.go @@ -0,0 +1,242 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "io" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// The CDC-event codec: the ISCP consumer's Fulltext2SqlWriter accumulates a +// flush's source-row mutations into a Cdc, ships it as one channel blob (Encode), +// and RunFulltext2 decodes it (DecodeCdc) and STREAMS it through a TailBuilder +// into capacity-capped tag=1 CdcTail segments. The blob is BINARY (typed pk) — +// like bm25's WandCdc — because a fulltext2 pk is `any` (int64 OR varchar) and a +// JSON round-trip would corrupt a non-integer pk. + +type cdcOp byte + +const ( + cdcInsert cdcOp = 'I' + cdcUpsert cdcOp = 'U' + cdcDelete cdcOp = 'D' +) + +// CdcEvent is one source-row mutation: an INSERT/UPSERT carries the row's text +// (tokenized at build) and its INCLUDE column values; a DELETE carries only the pk. +type CdcEvent struct { + Op cdcOp + Pk any + Text string + Include []any // INCLUDE column values in column order (nil element = NULL); nil for DELETE +} + +// Cdc is the per-flush CDC batch the sinker accumulates and ships as one blob. +// IncludeTypes is the index's INCLUDE column schema (types.T per column, column order), +// needed to typed-encode/decode each event's Include values; nil ⇒ no INCLUDE columns. +type Cdc struct { + PkType int32 + IncludeTypes []int32 + Events []CdcEvent +} + +func NewCdc(pkType int32) *Cdc { return &Cdc{PkType: pkType} } + +func (c *Cdc) Insert(pk any, text string, include []any) { + c.Events = append(c.Events, CdcEvent{cdcInsert, pk, text, include}) +} +func (c *Cdc) Upsert(pk any, text string, include []any) { + c.Events = append(c.Events, CdcEvent{cdcUpsert, pk, text, include}) +} +func (c *Cdc) Delete(pk any) { c.Events = append(c.Events, CdcEvent{cdcDelete, pk, "", nil}) } +func (c *Cdc) Len() int { return len(c.Events) } + +const cdcMagic uint32 = 0x46540200 // 'F' 'T' 02 00 — fulltext2 cdc events + +// Encode serializes the batch: magic | pkType | count | +// [op:1 pkLen:u32 pk textLen:u32 text]* | crc32. Self-describing + CRC-checked. +func (c *Cdc) Encode() ([]byte, error) { + var b bytes.Buffer + _ = binary.Write(&b, binary.LittleEndian, cdcMagic) + _ = binary.Write(&b, binary.LittleEndian, c.PkType) + // INCLUDE column schema: count + per-col type (0 ⇒ no INCLUDE columns). Lets the + // decoder typed-decode each event's Include values without an external schema. + _ = binary.Write(&b, binary.LittleEndian, uint32(len(c.IncludeTypes))) + for _, t := range c.IncludeTypes { + _ = binary.Write(&b, binary.LittleEndian, t) + } + _ = binary.Write(&b, binary.LittleEndian, int64(len(c.Events))) + for _, e := range c.Events { + pkb, err := encodePk(c.PkType, e.Pk) + if err != nil { + return nil, err + } + b.WriteByte(byte(e.Op)) + _ = binary.Write(&b, binary.LittleEndian, uint32(len(pkb))) + b.Write(pkb) + _ = binary.Write(&b, binary.LittleEndian, uint32(len(e.Text))) + b.WriteString(e.Text) + // INCLUDE values (column order); a missing/nil value is SQL NULL. Same + // [nullFlag][u32 len][content] shape as the docmap include section. + for ci, t := range c.IncludeTypes { + var v any + if ci < len(e.Include) { + v = e.Include[ci] + } + if v == nil { + b.WriteByte(includeNull) + continue + } + vb, err := encodePk(t, v) + if err != nil { + return nil, err + } + b.WriteByte(includeNotNull) + _ = binary.Write(&b, binary.LittleEndian, uint32(len(vb))) + b.Write(vb) + } + } + sum := crc32.ChecksumIEEE(b.Bytes()) + _ = binary.Write(&b, binary.LittleEndian, sum) + return b.Bytes(), nil +} + +// DecodeCdc reverses Encode, validating magic + CRC. +func DecodeCdc(buf []byte) (*Cdc, error) { + if len(buf) < 4+4+8+4 { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: truncated") + } + body := buf[:len(buf)-4] + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(buf[len(buf)-4:]) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: checksum mismatch") + } + r := bytes.NewReader(body) + var magic uint32 + _ = binary.Read(r, binary.LittleEndian, &magic) + if magic != cdcMagic { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: bad magic") + } + c := &Cdc{} + if err := binary.Read(r, binary.LittleEndian, &c.PkType); err != nil { + return nil, err + } + var nIncl uint32 + if err := binary.Read(r, binary.LittleEndian, &nIncl); err != nil { + return nil, err + } + if nIncl > maxIncludeCols { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: bad include col count") + } + if nIncl > 0 { + c.IncludeTypes = make([]int32, nIncl) + for i := range c.IncludeTypes { + if err := binary.Read(r, binary.LittleEndian, &c.IncludeTypes[i]); err != nil { + return nil, err + } + } + } + var n int64 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return nil, err + } + if n < 0 { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: bad count") + } + // n is read straight off a (possibly corrupt) blob; a bogus huge count would make() + // gigabytes up front. Cap the pre-alloc hint — append grows it for a genuinely large + // frame, and a truncated stream errors out per-event in the loop below. + capHint := n + if capHint > 4096 { + capHint = 4096 + } + c.Events = make([]CdcEvent, 0, capHint) + for i := int64(0); i < n; i++ { + op, err := r.ReadByte() + if err != nil { + return nil, err + } + pk, err := cdcReadLenBytes(r, c.PkType) + if err != nil { + return nil, err + } + text, err := cdcReadLenString(r) + if err != nil { + return nil, err + } + var include []any + if nIncl > 0 { + include = make([]any, nIncl) + for ci := 0; ci < int(nIncl); ci++ { + v, ierr := cdcReadInclude(r, c.IncludeTypes[ci]) + if ierr != nil { + return nil, ierr + } + include[ci] = v + } + } + c.Events = append(c.Events, CdcEvent{Op: cdcOp(op), Pk: pk, Text: text, Include: include}) + } + return c, nil +} + +// cdcReadInclude reads one INCLUDE value: a null flag, then (if not null) a u32 length + +// typed content decoded via decodePk. Returns nil for a NULL value. +func cdcReadInclude(r *bytes.Reader, t int32) (any, error) { + flag, err := r.ReadByte() + if err != nil { + return nil, err + } + if flag == includeNull { + return nil, nil + } + if flag != includeNotNull { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: bad include null-flag") + } + return cdcReadLenBytes(r, t) +} + +func cdcReadLenBytes(r *bytes.Reader, pkType int32) (any, error) { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return nil, err + } + if int64(l) > int64(r.Len()) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 cdc: truncated pk") + } + pkb := make([]byte, l) + if _, err := io.ReadFull(r, pkb); err != nil { + return nil, err + } + return decodePk(pkType, pkb) +} + +func cdcReadLenString(r *bytes.Reader) (string, error) { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return "", err + } + if int64(l) > int64(r.Len()) { + return "", moerr.NewInternalErrorNoCtx("fulltext2 cdc: truncated text") + } + sb := make([]byte, l) + if _, err := io.ReadFull(r, sb); err != nil { + return "", err + } + return string(sb), nil +} diff --git a/pkg/fulltext2/cdc_lww_test.go b/pkg/fulltext2/cdc_lww_test.go new file mode 100644 index 0000000000000..bd267e12ef66b --- /dev/null +++ b/pkg/fulltext2/cdc_lww_test.go @@ -0,0 +1,408 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CDC last-write-wins semantics (engine layer, no live DB): a later upsert of a pk +// must REPLACE — not merge with — its earlier terms, and an upsert to empty/NULL text +// must shadow the old version. Drives the whole write path (TailBuilder -> frames -> +// Index-with-liveness) and asserts the resolved search results, without reordering the +// delete-before-insert frame order (which CDC relies on for same-batch updates). +package fulltext2 + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// mkBase builds a tag=0 base segment (recency 0) from pk->text. +func mkBase(t *testing.T, pkTexts map[int64]string) *Segment { + t.Helper() + bb := NewBuilder("base", int32(types.T_int64)) + for pk, txt := range pkTexts { + feed(t, bb, pk, strings.Fields(txt)...) + } + seg, err := bb.Finish() + require.NoError(t, err) + seg.Recency = 0 + return seg +} + +// buildTailIndexLWW runs c through a TailBuilder at capacity, decodes the spilled frames in +// Finish order (recency = position, above the base) exactly like LoadTailSegments, and assembles +// the queryable Index. Single-blob convenience over buildTailIndexLWWMulti. +func buildTailIndexLWW(t *testing.T, base *Segment, c *Cdc, capacity int64) *Index { + t.Helper() + return buildTailIndexLWWMulti(t, base, capacity, c) +} + +// buildTailIndexLWWMulti runs SEVERAL CDC blobs through ONE TailBuilder — mimicking the real +// producer, which flushes a new blob on every op change, so an INSERT and a later DELETE of the +// same pk arrive as SEPARATE AddBatch calls. It then assembles the Index from the Finish-order +// frames (recency = position), exercising cross-blob last-writer-wins. +func buildTailIndexLWWMulti(t *testing.T, base *Segment, capacity int64, cdcs ...*Cdc) *Index { + t.Helper() + tb, err := NewTailBuilder(int32(types.T_int64), capacity, 0, "", wsTokenize) + require.NoError(t, err) + defer tb.Cleanup() + for _, c := range cdcs { + require.NoError(t, tb.AddBatch(c)) + } + segs, err := tb.Finish() + require.NoError(t, err) + return framesToIndex(t, base, segs) +} + +// framesToIndex decodes packed tail frames in Finish order (recency = position, above the base) +// exactly like LoadTailSegments, and assembles the queryable Index. Shared by the LWW helpers and +// the packed-spool test. +func framesToIndex(t *testing.T, base *Segment, segs []TailSegment) *Index { + t.Helper() + var tails []*Segment + if base != nil { + tails = append(tails, base) + } + deletes := map[any]int64{} + recency := int64(100) + for i, ts := range segs { + framed := readTailFrame(t, ts) + seg, dels, uerr := UnframeTail("tail", framed) + require.NoError(t, uerr) + switch { + case seg != nil: + seg.Recency = recency + int64(i) + tails = append(tails, seg) + case dels != nil: + deletes = foldDeleteFrame(deletes, dels, recency+int64(i)) + } + } + return NewIndex(tails, deletes) +} + +// distinctPaths counts the distinct spool files a frame set occupies. +func distinctPaths(segs []TailSegment) int { + seen := map[string]struct{}{} + for _, s := range segs { + seen[s.Path] = struct{}{} + } + return len(seen) +} + +// TestCdcInsertThenDeleteAcrossCdcBlobs is the cross-blob regression: the producer flushes a new +// CDC blob on each op change, so INSERT(pk) then a later DELETE(pk) reach ONE TailBuilder as TWO +// AddBatch calls. The later DELETE must win — the covered 0-JOIN path has no source join to mask a +// resurrected row. (Before the session-wide fix, delete-first framing gave the older insert the +// higher recency and returned the deleted doc.) +func TestCdcInsertThenDeleteAcrossCdcBlobs(t *testing.T) { + ins := NewCdc(int32(types.T_int64)) + ins.Insert(int64(1), "ghost", nil) + del := NewCdc(int32(types.T_int64)) + del.Delete(int64(1)) + idx := buildTailIndexLWWMulti(t, nil, 1000, ins, del) // two blobs, one builder + require.Empty(t, queryIDs(t, idx, "ghost"), "the later DELETE must win across CDC blobs") +} + +// TestCdcDeleteThenReinsertAcrossCdcBlobs: DELETE(pk) then a later INSERT(pk) across blobs — the +// re-insert wins (it removes the pending tombstone), no phantom delete. +func TestCdcDeleteThenReinsertAcrossCdcBlobs(t *testing.T) { + del := NewCdc(int32(types.T_int64)) + del.Delete(int64(1)) + ins := NewCdc(int32(types.T_int64)) + ins.Insert(int64(1), "reborn", nil) + idx := buildTailIndexLWWMulti(t, nil, 1000, del, ins) + require.Equal(t, []any{int64(1)}, queryIDs(t, idx, "reborn"), "re-insert after delete must be live") +} + +// TestCdcUpdateAcrossCdcBlobs: an UPDATE split across blobs (DELETE(base pk) then INSERT(pk,new)) +// keeps the new version live and the old gone — no lingering tombstone resurrects the delete. +func TestCdcUpdateAcrossCdcBlobs(t *testing.T) { + base := mkBase(t, map[int64]string{1: "oldterm"}) + del := NewCdc(int32(types.T_int64)) + del.Delete(int64(1)) + ins := NewCdc(int32(types.T_int64)) + ins.Insert(int64(1), "newterm", nil) + idx := buildTailIndexLWWMulti(t, base, 1000, del, ins) + require.Empty(t, queryIDs(t, idx, "oldterm"), "old version superseded by the update") + require.Equal(t, []any{int64(1)}, queryIDs(t, idx, "newterm"), "updated version is live") +} + +// readTailFrame reads one frame's bytes from its (now packed) spool file, honoring Offset/FrameLen +// exactly as persist's load_file range does — so tests decode the same bytes the DB would store. +func readTailFrame(t *testing.T, ts TailSegment) []byte { + t.Helper() + all, err := os.ReadFile(ts.Path) + require.NoError(t, err) + require.LessOrEqual(t, int(ts.Offset)+ts.FrameLen, len(all), "frame range must lie within the spool file") + return all[ts.Offset : ts.Offset+int64(ts.FrameLen)] +} + +// TestCdcPackedSpoolBoundedFiles is the regression for the reviewer's unbounded-file finding: an +// alternating DELETE,INSERT,… update stream still produces one FRAME per op run (chronological seal +// preserved — LWW unchanged), but the frames are PACKED into a bounded number of spool files instead +// of one file per frame. Proves file count is bounded while frame/chunk count and LWW are unchanged. +func TestCdcPackedSpoolBoundedFiles(t *testing.T) { + const n = 200 + baseTexts := map[int64]string{} + for i := int64(1); i <= n; i++ { + baseTexts[i] = fmt.Sprintf("old%d", i) + } + base := mkBase(t, baseTexts) + + var blobs []*Cdc + for i := int64(1); i <= n; i++ { + del := NewCdc(int32(types.T_int64)) + del.Delete(i) + ins := NewCdc(int32(types.T_int64)) + ins.Insert(i, fmt.Sprintf("new%d", i), nil) + blobs = append(blobs, del, ins) + } + + tb, err := NewTailBuilder(int32(types.T_int64), 1000, 0, "", wsTokenize) + require.NoError(t, err) + defer tb.Cleanup() + for _, c := range blobs { + require.NoError(t, tb.AddBatch(c)) + } + segs, err := tb.Finish() + require.NoError(t, err) + + // Frames are NOT collapsed — one per op run (the faithful op log is preserved). + require.Greater(t, len(segs), n, "chronological seal keeps one frame per op run") + // ...but they pack into a SINGLE default spool file — not len(segs) files. + require.Equal(t, 1, distinctPaths(segs), "tiny frames pack into one spool file, not one file per frame") + + // LWW still correct after packing: each pk updated to its new text, old gone. + idx := framesToIndex(t, base, segs) + for i := int64(1); i <= n; i++ { + require.Emptyf(t, queryIDs(t, idx, fmt.Sprintf("old%d", i)), "old version of pk %d gone", i) + require.Equalf(t, []any{i}, queryIDs(t, idx, fmt.Sprintf("new%d", i)), "updated pk %d live", i) + } + + // With a roll cap of ~1/4 the total frame bytes the spool ROLLS to a handful of files, but the + // count stays bounded (bytes/rollCap ≈ 4–5) — far below one file per frame — and LWW is + // unaffected by which file a frame lands in. + var total int64 + for _, s := range segs { + total += int64(s.FrameLen) + } + tb2, err := NewTailBuilder(int32(types.T_int64), 1000, 0, "", wsTokenize) + require.NoError(t, err) + defer tb2.Cleanup() + tb2.rollCap = total / 4 // force a few rolls, many frames per file + for _, c := range blobs { + require.NoError(t, tb2.AddBatch(c)) + } + segs2, err := tb2.Finish() + require.NoError(t, err) + files := distinctPaths(segs2) + require.GreaterOrEqual(t, files, 2, "roll cap forces multiple spool files") + require.Less(t, files, len(segs2)/10, "each spool file still packs many frames (not one-per-file)") + idx2 := framesToIndex(t, base, segs2) + require.Equal(t, []any{int64(1)}, queryIDs(t, idx2, "new1"), "LWW holds across rolled spool files") + require.Empty(t, queryIDs(t, idx2, "old1")) +} + +func queryIDs(t *testing.T, idx *Index, word string) []any { + t.Helper() + res, err := idx.SearchQuery([]byte(word), false, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(res) +} + +// TestDirtyVsCompactedScoreParity pins the live-DF fix: a base+tail carrying DEAD copies +// (rows updated in the tail, their stale base copies not yet reclaimed) must score and rank a +// query IDENTICALLY to a clean index holding only the same LIVE rows. Before live-DF, df was +// summed over physical postings (dead copies included), inflating df past the live N and giving +// the dirty corpus different idf — hence a different ranking — until a MERGE reclaimed the dead +// copies. Covers BM25 and TF-IDF across EVERY path that consumes global DF: WAND bag-of-words +// disjunction and boolean OR (term df), and NL / boolean phrase clauses (phrase df). +func TestDirtyVsCompactedScoreParity(t *testing.T) { + // The final LIVE state a compacted index holds. + live := map[int64]string{ + 1: "x y y", + 2: "x x y", + 3: "y y y", + } + compacted := NewIndex([]*Segment{mkBase(t, live)}, nil) + + // Dirty: a base of STALE versions (extra "x" copies inflate df(x)) + a tail that upserts + // every pk to its live text, leaving the stale base copies dead (df-inflating until MERGE). + dirtyBase := mkBase(t, map[int64]string{1: "x x x x", 2: "x x x x", 3: "x x x x"}) + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "x y y", nil) + c.Upsert(int64(2), "x x y", nil) + c.Upsert(int64(3), "y y y", nil) + dirty := buildTailIndexLWW(t, dirtyBase, c, 1000) + + require.Equal(t, compacted.NumDocs(), dirty.NumDocs(), "same live doc count") + + scoresOf := func(res []Result, err error) map[int64]float64 { + require.NoError(t, err) + m := make(map[int64]float64, len(res)) + for _, r := range res { + m[r.Pk.(int64)] = float64(r.Score) + } + return m + } + // Every scoring path that reads global DF, so both the term df() (WAND) and the phraseDf() + // live-fixes are exercised — SearchQuery(boolean=false) alone would only hit the phrase path. + modes := []struct { + name string + run func(idx *Index, algo ScoreAlgo, q string) map[int64]float64 + }{ + {"wand-bag", func(idx *Index, algo ScoreAlgo, q string) map[int64]float64 { // WAND disjunction -> df() + return scoresOf(idx.SearchBagOfWords([]byte(q), ParserDefault, algo, 100, nil)) + }}, + {"bool-or", func(idx *Index, algo ScoreAlgo, q string) map[int64]float64 { // boolean OR -> WAND -> df() + return scoresOf(idx.SearchQuery([]byte(q), true, ParserDefault, algo, 100, nil)) + }}, + {"nl-phrase", func(idx *Index, algo ScoreAlgo, q string) map[int64]float64 { // NL / phrase -> phraseDf() + return scoresOf(idx.SearchQuery([]byte(q), false, ParserDefault, algo, 100, nil)) + }}, + } + for _, algo := range []ScoreAlgo{BM25, TfIdf} { + for _, m := range modes { + for _, q := range []string{"x", "y", "x y"} { + cs := m.run(compacted, algo, q) + ds := m.run(dirty, algo, q) + require.Equalf(t, len(cs), len(ds), "algo=%v mode=%s q=%q same match count", algo, m.name, q) + for pk, cscore := range cs { + dscore, ok := ds[pk] + require.Truef(t, ok, "algo=%v mode=%s q=%q pk %d present in dirty result", algo, m.name, q, pk) + require.InDeltaf(t, cscore, dscore, 1e-6, "algo=%v mode=%s q=%q pk %d dirty-vs-compacted score parity", algo, m.name, q, pk) + } + } + } + } +} + +// TestBuilderSetDocReplace pins the LWW primitive: SetDoc REPLACES a pk's terms (not +// append), and a zero-word SetDoc keeps a live 0-term doc (the empty-upsert shadow). +func TestBuilderSetDocReplace(t *testing.T) { + b := NewBuilder("b", int32(types.T_int64)) + b.SetDoc(int64(1), []WordPos{{Word: "alpha", Pos: 0}}, nil) + b.SetDoc(int64(1), []WordPos{{Word: "beta", Pos: 0}, {Word: "gamma", Pos: 6}}, nil) + require.Equal(t, 1, b.NumDocs()) // same pk => one doc + require.Equal(t, 2, b.NumPostings()) // replaced: 1 -> 2, not appended to 3 + require.Equal(t, []string{"beta", "gamma"}, b.docs[0].Terms) + + // zero-word upsert => a retained 0-term doc (carries pk + recency to shadow). + b.SetDoc(int64(2), nil, nil) + require.Equal(t, 2, b.NumDocs()) + require.Empty(t, b.docs[1].Terms) + require.Equal(t, int64(2), b.docs[1].Pk) +} + +// TestCdcUpsertReplaceNoMerge: two upserts of one pk in a flush must not leave BOTH +// terms searchable (the merge bug). Only the final version survives. +func TestCdcUpsertReplaceNoMerge(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha"}) + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "beta", nil) + c.Upsert(int64(1), "gamma", nil) + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha"), "base term superseded by the upsert") + require.Empty(t, queryIDs(t, idx, "beta"), "intermediate upsert term must NOT linger (no merge)") + require.Equal(t, []any{int64(1)}, queryIDs(t, idx, "gamma"), "only the final upsert term is live") +} + +// TestCdcUpsertEmptyShadows: an upsert to empty text (also how a NULL indexed column +// arrives — rowText returns "") must shadow the old indexed version. A row with no +// searchable content must stop matching its old terms. +func TestCdcUpsertEmptyShadows(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha", 2: "delta"}) + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "", nil) // empty / NULL text + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha"), "emptied row must no longer match its old term") + require.Equal(t, []any{int64(2)}, queryIDs(t, idx, "delta"), "untouched row still matches") +} + +// TestCdcNonemptyThenEmptySameFlush: nonempty upsert then empty upsert of the SAME pk +// in one flush — the final (empty) state wins via in-segment replace, so neither the +// base term nor the intermediate term matches. This case is NOT masked by the +// source-join (the pk still exists), so the engine must get it right itself. +func TestCdcNonemptyThenEmptySameFlush(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha"}) + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "beta", nil) + c.Upsert(int64(1), "", nil) // then emptied + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha")) + require.Empty(t, queryIDs(t, idx, "beta"), "final empty state shadows the intermediate term") +} + +// TestCdcUpsertReplaceAcrossCapacity: a repeated-pk upsert that straddles a sealed +// segment boundary — the newer copy (higher-recency segment) wins via liveness, the +// stale copy in the earlier sealed segment is dead. +func TestCdcUpsertReplaceAcrossCapacity(t *testing.T) { + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "aaa", nil) + c.Upsert(int64(2), "bbb", nil) // seals seg0 (capacity 2) + c.Upsert(int64(1), "ccc", nil) // pk 1 again, now in seg1 + idx := buildTailIndexLWW(t, nil, c, 2) + + require.Empty(t, queryIDs(t, idx, "aaa"), "stale copy of pk 1 in the sealed segment is superseded") + require.Equal(t, []any{int64(2)}, queryIDs(t, idx, "bbb")) + require.Equal(t, []any{int64(1)}, queryIDs(t, idx, "ccc"), "newest copy of pk 1 wins") +} + +// TestCdcUpsertThenDeleteLWW: an UPSERT followed by a DELETE of the same pk in one flush +// resolves to the DELETE (last-writer-wins collapse) — the row is gone from search, no phantom. +// Previously the delete framed below the insert (delete-first) so the engine still returned the +// upserted term; that phantom was masked only by the source-table join, which the covered +// 0-JOIN path removes. The collapse fixes it at the source (no pk is both an insert and a +// delete), so BOTH the base term and the upserted term are gone. +func TestCdcUpsertThenDeleteLWW(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha"}) + c := NewCdc(int32(types.T_int64)) + c.Upsert(int64(1), "beta", nil) + c.Delete(int64(1)) + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha"), "base term superseded by the delete") + require.Empty(t, queryIDs(t, idx, "beta"), "upsert-then-delete: the row is gone, no phantom") +} + +// TestCdcDeleteThenUpsertLWW: an UPDATE (DELETE then INSERT of the same pk in one flush) +// resolves to the INSERT — the new version is live and supersedes the base. Confirms the LWW +// collapse does not break UPDATE (the case delete-first framing existed to protect). +func TestCdcDeleteThenUpsertLWW(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha"}) + c := NewCdc(int32(types.T_int64)) + c.Delete(int64(1)) + c.Upsert(int64(1), "beta", nil) + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha"), "old version superseded") + require.Equal(t, []any{int64(1)}, queryIDs(t, idx, "beta"), "update: new version is live") +} + +// TestCdcPlainDelete: a straight delete (no preceding same-flush upsert) removes the +// base doc — the delete tombstone (higher recency) shadows the base copy. +func TestCdcPlainDelete(t *testing.T) { + base := mkBase(t, map[int64]string{1: "alpha", 2: "delta"}) + c := NewCdc(int32(types.T_int64)) + c.Delete(int64(1)) + idx := buildTailIndexLWW(t, base, c, 1000) + + require.Empty(t, queryIDs(t, idx, "alpha"), "deleted doc is gone") + require.Equal(t, []any{int64(2)}, queryIDs(t, idx, "delta")) +} diff --git a/pkg/fulltext2/cdc_test.go b/pkg/fulltext2/cdc_test.go new file mode 100644 index 0000000000000..70c7d56957d83 --- /dev/null +++ b/pkg/fulltext2/cdc_test.go @@ -0,0 +1,200 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CDC write-path round trip (engine layer, no live DB): Cdc/DeleteLog codecs and +// the streaming TailBuilder -> frames -> Index-with-liveness path that the ISCP +// consumer drives end-to-end. +package fulltext2 + +import ( + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func wsTokenize(s string) []WordPos { + fields := strings.Fields(s) + out := make([]WordPos, 0, len(fields)) + i := 0 + for _, f := range fields { + pos := i + strings.Index(s[i:], f) + out = append(out, WordPos{Word: f, Pos: int32(pos)}) + i = pos + len(f) + } + return out +} + +func TestCdcCodecRoundTrip(t *testing.T) { + c := NewCdc(int32(types.T_int64)) + c.Insert(int64(2), "cherry pie", nil) + c.Delete(int64(0)) + c.Upsert(int64(1), "blueberry", nil) + require.Equal(t, 3, c.Len()) + + blob, err := c.Encode() + require.NoError(t, err) + got, err := DecodeCdc(blob) + require.NoError(t, err) + require.Equal(t, c.PkType, got.PkType) + require.Equal(t, c.Events, got.Events) + + // corruption is caught. + blob[len(blob)-1] ^= 0xFF + _, err = DecodeCdc(blob) + require.Error(t, err) +} + +func TestDeleteLogRoundTrip(t *testing.T) { + // varlena pk (length-prefixed) and int pk (bare) both round-trip. + for _, tc := range []struct { + pkType int32 + recs []DeleteRecord + }{ + {int32(types.T_int64), []DeleteRecord{{int64(0)}, {int64(7)}}}, + {int32(types.T_varchar), []DeleteRecord{{[]byte("a")}, {[]byte("bb")}}}, + } { + blob, err := EncodeDeleteLog(tc.pkType, tc.recs) + require.NoError(t, err) + got, err := DecodeDeleteLog(blob) + require.NoError(t, err) + require.Len(t, got, len(tc.recs)) + for i := range tc.recs { + require.Equal(t, normalizeKey(tc.recs[i].Pk), normalizeKey(got[i].Pk)) + } + } +} + +// TestTailBuilderLiveness drives the whole write path: a base segment + a CDC +// batch (insert new / delete old / update existing) -> TailBuilder frames -> +// Index with liveness. The deleted doc disappears, the updated doc's new text +// wins over the base copy, and the new doc appears. +func TestTailBuilderLiveness(t *testing.T) { + // tag=0 base (recency 0): doc 0 apple, doc 1 banana. + bb := NewBuilder("base", int32(types.T_int64)) + feed(t, bb, int64(0), "apple") + feed(t, bb, int64(1), "banana") + base, err := bb.Finish() + require.NoError(t, err) + base.Recency = 0 + + // One CDC flush: insert doc 2, delete doc 0, update doc 1's text. + c := NewCdc(int32(types.T_int64)) + c.Insert(int64(2), "cherry", nil) + c.Delete(int64(0)) + c.Upsert(int64(1), "blueberry", nil) + + tb, err := NewTailBuilder(int32(types.T_int64), 1000, 0, "", wsTokenize) + require.NoError(t, err) + defer tb.Cleanup() + require.NoError(t, tb.AddBatch(c)) + segs, err := tb.Finish() + require.NoError(t, err) + require.NotEmpty(t, segs) + + // Persist simulation: assign recencies in Finish() order (delete frame first, + // then insert segments), all above the base recency (0). + tails := []*Segment{base} + deletes := map[any]int64{} + recency := int64(100) + for i, ts := range segs { + framed := readTailFrame(t, ts) + seg, dels, uerr := UnframeTail("tail", framed) + require.NoError(t, uerr) + switch { + case seg != nil: + seg.Recency = recency + int64(i) + tails = append(tails, seg) + case dels != nil: + deletes = foldDeleteFrame(deletes, dels, recency+int64(i)) + } + } + + idx := NewIndex(tails, deletes) + q := func(w string) []any { + res, qerr := idx.SearchQuery([]byte(w), false, ParserDefault, BM25, 100, nil) + require.NoError(t, qerr) + return resultIDs(res) + } + require.Empty(t, q("apple"), "deleted doc 0 must be gone") + require.Empty(t, q("banana"), "updated doc 1's old text must be superseded") + require.Equal(t, []any{int64(1)}, q("blueberry"), "updated doc 1's new text wins") + require.Equal(t, []any{int64(2)}, q("cherry"), "inserted doc 2 present") +} + +// TestTailBuilderDeleteSpill: at a small capacity the delete batch spills into +// MULTIPLE delete frames instead of one. Folded liveness must be identical to the +// single-frame case — every tombstone still sorts below every insert — including a +// base-doc delete that lands in the last delete frame and an update that crosses the +// spill boundary. +func TestTailBuilderDeleteSpill(t *testing.T) { + c := NewCdc(int32(types.T_int64)) + // 5 no-op deletes + an update of base doc 1 (insert-side) + delete of base doc 0 + // + a fresh insert. Interleaved so deletes cross several spill boundaries. + c.Delete(int64(10)) + c.Delete(int64(11)) + c.Delete(int64(12)) + c.Delete(int64(13)) + c.Delete(int64(14)) + c.Upsert(int64(1), "blueberry muffin", nil) + c.Delete(int64(0)) + c.Insert(int64(2), "cherry cake", nil) + + // base (recency 0): doc 0 "apple pie", doc 1 "banana bread". + bb := NewBuilder("base", int32(types.T_int64)) + feed(t, bb, int64(0), "apple", "pie") + feed(t, bb, int64(1), "banana", "bread") + base, err := bb.Finish() + require.NoError(t, err) + base.Recency = 0 + + // capacity 2 => 6 deletes (10,11,12,13,14,0) seal into >=3 delete frames. + tb, err := NewTailBuilder(int32(types.T_int64), 2, 0, "", wsTokenize) + require.NoError(t, err) + defer tb.Cleanup() + require.NoError(t, tb.AddBatch(c)) + segs, err := tb.Finish() + require.NoError(t, err) + + tails := []*Segment{base} + deletes := map[any]int64{} + nDeleteFrames := 0 + recency := int64(100) + for i, ts := range segs { + framed := readTailFrame(t, ts) + seg, dels, uerr := UnframeTail("tail", framed) + require.NoError(t, uerr) + switch { + case seg != nil: + seg.Recency = recency + int64(i) + tails = append(tails, seg) + case dels != nil: + nDeleteFrames++ + deletes = foldDeleteFrame(deletes, dels, recency+int64(i)) + } + } + require.GreaterOrEqual(t, nDeleteFrames, 3, "6 deletes at capacity 2 must spill into multiple delete frames") + + idx := NewIndex(tails, deletes) + q := func(w string) []any { + res, qerr := idx.SearchQuery([]byte(w), false, ParserDefault, BM25, 100, nil) + require.NoError(t, qerr) + return resultIDs(res) + } + require.Empty(t, q("apple"), "deleted base doc 0 (tombstone in the last spilled frame) must be gone") + require.Empty(t, q("banana"), "updated doc 1's old base text superseded") + require.Equal(t, []any{int64(1)}, q("blueberry"), "updated doc 1's new text wins across the delete spill") + require.Equal(t, []any{int64(2)}, q("cherry"), "inserted doc 2 present") +} diff --git a/pkg/fulltext2/chinese_test.go b/pkg/fulltext2/chinese_test.go new file mode 100644 index 0000000000000..78a1b167fa6d1 --- /dev/null +++ b/pkg/fulltext2/chinese_test.go @@ -0,0 +1,105 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Chinese (CJK) search coverage over a jieba-friendly corpus (ported from the +// classic fulltext jieba tests' vocabulary: 北京/清华大学/苹果/香蕉/天安门). gojieba +// segments into dictionary words so NL is exact-phrase and boolean operands are +// whole words; ngram tiles CJK into 3-grams so NL is bag-of-words. +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// zhSimplified indexes the corpus under parser and returns a queryable Index. +// +// 0: 我来到北京清华大学 (我 来到 北京 清华大学) +// 1: 苹果香蕉都好吃 (苹果 香蕉 都 好吃) +// 2: 我爱北京天安门 (我 爱 北京 天安门) +// 3: 清华大学在北京 (清华大学 在 北京) +// 4: 香蕉和苹果 (香蕉 和 苹果) +func zhSimplified(t *testing.T, parser string) *Index { + docs := []Doc{ + {int64(0), []byte("我来到北京清华大学")}, + {int64(1), []byte("苹果香蕉都好吃")}, + {int64(2), []byte("我爱北京天安门")}, + {int64(3), []byte("清华大学在北京")}, + {int64(4), []byte("香蕉和苹果")}, + } + seg, err := BuildSegmentFromDocsParser("zh", int32(types.T_int64), docs, parser) + require.NoError(t, err) + return NewIndex([]*Segment{seg}, nil) +} + +// --- gojieba: dictionary word segmentation --- + +func TestGojiebaWordSearch(t *testing.T) { + requireJieba(t) + idx := zhSimplified(t, ParserGojieba) + // single jieba words. + require.ElementsMatch(t, []any{int64(0), int64(2), int64(3)}, nlIDs(t, idx, ParserGojieba, "北京")) + require.ElementsMatch(t, []any{int64(0), int64(3)}, nlIDs(t, idx, ParserGojieba, "清华大学")) + require.ElementsMatch(t, []any{int64(1), int64(4)}, nlIDs(t, idx, ParserGojieba, "苹果")) + require.ElementsMatch(t, []any{int64(2)}, nlIDs(t, idx, ParserGojieba, "天安门")) +} + +func TestGojiebaPhrase(t *testing.T) { + requireJieba(t) + idx := zhSimplified(t, ParserGojieba) + // NL is exact phrase of jieba words: 北京·清华大学 is contiguous only in doc 0 + // (doc 3 is 清华大学·在·北京 — the words appear in the other order). + require.ElementsMatch(t, []any{int64(0)}, nlIDs(t, idx, ParserGojieba, "北京清华大学")) +} + +func TestGojiebaBoolean(t *testing.T) { + requireJieba(t) + idx := zhSimplified(t, ParserGojieba) + // +北京 -苹果: 北京 present, 苹果 absent. + require.ElementsMatch(t, []any{int64(0), int64(2), int64(3)}, boolIDs(t, idx, ParserGojieba, "+北京 -苹果")) + // +苹果 +香蕉: both present. + require.ElementsMatch(t, []any{int64(1), int64(4)}, boolIDs(t, idx, ParserGojieba, "+苹果 +香蕉")) + // bag-of-words OR. + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, boolIDs(t, idx, ParserGojieba, "苹果 天安门")) + // group: +北京 +(清华大学 天安门). + require.ElementsMatch(t, []any{int64(0), int64(2), int64(3)}, boolIDs(t, idx, ParserGojieba, "+北京 +(清华大学 天安门)")) +} + +// --- ngram: 3-gram tiling, CJK NL = bag-of-words --- + +func TestNgramChineseSearch(t *testing.T) { + idx := zhSimplified(t, ParserNgram) + // 清华大学 -> 3-grams 清华大/华大学 (bag); docs 0 and 3 contain them. + require.ElementsMatch(t, []any{int64(0), int64(3)}, nlIDs(t, idx, ParserNgram, "清华大学")) + // +苹果 +香蕉 boolean over ngrams: docs 1 and 4. + require.ElementsMatch(t, []any{int64(1), int64(4)}, boolIDs(t, idx, ParserNgram, "+苹果 +香蕉")) +} + +// --- non-int pk carrying Chinese (covers builderCopyPk/[]byte pk path) --- + +func TestGojiebaBytesPk(t *testing.T) { + requireJieba(t) + docs := []Doc{ + {[]byte("doc-甲"), []byte("北京清华大学")}, + {[]byte("doc-乙"), []byte("上海交通大学")}, + } + seg, err := BuildSegmentFromDocsParser("zhpk", int32(types.T_varchar), docs, ParserGojieba) + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + got := nlIDs(t, idx, ParserGojieba, "清华大学") + require.Len(t, got, 1) + require.Equal(t, "doc-甲", string(got[0].([]byte))) +} diff --git a/pkg/fulltext2/compact.go b/pkg/fulltext2/compact.go new file mode 100644 index 0000000000000..6deb66416e800 --- /dev/null +++ b/pkg/fulltext2/compact.go @@ -0,0 +1,177 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "time" + + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// CompactSegments folds the tag=1 CdcTail into the tag=0 base and reclaims dead +// space: it loads base + tail + deletes, reconstructs the LIVE docs from their +// positional postings (no re-tokenize), rebuilds capacity-bounded fresh tag=0 +// bases, and atomically replaces ALL prior bases + the whole tail — within the +// caller's (TVF statement) transaction. This is a FULL compaction, simpler than +// bm25's incremental tail-fold; an incremental variant is a later optimization. +// Returns the live-doc count. No tail delta → nothing to reclaim (returns 0). +func CompactSegments(sqlproc *sqlexec.SqlProcess, cfg TableConfig, capacity, postingCap int64) (int, error) { + bases, err := LoadAllBases(sqlproc, cfg) + if err != nil { + return 0, err + } + tails, deletes, err := LoadTailSegments(sqlproc, cfg) + if err != nil { + freeSegs(bases) + return 0, err + } + // Free the loaded (off-heap) base + tail buffers on every return path; the fresh + // rebuilt segments below are build-side (Go heap, no deallocators — Free no-op). + defer freeSegs(bases) + defer freeSegs(tails) + if len(tails) == 0 && len(deletes) == 0 { + return 0, nil // no CDC delta since the last build/compaction + } + + // Recency for the fresh base: above every existing base + tail chunk_id, so a + // later tail append (NextTailChunkId = base+1) stays monotonic and newer. + recency, err := NextTailChunkId(sqlproc, cfg) + if err != nil { + return 0, err + } + + segs := append(bases, tails...) + pkType := int32(0) + if len(segs) > 0 { + pkType = segs[0].PkType + } + idx := NewIndex(segs, deletes) + if capacity <= 0 { + capacity = DefaultBuildCapacity // floor so the fresh base is sealed+spilled per ~1M docs + } + if postingCap <= 0 { + postingCap = DefaultPostingCapacity // floor so a long-doc corpus's rebuild stays memory-bounded + } + + ts := time.Now().UnixMicro() + uid := fmt.Sprintf("%s:%d", cfg.IndexTable, ts) + var bopts []BuildOpt + if cfg.PositionFree { + bopts = append(bopts, WithPositionFree()) + } + // Carry the INCLUDE schema into the rebuilt base so the fresh segments keep the docmap + // include section (the source segments all share one schema; read the first that has it). + var incTypes []int32 + for _, s := range segs { + if s.nIncludeCols() > 0 { + incTypes = s.includeTypes + break + } + } + if len(incTypes) > 0 { + bopts = append(bopts, WithIncludeTypes(incTypes)) + } + + // Atomic replace, all in the caller's (TVF statement) txn: drop every prior base + + // the whole tail FIRST, then STREAM the fresh dead-doc-free base(s). Deleting first + // is required because DeleteAllBases is indiscriminate over tag=0 — writing the fresh + // bases first would let it wipe them. It is safe because ReconstructLiveDocs reads the + // already-loaded in-memory idx (not the storage rows), and a mid-stream failure rolls + // the whole statement back. + for _, s := range DeleteAllBasesSqls(cfg) { + if e := runCompactSql(sqlproc, s); e != nil { + return 0, e + } + } + for _, s := range DeleteTailSqls(cfg) { + if e := runCompactSql(sqlproc, s); e != nil { + return 0, e + } + } + + // Stream the reconstructed live docs into a capacity-bounded builder, sealing + + // persisting + freeing each fresh base segment as it fills — so peak build memory is + // one open segment, not the whole rebuilt corpus. + var cur *Builder + segIdx := 0 + nlive := 0 + sealFresh := func() error { + if cur == nil || cur.NumDocs() == 0 { + cur = nil + return nil + } + seg, err := cur.Finish() + cur = nil + if err != nil { + return err + } + seg.Id = SubIndexId(uid, segIdx) + seg.Recency = recency + segIdx++ + sqls, cleanup, e := seg.ToInsertSqls(sqlproc, cfg, ts, 0 /* tag=0 base */) + if e != nil { + return e + } + defer cleanup() + return runCompactSqls(sqlproc, sqls) + } + for d, derr := range idx.ReconstructLiveDocs(cfg.PositionFree) { + if derr != nil { + return 0, derr + } + if cur == nil { + cur = NewBuilder(uid, pkType, bopts...) + } + for i, w := range d.Terms { + if e := cur.Add(w, d.Positions[i], d.Pk); e != nil { + return 0, e + } + } + // Attach the doc's INCLUDE values (only for docs the Add loop created; a 0-term + // shadow doc is intentionally dropped by MERGE — nothing left to shadow, never matches). + if len(incTypes) > 0 && len(d.Terms) > 0 { + cur.SetInclude(d.Pk, d.Include) + } + nlive++ + if ReachedSegmentCap(cur, capacity, postingCap) { + if e := sealFresh(); e != nil { + return 0, e + } + } + } + if e := sealFresh(); e != nil { + return 0, e + } + return nlive, nil +} + +func runCompactSql(sqlproc *sqlexec.SqlProcess, sql string) error { + res, err := runSql(sqlproc, sql) + if err != nil { + return err + } + res.Close() + return nil +} + +func runCompactSqls(sqlproc *sqlexec.SqlProcess, sqls []string) error { + for _, s := range sqls { + if err := runCompactSql(sqlproc, s); err != nil { + return err + } + } + return nil +} diff --git a/pkg/fulltext2/compact_test.go b/pkg/fulltext2/compact_test.go new file mode 100644 index 0000000000000..7fc4ad5866a0b --- /dev/null +++ b/pkg/fulltext2/compact_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// MERGE-compaction engine: ReconstructLiveDocs recovers each live doc's ordered +// terms from the positional postings (no re-tokenize), dropping dead/superseded +// copies — the input the fulltext2_compact TVF rebuilds a fresh base from. +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// loadedSeg round-trips a built segment through serialize so reconstruction runs +// over the FST/loaded path the storage loader produces (not the build-side map). +func loadedSeg(t *testing.T, b *Builder) *Segment { + seg, err := b.Finish() + require.NoError(t, err) + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize(seg.Id, bytes.NewReader(blob)) + require.NoError(t, err) + return loaded +} + +// collectLiveDocs drains the ReconstructLiveDocs iterator into a slice for +// assertions (the production consumer streams it one doc at a time). +func collectLiveDocs(idx *Index, positionFree bool) ([]TokenizedDoc, error) { + out := make([]TokenizedDoc, 0, idx.globalN) + for d, err := range idx.ReconstructLiveDocs(positionFree) { + if err != nil { + return nil, err + } + out = append(out, d) + } + return out, nil +} + +func TestReconstructLiveDocs(t *testing.T) { + // base (recency 0): doc 0 "apple pie", doc 1 "banana bread". + bb := NewBuilder("base", int32(types.T_int64)) + feed(t, bb, int64(0), "apple", "pie") + feed(t, bb, int64(1), "banana", "bread") + base := loadedSeg(t, bb) + base.Recency = 0 + + // tail (recency 100): insert doc 2, update doc 1's text (supersedes base copy). + tb := NewBuilder("tail", int32(types.T_int64)) + feed(t, tb, int64(2), "cherry", "cake") + feed(t, tb, int64(1), "blueberry", "muffin") + tail := loadedSeg(t, tb) + tail.Recency = 100 + + // delete doc 0 at recency 100. + idx := NewIndex([]*Segment{base, tail}, map[any]int64{normalizeKey(int64(0)): 100}) + + docs, err := collectLiveDocs(idx, false) + require.NoError(t, err) + + got := map[any][]string{} + for _, d := range docs { + got[normalizeKey(d.Pk)] = d.Terms + } + // doc 0 deleted, doc 1 = updated tail copy, doc 2 inserted. Term ORDER preserved. + require.NotContains(t, got, normalizeKey(int64(0))) + require.Equal(t, []string{"blueberry", "muffin"}, got[normalizeKey(int64(1))]) + require.Equal(t, []string{"cherry", "cake"}, got[normalizeKey(int64(2))]) + require.Len(t, docs, 2) + + // Rebuilding from the reconstruction gives an index with identical results. + nb := NewBuilder("merged", int32(types.T_int64)) + for _, d := range docs { + for i, w := range d.Terms { + require.NoError(t, nb.Add(w, d.Positions[i], d.Pk)) + } + } + merged, err := nb.Finish() + require.NoError(t, err) + midx := NewIndex([]*Segment{merged}, nil) + q := func(w string) []any { + res, qerr := midx.SearchQuery([]byte(w), false, ParserDefault, BM25, 100, nil) + require.NoError(t, qerr) + return resultIDs(res) + } + require.Empty(t, q("apple")) + require.Empty(t, q("banana")) + require.Equal(t, []any{int64(1)}, q("blueberry")) + require.Equal(t, []any{int64(2)}, q("cherry")) +} diff --git a/pkg/fulltext2/coverage_extra_test.go b/pkg/fulltext2/coverage_extra_test.go new file mode 100644 index 0000000000000..f07db3efb2ab9 --- /dev/null +++ b/pkg/fulltext2/coverage_extra_test.go @@ -0,0 +1,129 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/stretchr/testify/require" +) + +// TestResultScalarInt64EmptyAndNil covers the two fall-through branches of the generation +// scalar reader: an empty result and a nil batch both yield 0 (never panic on Vecs[0]). +func TestResultScalarInt64EmptyAndNil(t *testing.T) { + require.Equal(t, int64(0), resultScalarInt64(executor.Result{})) // no batches + require.Equal(t, int64(0), resultScalarInt64(executor.Result{Batches: []*batch.Batch{nil}})) // nil batch skipped +} + +// TestCdcReadLenGuards covers the corrupt-input guards of the CDC length-prefixed readers: a +// header too short for the uint32 length, and a length that overruns the remaining bytes. Both +// are defense-in-depth on already-CRC'd data that BVT never exercises. +func TestCdcReadLenGuards(t *testing.T) { + // bytes: only 2 bytes present, uint32 length read fails. + _, err := cdcReadLenBytes(bytes.NewReader([]byte{0x01, 0x02}), int32(types.T_int64)) + require.Error(t, err) + _, err = cdcReadLenString(bytes.NewReader([]byte{0x01, 0x02})) + require.Error(t, err) + + // length=0xFFFF but no payload bytes remain → "truncated". + truncated := []byte{0xFF, 0xFF, 0x00, 0x00} + _, err = cdcReadLenBytes(bytes.NewReader(truncated), int32(types.T_int64)) + require.ErrorContains(t, err, "truncated") + _, err = cdcReadLenString(bytes.NewReader(truncated)) + require.ErrorContains(t, err, "truncated") +} + +// TestDecodePostingsGuards covers the ranking-blob format guards: empty (clears the segment), +// too short for the 1-byte format tag + 8-byte header, and an unsupported format byte. The +// happy V6 path is exercised too so the assignment line is covered. +func TestDecodePostingsGuards(t *testing.T) { + s := &Segment{} + require.NoError(t, s.decodePostings(nil, nil, nil)) // empty → nil postings + require.Nil(t, s.ranking) + + require.ErrorContains(t, s.decodePostings([]byte{1, 2, 3}, nil, nil), "too short") + + bad := append([]byte{0xFF}, make([]byte, 8)...) // 9 bytes, wrong format tag + require.ErrorContains(t, s.decodePostings(bad, nil, nil), "unsupported postings format") + + ok := append([]byte{postingsFormatV1}, make([]byte, 8)...) + require.NoError(t, s.decodePostings(ok, []byte{1}, []byte{2})) + require.Equal(t, ok, s.ranking) +} + +// TestDecodeTermEntryCorrupt covers the out-of-bounds / bad-uvarint guards in the lazy term +// directory decoder — a corrupt FST value must return (nil,false), never panic or index past +// the ranking slice. +func TestDecodeTermEntryCorrupt(t *testing.T) { + // off past the end of the (empty) ranking. + tp, ok := (&Segment{}).decodeTermEntry(0) + require.False(t, ok) + require.Nil(t, tp) + + // leading uvarint overflows (10 continuation bytes) → first read fails. + overflow := bytes.Repeat([]byte{0xFF}, 10) + _, ok = (&Segment{ranking: overflow}).decodeTermEntry(0) + require.False(t, ok) + + // one valid uvarint then EOF → the second read fails. + _, ok = (&Segment{ranking: []byte{0x01}}).decodeTermEntry(0) + require.False(t, ok) + + // four valid uvarints then nothing → the termMaxTf byte read is out of bounds. + _, ok = (&Segment{ranking: []byte{1, 1, 1, 1}}).decodeTermEntry(0) + require.False(t, ok) +} + +// TestFrameSegmentRoundTrip covers the insert-frame codec: FrameSegment → UnframeSegment recovers +// the same doc count, UnframeTail dispatches the insert frame to the segment (not deletes) arm, and +// a corrupt frame is rejected. +func TestFrameSegmentRoundTrip(t *testing.T) { + b := NewBuilder("f", int32(types.T_int64)) + feed(t, b, int64(0), "quick", "brown") + feed(t, b, int64(1), "quick", "fox") + seg, err := b.Finish() + require.NoError(t, err) + + framed, err := FrameSegment(seg) + require.NoError(t, err) + require.NotEmpty(t, framed) + + seg2, err := UnframeSegment("f", framed) + require.NoError(t, err) + require.Equal(t, seg.numDocs(), seg2.numDocs()) + + // UnframeTail on the same insert frame → segment set, no deletes. + s3, dels, err := UnframeTail("f", framed) + require.NoError(t, err) + require.NotNil(t, s3) + require.Nil(t, dels) + + // corrupt frame → error (UnframeCdcChunk rejects it). + _, err = UnframeSegment("f", []byte{0x00, 0x01, 0x02}) + require.Error(t, err) +} + +// TestAsBytes covers the pk raw-bytes coercion: []byte and string pass through, everything else +// (a boxed fixed-width pk) returns nil so callers fall back to typed encoding. +func TestAsBytes(t *testing.T) { + require.Equal(t, []byte("raw"), asBytes([]byte("raw"))) + require.Equal(t, []byte("str"), asBytes("str")) + require.Nil(t, asBytes(int64(5))) + require.Nil(t, asBytes(nil)) +} diff --git a/pkg/fulltext2/coverage_index_test.go b/pkg/fulltext2/coverage_index_test.go new file mode 100644 index 0000000000000..6c098fb42effe --- /dev/null +++ b/pkg/fulltext2/coverage_index_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "strconv" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestSortKey pins the injective tie-break key for every pk shape: []byte/string pass through, +// the temporal types key by their underlying int64 (so DATETIME(6) sub-second precision is not +// truncated), and everything else falls to %v. +func TestSortKey(t *testing.T) { + require.Equal(t, "raw", sortKey([]byte("raw"))) + require.Equal(t, "str", sortKey("str")) + require.Equal(t, strconv.FormatInt(9_000_001, 10), sortKey(types.Datetime(9_000_001))) + require.Equal(t, strconv.FormatInt(42, 10), sortKey(types.Time(42))) + require.Equal(t, strconv.FormatInt(7, 10), sortKey(types.Timestamp(7))) + require.Equal(t, "123", sortKey(int64(123))) // default %v +} + +// TestBoundedTopK covers the min-heap bounded top-k: it retains only the k highest scores, a +// push below the current k-th best is dropped, and k<=0 is a no-op. +func TestBoundedTopK(t *testing.T) { + b := newBoundedTopK(2) + b.push("a", 1.0, nil) + b.push("b", 3.0, nil) + require.Equal(t, 2, b.len()) + b.push("c", 2.0, nil) // evicts "a" (1.0), keeps {b:3, c:2} + b.push("d", 0.5, nil) // below the k-th best → dropped + require.Equal(t, 2, b.len()) + + res := b.resultsDescScaled(1.0) + require.Len(t, res, 2) + require.Equal(t, "b", res[0].Pk) // score-descending + require.Equal(t, "c", res[1].Pk) + require.Greater(t, res[0].Score, res[1].Score) + + // scale multiplies the retained partial scores. + scaled := b.resultsDescScaled(2.0) + require.Equal(t, float32(6.0), scaled[0].Score) + + // k<=0 → push is a no-op. + z := newBoundedTopK(0) + z.push("x", 5.0, nil) + require.Equal(t, 0, z.len()) +} + +// TestTopKResults covers the FastMaxHeap selection: empty/k<=0 → nil, k>len is clamped, and the +// output is score-descending. +func TestTopKResults(t *testing.T) { + require.Nil(t, topKResults(nil, 3)) + require.Nil(t, topKResults([]Result{{Pk: "a", Score: 1}}, 0)) + + in := []Result{{Pk: "a", Score: 1}, {Pk: "b", Score: 3}, {Pk: "c", Score: 2}} + out := topKResults(in, 2) + require.Len(t, out, 2) + require.Equal(t, "b", out[0].Pk) // highest first + require.Equal(t, "c", out[1].Pk) + + // k > len is clamped to len. + all := topKResults(in, 10) + require.Len(t, all, 3) + require.Equal(t, "b", all[0].Pk) +} diff --git a/pkg/fulltext2/coverage_query_membership_test.go b/pkg/fulltext2/coverage_query_membership_test.go new file mode 100644 index 0000000000000..451bc63e6efaf --- /dev/null +++ b/pkg/fulltext2/coverage_query_membership_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/bytejson" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// TestContainsPkTypes exercises the fast per-PK-type encode branches of docFilterMembership.Contains +// (int64 is covered elsewhere): the uint64 / int32 / uint32 arms must encode byte-identically to the +// docfilter build so even pks pass and odd pks reject, and an out-of-range ord returns false. +func TestContainsPkTypes(t *testing.T) { + mp := mpool.MustNewZero() + + // uint64 + t.Run("uint64", func(t *testing.T) { + b := NewBuilder("u64", int32(types.T_uint64)) + for i := 0; i < 6; i++ { + feed(t, b, uint64(i), "x") + } + seg, err := b.Finish() + require.NoError(t, err) + vec := vector.NewVec(types.New(types.T_uint64, 8, 0)) + for i := 0; i < 6; i += 2 { + require.NoError(t, vector.AppendFixed(vec, uint64(i), false, mp)) + } + fb, err := docfilter.Build(vec) + require.NoError(t, err) + f, err := docfilter.New(fb) + require.NoError(t, err) + defer f.Free() + dfm := &docFilterMembership{seg: seg, f: f} + for i := int64(0); i < 6; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + require.False(t, dfm.Contains(-1)) // out of range low + require.False(t, dfm.Contains(int64(1<<40))) // out of range high + }) + + // int32 + t.Run("int32", func(t *testing.T) { + b := NewBuilder("i32", int32(types.T_int32)) + for i := 0; i < 6; i++ { + feed(t, b, int32(i), "x") + } + seg, err := b.Finish() + require.NoError(t, err) + vec := vector.NewVec(types.New(types.T_int32, 4, 0)) + for i := 0; i < 6; i += 2 { + require.NoError(t, vector.AppendFixed(vec, int32(i), false, mp)) + } + fb, err := docfilter.Build(vec) + require.NoError(t, err) + f, err := docfilter.New(fb) + require.NoError(t, err) + defer f.Free() + dfm := &docFilterMembership{seg: seg, f: f} + for i := int64(0); i < 6; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + }) + + // uint32 + t.Run("uint32", func(t *testing.T) { + b := NewBuilder("u32", int32(types.T_uint32)) + for i := 0; i < 6; i++ { + feed(t, b, uint32(i), "x") + } + seg, err := b.Finish() + require.NoError(t, err) + vec := vector.NewVec(types.New(types.T_uint32, 4, 0)) + for i := 0; i < 6; i += 2 { + require.NoError(t, vector.AppendFixed(vec, uint32(i), false, mp)) + } + fb, err := docfilter.Build(vec) + require.NoError(t, err) + f, err := docfilter.New(fb) + require.NoError(t, err) + defer f.Free() + dfm := &docFilterMembership{seg: seg, f: f} + for i := int64(0); i < 6; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + }) + + // uuid: probed as the RAW 16 bytes (not the canonical string) — the dedicated arm. + t.Run("uuid", func(t *testing.T) { + b := NewBuilder("uuid", int32(types.T_uuid)) + mkUuid := func(i int) types.Uuid { var u types.Uuid; u[0] = byte(i); return u } + for i := 0; i < 6; i++ { + feed(t, b, mkUuid(i), "x") + } + seg, err := b.Finish() + require.NoError(t, err) + vec := vector.NewVec(types.New(types.T_uuid, 16, 0)) + for i := 0; i < 6; i += 2 { + require.NoError(t, vector.AppendFixed(vec, mkUuid(i), false, mp)) + } + fb, err := docfilter.Build(vec) + require.NoError(t, err) + f, err := docfilter.New(fb) + require.NoError(t, err) + defer f.Free() + dfm := &docFilterMembership{seg: seg, f: f} + for i := int64(0); i < 6; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + }) + + // varchar: the default arm (encodePk), not one of the fast integer/uuid cases. + t.Run("varchar", func(t *testing.T) { + b := NewBuilder("vc", int32(types.T_varchar)) + keys := []string{"k0", "k1", "k2", "k3", "k4", "k5"} + for _, k := range keys { + feed(t, b, k, "x") + } + seg, err := b.Finish() + require.NoError(t, err) + vec := vector.NewVec(types.New(types.T_varchar, 64, 0)) + for i := 0; i < 6; i += 2 { + require.NoError(t, vector.AppendBytes(vec, []byte(keys[i]), false, mp)) + } + fb, err := docfilter.Build(vec) + require.NoError(t, err) + f, err := docfilter.New(fb) + require.NoError(t, err) + defer f.Free() + dfm := &docFilterMembership{seg: seg, f: f} + for i := int64(0); i < 6; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + }) +} + +// TestIsJSONParser pins the json-family predicate. +func TestIsJSONParser(t *testing.T) { + require.True(t, IsJSONParser(ParserJSON)) + require.True(t, IsJSONParser(ParserJSONValue)) + require.False(t, IsJSONParser(ParserDefault)) + require.False(t, IsJSONParser("ngram")) + require.True(t, IsJSONValueParser(ParserJSONValue)) + require.False(t, IsJSONValueParser(ParserJSON)) +} + +// TestFlattenJSONColumns covers the input-type dispatch of the two CDC-side json flatteners for +// each accepted form (ByteJson, []byte, string) plus the default (non-json input → nil, nil). +func TestFlattenJSONColumns(t *testing.T) { + raw := `{"a":"hello world","b":42}` + bj, err := bytejson.ParseFromString(raw) + require.NoError(t, err) + + for _, fn := range []func(any) ([]byte, error){FlattenJSONColumn, FlattenJSONValueColumn} { + // ByteJson + out, err := fn(bj) + require.NoError(t, err) + require.NotEmpty(t, out) + // []byte + out, err = fn([]byte(raw)) + require.NoError(t, err) + require.NotEmpty(t, out) + // string + out, err = fn(raw) + require.NoError(t, err) + require.NotEmpty(t, out) + // default (unsupported type) → nil, nil + out, err = fn(12345) + require.NoError(t, err) + require.Nil(t, out) + } +} + +// TestCdcTokenizer covers the three arms: json_value (whole-value tokens), a normal parser (ngram +// words), and an unknown parser (DocTokenizer error propagated). +func TestCdcTokenizer(t *testing.T) { + jv, err := CdcTokenizer(ParserJSONValue) + require.NoError(t, err) + require.NotNil(t, jv) + + def, err := CdcTokenizer(ParserDefault) + require.NoError(t, err) + require.NotNil(t, def) + require.NotEmpty(t, def("hello world")) // tokenizes into words + + _, err = CdcTokenizer("no-such-parser") + require.Error(t, err) +} diff --git a/pkg/fulltext2/deletes.go b/pkg/fulltext2/deletes.go new file mode 100644 index 0000000000000..6217297223210 --- /dev/null +++ b/pkg/fulltext2/deletes.go @@ -0,0 +1,145 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "hash/crc32" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// DeleteRecord is one tombstone: a pk deleted by a CDC batch. Its order is the +// containing frame's chunk_id (assigned at load), not a stored field. Mirrors +// bm25's DeleteRecord. +type DeleteRecord struct { + Pk any +} + +const deleteLogMagic uint32 = 0x46540100 // 'F' 'T' 01 00 — fulltext2 delete log + +// pkFixedWidth reports the fixed byte width of an integer pk type (so the delete +// log can skip the per-pk length prefix); varlena/other types are length-prefixed. +func pkFixedWidth(pkType int32) (int, bool) { + switch t := types.T(pkType); t { + case types.T_int64, types.T_uint64, types.T_int32, types.T_uint32: + return t.FixedLength(), true + default: + return -1, false + } +} + +// EncodeDeleteLog serializes delete records into one self-describing, CRC32-checked +// chunk: magic | pkType | count | pks | crc. A fixed-width int pk is stored bare +// (width implied by pkType); a varlena pk is length-prefixed. Mirrors bm25's +// EncodeDeleteLog. +func EncodeDeleteLog(pkType int32, recs []DeleteRecord) ([]byte, error) { + var w leBuf + w.u32(deleteLogMagic) + w.i32(pkType) + w.i64(int64(len(recs))) + width, fixed := pkFixedWidth(pkType) + for _, r := range recs { + pkb, err := encodePk(pkType, r.Pk) + if err != nil { + return nil, err + } + if fixed { + if len(pkb) != width { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 delete log: pk width %d != %d for fixed type %d", len(pkb), width, pkType) + } + } else { + w.u32(uint32(len(pkb))) + } + w.b.Write(pkb) + } + sum := crc32.ChecksumIEEE(w.b.Bytes()) + w.u32(sum) + return w.b.Bytes(), nil +} + +// DecodeDeleteLog reverses EncodeDeleteLog, validating magic + CRC. +func DecodeDeleteLog(buf []byte) ([]DeleteRecord, error) { + if len(buf) < 4+4+8+4 { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: truncated") + } + body := buf[:len(buf)-4] + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(buf[len(buf)-4:]) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: checksum mismatch") + } + if binary.LittleEndian.Uint32(body[0:4]) != deleteLogMagic { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: bad magic") + } + pkType := int32(binary.LittleEndian.Uint32(body[4:8])) + n := int64(binary.LittleEndian.Uint64(body[8:16])) + if n < 0 { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: bad count") + } + width, fixed := pkFixedWidth(pkType) + // n is off the blob; each record is >= minRec bytes (a fixed pk, or the 4-byte length + // prefix of a varlen pk), so a count that can't fit in the remaining buffer is corrupt. + // Reject it before make() / the loop so a bogus count can't drive a huge allocation. + minRec := 4 + if fixed { + minRec = width + } + if minRec > 0 && n > int64((len(body)-16)/minRec) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: count exceeds buffer") + } + pos := 16 + out := make([]DeleteRecord, 0, n) + for i := int64(0); i < n; i++ { + l := width + if !fixed { + if pos+4 > len(body) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: truncated pk length") + } + l = int(binary.LittleEndian.Uint32(body[pos:])) + pos += 4 + } + if l < 0 || pos+l > len(body) { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 delete log: truncated pk") + } + pk, err := decodePk(pkType, body[pos:pos+l]) + if err != nil { + return nil, err + } + pos += l + out = append(out, DeleteRecord{Pk: pk}) + } + return out, nil +} + +// foldDeleteFrame folds one decoded delete frame (all records share the frame's +// chunk_id) into the pk-key -> max-delete-chunk_id map that Index liveness +// consumes: a copy with Recency < the delete chunk_id is shadowed. Keyed by +// normalizeKey(pk) (matching resolve()); the max chunk_id wins, so re-delivered deletes +// only raise the bound. Mirrors bm25's FoldDeleteFrame (string-keyed here). +func foldDeleteFrame(m map[any]int64, recs []DeleteRecord, chunkId int64) map[any]int64 { + if len(recs) == 0 { + return m + } + if m == nil { + m = make(map[any]int64) + } + for _, r := range recs { + k := normalizeKey(r.Pk) + if cur, ok := m[k]; !ok || chunkId > cur { + m[k] = chunkId + } + } + return m +} diff --git a/pkg/fulltext2/docmap_include_test.go b/pkg/fulltext2/docmap_include_test.go new file mode 100644 index 0000000000000..26c7e68b27a6b --- /dev/null +++ b/pkg/fulltext2/docmap_include_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestDocmapIncludeRoundTrip pins the INCLUDE-values docmap section: a build-side segment +// with per-doc include values (mixed int64 + varchar, including NULLs) encodes and decodes +// back to the same typed values via the loaded-side offset-table accessor, without +// disturbing the pk / docLen sections. +func TestDocmapIncludeRoundTrip(t *testing.T) { + build := &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(10), int64(20), int64(30)}, + docLen: []int32{3, 5, 7}, + includeTypes: []int32{int32(types.T_int64), int32(types.T_varchar)}, + includeVals: [][]any{ + {int64(100), []byte("active")}, + {int64(200), nil}, // NULL varchar + {nil, []byte("archived")}, // NULL int + }, + } + data, err := build.encodeDocmap() + require.NoError(t, err) + + loaded := &Segment{} + require.NoError(t, loaded.decodeDocmap(data)) + require.EqualValues(t, 3, loaded.N) + require.Equal(t, 2, loaded.nIncludeCols()) + + check := func(ord int64, col int, wantVal any, wantNull bool) { + v, isNull, verr := loaded.includeVal(ord, col) + require.NoError(t, verr) + require.Equal(t, wantNull, isNull, "ord=%d col=%d null", ord, col) + if !wantNull { + require.Equal(t, wantVal, v, "ord=%d col=%d val", ord, col) + } + } + check(0, 0, int64(100), false) + check(0, 1, []byte("active"), false) + check(1, 0, int64(200), false) + check(1, 1, nil, true) // NULL varchar + check(2, 0, nil, true) // NULL int + check(2, 1, []byte("archived"), false) + + // pk + docLen sections are unaffected by the trailing INCLUDE section. + require.Equal(t, int64(20), loaded.pk(1)) + require.Equal(t, int32(7), loaded.docLen[2]) + + // out-of-range column returns null, not a panic. + _, isNull, verr := loaded.includeVal(0, 5) + require.NoError(t, verr) + require.True(t, isNull) +} + +// TestDocmapNoIncludeRoundTrip: a segment with no INCLUDE columns produces a docmap with no +// trailing section (byte-identical to before), and decode leaves the include fields nil. +func TestDocmapNoIncludeRoundTrip(t *testing.T) { + build := &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(1), int64(2)}, + docLen: []int32{4, 4}, + } + data, err := build.encodeDocmap() + require.NoError(t, err) + + loaded := &Segment{} + require.NoError(t, loaded.decodeDocmap(data)) + require.Equal(t, 0, loaded.nIncludeCols()) + require.Nil(t, loaded.includeTypes) + + v, isNull, err := loaded.includeVal(0, 0) + require.NoError(t, err) + require.Nil(t, v) + require.True(t, isNull) +} + +// TestDocmapIncludeMismatchLen: encode rejects a build-side segment whose includeVals count +// does not match the doc count (a builder bug), rather than writing a corrupt section. +func TestDocmapIncludeMismatchLen(t *testing.T) { + build := &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(1), int64(2)}, + docLen: []int32{1, 1}, + includeTypes: []int32{int32(types.T_int64)}, + includeVals: [][]any{{int64(9)}}, // only 1 row for 2 docs + } + _, err := build.encodeDocmap() + require.Error(t, err) +} + +// TestDocmapIncludeFixedNoOffsetTable pins the fixed-width optimization: an all-integer +// INCLUDE index builds NO resident varlena offset table (fixed cols are stride-addressed), +// while an index with a varchar col builds an offset table only for the varlena column(s). +func TestDocmapIncludeFixedNoOffsetTable(t *testing.T) { + // All fixed (int64 + int32): zero varlena offsets. + allFixed := &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(1), int64(2)}, + docLen: []int32{1, 1}, + includeTypes: []int32{int32(types.T_int64), int32(types.T_int32)}, + includeVals: [][]any{{int64(10), int32(100)}, {nil, int32(200)}}, + } + data, err := allFixed.encodeDocmap() + require.NoError(t, err) + loaded := &Segment{} + require.NoError(t, loaded.decodeDocmap(data)) + require.Nil(t, loaded.includeVarOffsets, "numeric-only INCLUDE index must build no offset table") + require.Equal(t, 2, loaded.includeLay.nFixed) + require.Equal(t, 0, loaded.includeLay.nVarlena) + // values round-trip (incl. NULL int64 at ord1 col0). + _, isNull, _ := loaded.includeVal(1, 0) + require.True(t, isNull) + v, _, _ := loaded.includeVal(1, 1) + require.Equal(t, int32(200), v) + v, _, _ = loaded.includeVal(0, 0) + require.Equal(t, int64(10), v) + + // Mixed (int64 + varchar): offset table sized only for the 1 varlena column. + mixed := &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(1), int64(2)}, + docLen: []int32{1, 1}, + includeTypes: []int32{int32(types.T_int64), int32(types.T_varchar)}, + includeVals: [][]any{{int64(10), []byte("a")}, {int64(20), nil}}, + } + data, err = mixed.encodeDocmap() + require.NoError(t, err) + lm := &Segment{} + require.NoError(t, lm.decodeDocmap(data)) + require.Len(t, lm.includeVarOffsets, 2, "offset table sized N*nVarlena = 2*1") // only the varchar col + require.Equal(t, 1, lm.includeLay.nFixed) + require.Equal(t, 1, lm.includeLay.nVarlena) + v, _, _ = lm.includeVal(0, 1) + require.Equal(t, []byte("a"), v) + _, isNull, _ = lm.includeVal(1, 1) + require.True(t, isNull) +} diff --git a/pkg/fulltext2/frames.go b/pkg/fulltext2/frames.go new file mode 100644 index 0000000000000..0a5f4b1a23c5f --- /dev/null +++ b/pkg/fulltext2/frames.go @@ -0,0 +1,192 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "fmt" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" +) + +// CDC framing for the tag=1 tail. Fulltext v2 reuses cuVS's payload-agnostic +// FrameCdcChunk envelope (magic + version + op-counts + crc32) — exactly what +// bm25 does — carrying a serialized positional Segment as the insert payload. A +// frame larger than the storage data column is split across consecutive chunks +// at ascending chunk_ids; the load path reassembles them by each frame's +// self-describing header length. The chunk_id is the recency key, assigned by the +// storage layer, never stored inside the frame. + +// FrameSegment serializes seg and wraps it as one insert-segment frame. nInserts +// carries the doc count so a header-only scan can sum delta docs without decoding. +func FrameSegment(seg *Segment) ([]byte, error) { + blob, err := seg.Serialize() + if err != nil { + return nil, err + } + return cuvscdc.FrameCdcChunk(blob, nil, uint32(seg.N), 0, 0), nil +} + +// FrameDeletes encodes a delete batch as one delete-log frame (nDeletes carries +// the tombstone count so a header scan can distinguish it from an insert frame). +// Mirrors bm25's FrameDeletes. +func FrameDeletes(pkType int32, recs []DeleteRecord) ([]byte, error) { + blob, err := EncodeDeleteLog(pkType, recs) + if err != nil { + return nil, err + } + return cuvscdc.FrameCdcChunk(blob, nil, 0, uint32(len(recs)), 0), nil +} + +// UnframeSegment decodes an insert-segment frame back into a loaded Segment +// (id set to id). +func UnframeSegment(id string, framed []byte) (*Segment, error) { + records, _, nInserts, _, _, err := cuvscdc.UnframeCdcChunk(framed) + if err != nil { + return nil, err + } + if nInserts == 0 { + return nil, moerr.NewInternalErrorNoCtx("fulltext2: frame carries no insert segment") + } + return Deserialize(id, bytes.NewReader(records)) +} + +// UnframeTail decodes a tag=1 tail frame into EITHER an insert Segment (id set to +// id, exactly one non-nil) or a delete batch — the same per-frame dispatch +// LoadTailSegments does, exposed for callers/tests that already hold the framed +// bytes (e.g. TailBuilder's spilled files). +func UnframeTail(id string, framed []byte) (*Segment, []DeleteRecord, error) { + records, _, nInserts, nDeletes, _, err := cuvscdc.UnframeCdcChunk(framed) + if err != nil { + return nil, nil, err + } + switch { + case nInserts > 0: + seg, derr := Deserialize(id, bytes.NewReader(records)) + return seg, nil, derr + case nDeletes > 0: + recs, derr := DecodeDeleteLog(records) + return nil, recs, derr + } + return nil, nil, nil +} + +// TailChunk is one raw tag=1 storage row: a MaxChunkSize-bounded piece of a frame +// at its chunk_id. +type TailChunk struct { + ChunkId int64 + Data []byte +} + +// TailFrame is a reassembled complete frame plus the chunk_id of its first chunk +// (its recency key). +type TailFrame struct { + ChunkId int64 + Data []byte +} + +// splitFrameChunks splits a complete frame into MaxChunkSize-bounded chunks at +// consecutive chunk_ids from startChunkId. A frame <= MaxChunkSize yields a single +// chunk. +func splitFrameChunks(startChunkId int64, framed []byte) []TailChunk { + out := make([]TailChunk, 0, (len(framed)+vectorindex.MaxChunkSize-1)/vectorindex.MaxChunkSize) + cid := startChunkId + for off := 0; off < len(framed); off += vectorindex.MaxChunkSize { + end := off + vectorindex.MaxChunkSize + if end > len(framed) { + end = len(framed) + } + out = append(out, TailChunk{ChunkId: cid, Data: framed[off:end]}) + cid++ + } + return out +} + +// orderTailChunks returns the chunks ordered by chunk_id WITHOUT a comparison +// sort: it places each at index (chunk_id - min) in a preallocated slice (O(n)). +// tag=1 chunk_ids are a gapless run (the writer appends consecutive ids, and +// compaction deletes a whole low prefix — never a hole), so [min..max] must span +// exactly len(chunks) ids; a mismatch means a missing or duplicate chunk +// (corruption), reported rather than silently mis-assembled. +func orderTailChunks(chunks []TailChunk) ([]TailChunk, error) { + if len(chunks) == 0 { + return nil, nil + } + minC, maxC := chunks[0].ChunkId, chunks[0].ChunkId + for _, c := range chunks[1:] { + if c.ChunkId < minC { + minC = c.ChunkId + } + if c.ChunkId > maxC { + maxC = c.ChunkId + } + } + span := maxC - minC + 1 + if span != int64(len(chunks)) { + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( + "fulltext2 tail: chunk_id range [%d..%d] spans %d but got %d rows (gap or duplicate)", + minC, maxC, span, len(chunks))) + } + ordered := make([]TailChunk, span) + for _, c := range chunks { + ordered[c.ChunkId-minC] = c + } + return ordered, nil +} + +// reassembleFrames groups chunk_id-ordered rows back into complete frames using +// each frame's self-describing header length (CdcFrameLen): a frame occupies +// consecutive chunks whose bytes sum to that length, and its ordering key is the +// first chunk's chunk_id. Chunks MUST be pre-sorted by chunk_id ascending. +func reassembleFrames(chunks []TailChunk) ([]TailFrame, error) { + // Running count of bytes still available from the current chunk onward. A frame's bytes + // come from CONSECUTIVE chunks, so its self-declared length (CdcFrameLen, read from two + // untrusted uint32s in the first chunk) can never exceed this. Validate against it BEFORE + // make() so a corrupt chunk — start magic intact but garbage length fields declaring a + // near-4GiB frame — is rejected instead of OOM-killing the CN on the allocation. + // checkTailLoadBudget can't catch this: it measures the REAL stored bytes, not the length + // a corrupt header claims. + remaining := 0 + for _, c := range chunks { + remaining += len(c.Data) + } + var frames []TailFrame + i := 0 + for i < len(chunks) { + total, err := cuvscdc.CdcFrameLen(chunks[i].Data) + if err != nil { + return nil, err + } + if total <= 0 || total > remaining { + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( + "fulltext2 tail: frame at chunk %d declares %d bytes but only %d remain across %d chunks (corrupt frame header)", + chunks[i].ChunkId, total, remaining, len(chunks)-i)) + } + firstChunkId := chunks[i].ChunkId + buf := make([]byte, 0, total) + for len(buf) < total && i < len(chunks) { + remaining -= len(chunks[i].Data) + buf = append(buf, chunks[i].Data...) + i++ + } + if len(buf) < total { + return nil, moerr.NewInternalErrorNoCtx("fulltext2 tail: truncated frame (missing chunk rows)") + } + frames = append(frames, TailFrame{ChunkId: firstChunkId, Data: buf[:total]}) + } + return frames, nil +} diff --git a/pkg/fulltext2/frames_test.go b/pkg/fulltext2/frames_test.go new file mode 100644 index 0000000000000..462e1670c36c5 --- /dev/null +++ b/pkg/fulltext2/frames_test.go @@ -0,0 +1,123 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "fmt" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// TestReassembleFramesRejectsCorruptLength pins the OOM guard: a chunk whose START MAGIC is +// intact but whose length fields are garbage (declaring a ~8GiB frame) must be rejected up +// front by the remaining-bytes check, NOT passed to make([]byte, 0, total) which would +// OOM-kill the CN. checkTailLoadBudget cannot catch this — it measures the real stored bytes, +// not the length a corrupt header claims. +func TestReassembleFramesRejectsCorruptLength(t *testing.T) { + const cdcChunkMagic uint32 = 0xCDC51A11 // must match pkg/vectorindex/cuvs cdcChunkMagic + data := make([]byte, 28) // cdcHeaderSize + binary.LittleEndian.PutUint32(data[0:4], cdcChunkMagic) + binary.LittleEndian.PutUint32(data[20:24], 0xFFFFFFF0) // plen: absurd + binary.LittleEndian.PutUint32(data[24:28], 0xFFFFFFF0) // hlen: absurd + _, err := reassembleFrames([]TailChunk{{ChunkId: 7, Data: data}}) + require.Error(t, err) + require.Contains(t, err.Error(), "corrupt frame header") +} + +// bigSegment builds an n-doc segment plus one doc with a unique needle token, so +// its serialization is large enough to span multiple storage chunks. +func bigSegment(t *testing.T, n int) (*Segment, int64) { + t.Helper() + docs := make([]Doc, 0, n+1) + for i := 1; i <= n; i++ { + docs = append(docs, Doc{int64(i), []byte(fmt.Sprintf("word%d alpha beta", i))}) + } + needle := int64(n + 1) + docs = append(docs, Doc{needle, []byte("uniqueneedletoken")}) + s, err := BuildSegmentFromDocs("big", int32(types.T_int64), docs, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + return s, needle +} + +// TestFramesMultiChunkRoundtrip: a large segment framed, split into chunks, +// shuffled, re-ordered O(n), reassembled, and unframed back to a queryable +// segment. Exercises the multi-chunk path. +func TestFramesMultiChunkRoundtrip(t *testing.T) { + seg, needle := bigSegment(t, 4000) + framed, err := FrameSegment(seg) + require.NoError(t, err) + + chunks := splitFrameChunks(0, framed) + if len(framed) > vectorindex.MaxChunkSize { + require.Greater(t, len(chunks), 1, "a segment larger than MaxChunkSize must span multiple chunks") + } + + // storage returns rows in arbitrary order → reverse, then O(n) place. + shuffled := make([]TailChunk, len(chunks)) + for i := range chunks { + shuffled[i] = chunks[len(chunks)-1-i] + } + ordered, err := orderTailChunks(shuffled) + require.NoError(t, err) + frames, err := reassembleFrames(ordered) + require.NoError(t, err) + require.Len(t, frames, 1) + require.EqualValues(t, 0, frames[0].ChunkId) + + loaded, err := UnframeSegment("big", frames[0].Data) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + idx := NewIndex([]*Segment{loaded}, nil) + require.Equal(t, int64(4001), idx.NumDocs()) + require.Equal(t, []any{needle}, iquery(t, idx, "uniqueneedletoken")) +} + +// TestFramesSingleChunk: a small segment yields one chunk at its start chunk_id. +func TestFramesSingleChunk(t *testing.T) { + s, err := BuildSegmentFromDocs("s", int32(types.T_int64), + []Doc{{int64(1), []byte("quick brown fox")}}, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + + framed, err := FrameSegment(s) + require.NoError(t, err) + chunks := splitFrameChunks(5, framed) + require.Len(t, chunks, 1) + require.EqualValues(t, 5, chunks[0].ChunkId) + + frames, err := reassembleFrames(chunks) + require.NoError(t, err) + loaded, err := UnframeSegment("s", frames[0].Data) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + require.Equal(t, int64(1), loaded.N) +} + +// TestOrderTailChunksGap: a missing/duplicate chunk_id is corruption, reported. +func TestOrderTailChunksGap(t *testing.T) { + _, err := orderTailChunks([]TailChunk{{ChunkId: 0}, {ChunkId: 2}}) // gap at 1 + require.Error(t, err) + + ordered, err := orderTailChunks([]TailChunk{{ChunkId: 7}, {ChunkId: 5}, {ChunkId: 6}}) + require.NoError(t, err) + require.EqualValues(t, 5, ordered[0].ChunkId) + require.EqualValues(t, 6, ordered[1].ChunkId) + require.EqualValues(t, 7, ordered[2].ChunkId) +} diff --git a/pkg/fulltext2/fulltext_boolean_port_test.go b/pkg/fulltext2/fulltext_boolean_port_test.go new file mode 100644 index 0000000000000..185249df902ee --- /dev/null +++ b/pkg/fulltext2/fulltext_boolean_port_test.go @@ -0,0 +1,215 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// COMPLETE port of classic fulltext's boolean-mode tests +// (pkg/fulltext/fulltext_test.go: TestFullText{Or,PlusPlus,PlusOr,Minus,Tilda, +// 1,2,3,5,Group,JoinGroupTilda,GroupTilda,Star,Phrase}). The classic tests unit- +// test Pattern.Eval with synthetic per-term-frequency docvecs; here each docvec is +// reconstructed into a real corpus (word repeated tf times) and fulltext2's +// boolean membership is asserted against the SAME expected doc set — proving +// MySQL boolean-mode parity through fulltext2's own parser + WAND evaluator. +package fulltext2 + +import ( + "sort" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// tfSegment reconstructs a classic docvec corpus: doc id contains words[j] +// repeated tf[j] times (space-joined). Docs are added in ascending-id order. +func tfSegment(t *testing.T, words []string, docs map[int64][]int) *Segment { + ids := make([]int64, 0, len(docs)) + for id := range docs { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + ds := make([]Doc, 0, len(ids)) + for _, id := range ids { + var b strings.Builder + for j, w := range words { + for k := 0; k < docs[id][j]; k++ { + if b.Len() > 0 { + b.WriteByte(' ') + } + b.WriteString(w) + } + } + ds = append(ds, Doc{id, []byte(b.String())}) + } + seg, err := BuildSegmentFromDocs("port", int32(types.T_int64), ds, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + return seg +} + +// apple/banana corpus: doc 0 has both, 1 apple-only, 11/12 banana-only. +func abSeg(t *testing.T) *Segment { + return tfSegment(t, []string{"apple", "banana"}, map[int64][]int{ + 0: {2, 2}, 1: {3, 0}, 11: {0, 3}, 12: {0, 4}, + }) +} + +// we/are/so/happy corpus (TestFullText1 shape): one term per doc-group. +func washSeg(t *testing.T) *Segment { + return tfSegment(t, []string{"we", "are", "so", "happy"}, map[int64][]int{ + 0: {2, 0, 0, 0}, 1: {3, 0, 0, 0}, + 10: {0, 2, 0, 0}, 11: {0, 3, 0, 0}, 12: {0, 4, 0, 0}, + 20: {0, 0, 5, 0}, 21: {0, 0, 6, 0}, 22: {0, 0, 7, 0}, 23: {0, 0, 8, 0}, + 30: {0, 0, 0, 1}, 31: {0, 0, 0, 2}, 32: {0, 0, 0, 3}, 33: {0, 0, 0, 4}, + }) +} + +// ---- OR / +/- / ~ (apple-banana corpus) ---- + +// TestFullTextOr: "apple banana" — pure OR; every doc with apple OR banana. +func TestPortFullTextOr(t *testing.T) { + require.ElementsMatch(t, []any{int64(0), int64(1), int64(11), int64(12)}, + bquery(t, abSeg(t), "apple banana")) +} + +// TestFullTextPlusOr: "+apple banana" — apple MUST, banana SHOULD → {0,1}. +func TestPortFullTextPlusOr(t *testing.T) { + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, abSeg(t), "+apple banana")) +} + +// TestFullTextMinus: "+apple -banana" — apple MUST, banana MUST-NOT → {1}. +func TestPortFullTextMinus(t *testing.T) { + require.Equal(t, []any{int64(1)}, bquery(t, abSeg(t), "+apple -banana")) +} + +// TestFullTextPlusPlus (classic apple/orange): "+apple -orange" — orange is in +// doc 0 (docvec col 1), so doc 0 is excluded; only doc 1 matches. +func TestPortFullTextPlusMinus(t *testing.T) { + seg := tfSegment(t, []string{"apple", "orange"}, map[int64][]int{ + 0: {2, 2}, 1: {3, 0}, 11: {0, 3}, 12: {0, 4}, + }) + require.Equal(t, []any{int64(1)}, bquery(t, seg, "+apple -orange")) +} + +// "+apple +banana" — both MUST → only doc 0. +func TestPortFullTextPlusPlus(t *testing.T) { + require.Equal(t, []any{int64(0)}, bquery(t, abSeg(t), "+apple +banana")) +} + +// TestFullTextTilda: "+apple ~banana" — apple MUST, banana a score penalty, not an +// exclusion → docs 0 and 1 both match, the banana-free doc 1 ranked first. +func TestPortFullTextTilda(t *testing.T) { + got := bqueryOrdered(t, abSeg(t), "+apple ~banana") + require.ElementsMatch(t, []any{int64(0), int64(1)}, got) + require.Equal(t, int64(1), got[0]) +} + +// ---- multi-term MUST/SHOULD/MUST-NOT (we/are/so/happy corpus) ---- + +// TestFullText1: "we are so happy" — all SHOULD (OR); every doc matches. Uses +// the Index path (k=100) since all 13 docs match — more than bquery's top-10. +func TestPortFullText1AllOr(t *testing.T) { + idx := NewIndex([]*Segment{washSeg(t)}, nil) + require.ElementsMatch(t, + []any{int64(0), int64(1), int64(10), int64(11), int64(12), + int64(20), int64(21), int64(22), int64(23), + int64(30), int64(31), int64(32), int64(33)}, + boolIDs(t, idx, "", "we are so happy")) +} + +// TestFullText2: "+we +are +so +happy" — all MUST; only the doc with all four. +func TestPortFullText2AllMust(t *testing.T) { + seg := tfSegment(t, []string{"we", "are", "so", "happy"}, map[int64][]int{ + 0: {2, 2, 2, 2}, 1: {3, 0, 0, 0}, 11: {0, 3, 0, 0}, 12: {0, 4, 0, 0}, + 21: {0, 0, 6, 0}, 22: {0, 0, 7, 0}, 23: {0, 0, 8, 0}, + 31: {0, 0, 0, 2}, 32: {0, 0, 0, 3}, 33: {0, 0, 0, 4}, + }) + require.Equal(t, []any{int64(0)}, bquery(t, seg, "+we +are +so +happy")) +} + +// TestFullText3: "+we -are -so -happy" — we MUST, the rest MUST-NOT → doc 1 only +// (doc 0 has 'are', so it is excluded). +func TestPortFullText3MustNot(t *testing.T) { + seg := tfSegment(t, []string{"we", "are", "so", "happy"}, map[int64][]int{ + 0: {2, 2, 0, 0}, 1: {3, 0, 0, 0}, 11: {0, 3, 0, 0}, 12: {0, 4, 0, 0}, + 20: {0, 0, 5, 0}, 21: {0, 0, 6, 0}, 22: {0, 0, 7, 0}, 23: {0, 0, 8, 0}, + 30: {0, 0, 0, 1}, 31: {0, 0, 0, 2}, 32: {0, 0, 0, 3}, 33: {0, 0, 0, 4}, + }) + require.Equal(t, []any{int64(1)}, bquery(t, seg, "+we -are -so -happy")) +} + +// TestFullText5: "we are so +happy" — happy MUST but no doc contains it → empty. +func TestPortFullText5NoMatch(t *testing.T) { + seg := tfSegment(t, []string{"we", "are", "so", "happy"}, map[int64][]int{ + 0: {2, 0, 0, 0}, 1: {3, 0, 0, 0}, 11: {0, 3, 0, 0}, 12: {0, 4, 0, 0}, + }) + require.Empty(t, bquery(t, seg, "we are so +happy")) +} + +// ---- groups + weighted / tilda groups (we/are/so corpus) ---- + +// wasSeg: doc 0 has we+are, doc 1 has we+so, 11/12 are-only, 20-23 so-only. +func wasSeg(t *testing.T) *Segment { + return tfSegment(t, []string{"we", "are", "so"}, map[int64][]int{ + 0: {2, 2, 0}, 1: {3, 0, 5}, 11: {0, 3, 0}, 12: {0, 4, 0}, + 20: {0, 0, 5}, 21: {0, 0, 6}, 22: {0, 0, 7}, 23: {0, 0, 8}, + }) +} + +// TestFullTextGroup: "+we +(so)" — we MUST and (are demoted OR so boosted) +// present → {0,1} (the are-only / so-only docs lack 'we'). +func TestPortFullTextGroup(t *testing.T) { + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, wasSeg(t), "+we +(so)")) +} + +// TestFullTextGroupTilda: "+we ~(so)" — we MUST, the group only adjusts +// score (no inclusion/exclusion) → still {0,1}. +func TestPortFullTextGroupTilda(t *testing.T) { + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, wasSeg(t), "+we ~(so)")) +} + +// TestFullTextJoinGroupTilda: "+we +also ~(so)" — we MUST and also MUST → +// only the docs carrying both (0,1); the ~group merely adjusts score. +func TestPortFullTextJoinGroupTilda(t *testing.T) { + seg := tfSegment(t, []string{"we", "also", "are", "so"}, map[int64][]int{ + 0: {2, 2, 2, 0}, 1: {2, 2, 0, 5}, 11: {0, 0, 3, 0}, 12: {0, 0, 4, 0}, + 20: {0, 0, 0, 5}, 21: {0, 0, 0, 6}, 22: {0, 0, 0, 7}, 23: {0, 0, 0, 8}, + }) + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, seg, "+we +also ~(so)")) +} + +// ---- prefix + phrase ---- + +// TestFullTextStar: "apple*" — prefix matches apple and applesauce, not banana. +func TestPortFullTextStar(t *testing.T) { + seg, err := BuildSegmentFromDocs("star", int32(types.T_int64), []Doc{ + {int64(0), []byte("apple")}, + {int64(1), []byte("applesauce pie")}, + {int64(2), []byte("banana bread")}, + }, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + require.ElementsMatch(t, []any{int64(0), int64(1)}, bquery(t, seg, "apple*")) +} + +// TestFullTextPhrase: "\"we are so happy\"" — exact contiguous phrase; only doc 0 +// has the four words adjacent in order. +func TestPortFullTextPhrase(t *testing.T) { + seg, err := BuildSegmentFromDocs("phrase", int32(types.T_int64), []Doc{ + {int64(0), []byte("we are so happy")}, + {int64(1), []byte("we are happy so")}, + {int64(2), []byte("we are so")}, + {int64(3), []byte("so happy we are")}, + }, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + require.Equal(t, []any{int64(0)}, bquery(t, seg, `"we are so happy"`)) +} diff --git a/pkg/fulltext2/include_boxfree_test.go b/pkg/fulltext2/include_boxfree_test.go new file mode 100644 index 0000000000000..4ab8213cbb140 --- /dev/null +++ b/pkg/fulltext2/include_boxfree_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// boxfreeSeg builds a 4-doc segment with two INCLUDE columns (int64, varchar) and NULLs +// arranged so both the "first value is NULL after non-nulls" (backfill) and "non-null after +// NULL" (parallel append) branches of cbAppendNull/cbAppendVal are exercised: +// +// col0 int64: [100, NULL, 300, NULL] +// col1 varchar: ["active", "b", NULL, NULL] +func boxfreeSeg() *Segment { + return &Segment{ + PkType: int32(types.T_int64), + pks: []any{int64(1), int64(2), int64(3), int64(4)}, + docLen: []int32{1, 1, 1, 1}, + includeTypes: []int32{int32(types.T_int64), int32(types.T_varchar)}, + includeVals: [][]any{ + {int64(100), []byte("active")}, + {nil, []byte("b")}, + {int64(300), nil}, + {nil, nil}, + }, + } +} + +// TestAppendIncludeToBoxFree pins the box-free covered-output path (F4): appendIncludeTo +// encodes each INCLUDE column into a nullable ColumnBuffer without boxing, and +// AppendColumnBuffer decodes it straight into a typed vector preserving values AND NULLs. +// Runs on BOTH a build-side segment (includeVals resident) and a loaded segment +// (includeRaw / offset tables), which take the two arms of appendIncludeTo. +func TestAppendIncludeToBoxFree(t *testing.T) { + mp := mpool.MustNewZero() + + run := func(t *testing.T, seg *Segment) { + // col0: int64 with NULLs at ord 1 and 3. + k0 := &vectorindex.ColumnBuffer{Type: types.T_int64} + for ord := int64(0); ord < 4; ord++ { + require.NoError(t, seg.appendIncludeTo(k0, 0, ord)) + } + require.Equal(t, 4, k0.N) + require.Equal(t, []bool{false, true, false, true}, k0.Nulls) + vec0 := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(k0, vec0, mp)) + require.Equal(t, 4, vec0.Length()) + vals0 := vector.MustFixedColWithTypeCheck[int64](vec0) + require.Equal(t, int64(100), vals0[0]) + require.Equal(t, int64(300), vals0[2]) + require.False(t, vec0.IsNull(0)) + require.True(t, vec0.IsNull(1)) + require.False(t, vec0.IsNull(2)) + require.True(t, vec0.IsNull(3)) + + // col1: varchar with NULLs at ord 2 and 3 (first NULL comes AFTER two non-nulls, + // exercising the cbAppendNull backfill of the prior all-non-null Nulls slice). + k1 := &vectorindex.ColumnBuffer{Type: types.T_varchar} + for ord := int64(0); ord < 4; ord++ { + require.NoError(t, seg.appendIncludeTo(k1, 1, ord)) + } + require.Equal(t, []bool{false, false, true, true}, k1.Nulls) + vec1 := vector.NewVec(types.T_varchar.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(k1, vec1, mp)) + require.Equal(t, "active", vec1.GetStringAt(0)) + require.Equal(t, "b", vec1.GetStringAt(1)) + require.False(t, vec1.IsNull(0)) + require.False(t, vec1.IsNull(1)) + require.True(t, vec1.IsNull(2)) + require.True(t, vec1.IsNull(3)) + } + + t.Run("build-side", func(t *testing.T) { run(t, boxfreeSeg()) }) + + t.Run("loaded-side", func(t *testing.T) { + data, err := boxfreeSeg().encodeDocmap() + require.NoError(t, err) + loaded := &Segment{} + require.NoError(t, loaded.decodeDocmap(data)) + run(t, loaded) + }) +} + +// TestColumnBufferAllNonNull: a column with NO NULLs never materializes k.Nulls (stays nil, +// so AppendColumnBuffer's nul(i) is always false — the pk path is byte-for-byte unchanged). +func TestColumnBufferAllNonNull(t *testing.T) { + mp := mpool.MustNewZero() + seg := boxfreeSeg() + k := &vectorindex.ColumnBuffer{Type: types.T_varchar} + // ords 0,1 have non-null varchar; append only those. + require.NoError(t, seg.appendIncludeTo(k, 1, 0)) + require.NoError(t, seg.appendIncludeTo(k, 1, 1)) + require.Nil(t, k.Nulls, "no NULL seen -> Nulls stays nil") + vec := vector.NewVec(types.T_varchar.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(k, vec, mp)) + require.False(t, vec.IsNull(0)) + require.False(t, vec.IsNull(1)) +} + +// TestIncludeRawAtBounds pins the belt-and-suspenders bounds (F3): a malformed / truncated +// loaded segment makes includeRawAt (and thus includeVal / appendIncludeTo) return an error +// instead of panicking on an out-of-range slice. +func TestIncludeRawAtBounds(t *testing.T) { + data, err := boxfreeSeg().encodeDocmap() + require.NoError(t, err) + + // Fixed column (col0 int64): truncate includeRaw so the stride-computed row overruns. + trunc := &Segment{} + require.NoError(t, trunc.decodeDocmap(data)) + trunc.includeRaw = trunc.includeRaw[:2] + _, _, err = trunc.includeRawAt(3, 0) + require.Error(t, err) + _, _, err = trunc.includeVal(3, 0) + require.Error(t, err) + require.Error(t, trunc.appendIncludeTo(&vectorindex.ColumnBuffer{Type: types.T_int64}, 0, 3)) + + // Varlena column (col1 varchar): corrupt the offset table so it points past includeRaw. + badoff := &Segment{} + require.NoError(t, badoff.decodeDocmap(data)) + for i := range badoff.includeVarOffsets { + badoff.includeVarOffsets[i] = int32(len(badoff.includeRaw) + 1000) + } + _, _, err = badoff.includeRawAt(0, 1) + require.Error(t, err) + require.Error(t, badoff.appendIncludeTo(&vectorindex.ColumnBuffer{Type: types.T_varchar}, 1, 0)) + + // Out-of-range column / negative ord are rejected, not panicked. + good := &Segment{} + require.NoError(t, good.decodeDocmap(data)) + _, _, err = good.includeRawAt(0, 99) + require.Error(t, err) + _, _, err = good.includeRawAt(-1, 0) + require.Error(t, err) +} diff --git a/pkg/fulltext2/include_build_test.go b/pkg/fulltext2/include_build_test.go new file mode 100644 index 0000000000000..7b541841db414 --- /dev/null +++ b/pkg/fulltext2/include_build_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +var incTypesTest = []int32{int32(types.T_int64), int32(types.T_varchar)} + +// TestCdcIncludeRoundTrip: a CDC batch with an INCLUDE schema encodes+decodes its per-event +// include values (typed, incl. NULL) — the wire path from the sink into the tail builder. +func TestCdcIncludeRoundTrip(t *testing.T) { + c := NewCdc(int32(types.T_int64)) + c.IncludeTypes = incTypesTest + c.Insert(int64(1), "alpha beta", []any{int64(100), []byte("active")}) + c.Upsert(int64(2), "gamma", []any{int64(200), nil}) // NULL varchar + c.Delete(int64(3)) // delete carries no include + + buf, err := c.Encode() + require.NoError(t, err) + got, err := DecodeCdc(buf) + require.NoError(t, err) + + require.Equal(t, incTypesTest, got.IncludeTypes) + require.Len(t, got.Events, 3) + require.Equal(t, []any{int64(100), []byte("active")}, got.Events[0].Include) + require.Equal(t, int64(200), got.Events[1].Include[0]) + require.Nil(t, got.Events[1].Include[1]) // NULL + require.Equal(t, cdcDelete, got.Events[2].Op) +} + +// TestBuilderIncludeSetDoc: a Builder with WithIncludeTypes carries per-doc include values +// (via SetDoc and SetInclude) into the finished segment, decodable by includeVal. +func TestBuilderIncludeSetDoc(t *testing.T) { + b := NewBuilder("b", int32(types.T_int64), WithIncludeTypes(incTypesTest)) + // SetDoc (the CDC upsert primitive) sets terms + include together. + b.SetDoc(int64(1), []WordPos{{Word: "alpha", Pos: 0}}, []any{int64(10), []byte("x")}) + // Add + SetInclude (the base-build primitive). + require.NoError(t, b.Add("beta", 0, int64(2))) + b.SetInclude(int64(2), []any{int64(20), []byte("y")}) + + seg, err := b.Finish() + require.NoError(t, err) + + // Round-trip through serialize so we exercise the loaded-side decode path. + data, err := seg.encodeDocmap() + require.NoError(t, err) + loaded := &Segment{} + require.NoError(t, loaded.decodeDocmap(data)) + require.Equal(t, 2, loaded.nIncludeCols()) + + // ords are pk-assignment order: pk1 -> ord0, pk2 -> ord1. + v0, null0, _ := loaded.includeVal(0, 0) + require.False(t, null0) + require.Equal(t, int64(10), v0) + v1s, _, _ := loaded.includeVal(1, 1) + require.Equal(t, []byte("y"), v1s) +} + +// TestReconstructPreservesInclude: ReconstructLiveDocs (the MERGE input) yields each live +// doc's INCLUDE values, so a compaction folds them into the rebuilt base instead of dropping +// them — the key MERGE correctness invariant for covering/prefilter. +func TestReconstructPreservesInclude(t *testing.T) { + docs := []TokenizedDoc{ + {Pk: int64(1), Terms: []string{"alpha"}, Positions: []int32{0}, Include: []any{int64(10), []byte("a")}}, + {Pk: int64(2), Terms: []string{"beta"}, Positions: []int32{0}, Include: []any{int64(20), nil}}, + } + seg, err := BuildSegmentFromTokenized("s", int32(types.T_int64), docs, WithIncludeTypes(incTypesTest)) + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + got := map[int64][]any{} + for d, derr := range idx.ReconstructLiveDocs(false) { + require.NoError(t, derr) + require.Len(t, d.Include, 2) + got[d.Pk.(int64)] = d.Include + } + require.Equal(t, int64(10), got[1][0]) + require.Equal(t, []byte("a"), got[1][1]) + require.Equal(t, int64(20), got[2][0]) + require.Nil(t, got[2][1]) // NULL preserved +} diff --git a/pkg/fulltext2/include_coverage_test.go b/pkg/fulltext2/include_coverage_test.go new file mode 100644 index 0000000000000..b5cf9fc2ace8c --- /dev/null +++ b/pkg/fulltext2/include_coverage_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// TestFillInclude pins the box-free covered side of SearchInto: fillInclude builds one +// nullable ColumnBuffer per FULL index INCLUDE column (segment order) from each result's +// positional Include values, into out.Include — so the TVF emits the projected INCLUDE +// columns (mapped by segment position) with no base-table JOIN, no map[string][]any, and no +// per-row reflection. NULLs round-trip to SQL NULL, and out is reused across calls. +func TestFillInclude(t *testing.T) { + mp := mpool.MustNewZero() + idx := incIdx(t) // include cols: [0]=status(varchar), [1]=prio(int64) + s := &Fulltext2Search{idx: idx, cfg: TableConfig{IncludeColumns: []string{"status", "prio"}}} + results := []Result{ + {Pk: int64(1), Score: 1, Include: []any{[]byte("active"), int64(10)}}, + {Pk: int64(2), Score: 1, Include: []any{nil, int64(20)}}, // NULL status + } + + out := &vectorindex.SearchOutput{} + // The requested set only gates the no-op; buffers are built in FULL include order. + rt := vectorindex.RuntimeConfig{RequestedIncludeColumns: []string{"prio", "status"}} + require.NoError(t, s.fillInclude(rt, results, out)) + require.Len(t, out.Include, 2) + + // Col 0 = status (varchar): ["active", NULL]. + statusVec := vector.NewVec(types.T_varchar.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(out.Include[0], statusVec, mp)) + require.Equal(t, "active", statusVec.GetStringAt(0)) + require.False(t, statusVec.IsNull(0)) + require.True(t, statusVec.IsNull(1)) // pk2 status is NULL + + // Col 1 = prio (int64): [10, 20], no NULLs. + prioVec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(out.Include[1], prioVec, mp)) + require.Equal(t, []int64{10, 20}, vector.MustFixedColWithTypeCheck[int64](prioVec)) + require.False(t, prioVec.IsNull(0)) + require.False(t, prioVec.IsNull(1)) + + // AppendColumnBufferRange pages a subset: row [1,2) of prio = just 20. + pageVec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vectorindex.AppendColumnBufferRange(out.Include[1], pageVec, 1, 1, mp)) + require.Equal(t, []int64{20}, vector.MustFixedColWithTypeCheck[int64](pageVec)) + + // Reuse: a second fill on the SAME out Resets its buffers (no stale rows appended). + require.NoError(t, s.fillInclude(rt, results[:1], out)) + require.Equal(t, 1, out.Include[1].N) + + // No requested columns → out.Include emptied (no panic). + require.NoError(t, s.fillInclude(vectorindex.RuntimeConfig{}, results, out)) + require.Empty(t, out.Include) +} diff --git a/pkg/fulltext2/include_predicate.go b/pkg/fulltext2/include_predicate.go new file mode 100644 index 0000000000000..b3b7401b57f0b --- /dev/null +++ b/pkg/fulltext2/include_predicate.go @@ -0,0 +1,457 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "encoding/json" + "strconv" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// includeFilterPred is the pushed-down predicate wire form — the SAME JSON shape the vector +// plugins use (pkg/sql/plan filterJSONPred): identical field names (col/op/val/lo/hi/vals), +// the same symbolic op strings (=,!=,<,<=,>,>=,between,in,is_null,is_not_null), the same +// bare-array serialization, and the same pk sentinel col (-1). One shape means a single +// planner predicate-peeler can target both engines. fulltext2 extends it in two ways its Go +// evaluator supports (and the numeric C++ FilterStore cannot): Val/Vals may be JSON STRINGS +// (varchar INCLUDE/pk columns store the actual value), and a "prefix" op (LIKE 'x%'). The +// predicates form a conjunction (WHERE a AND b): a row is admitted iff ALL pass. +type includeFilterPred struct { + Col int `json:"col"` + Op string `json:"op"` + Val any `json:"val,omitempty"` // scalar comparison (=,!=,<,<=,>,>=,prefix) + Lo any `json:"lo,omitempty"` // between + Hi any `json:"hi,omitempty"` // between + Vals []any `json:"vals,omitempty"` // in +} + +// IncludePredPkCol is the "col" sentinel for a predicate on the PRIMARY KEY (doc_id) — +// identical to the vector plugins' pkHostIdSentinelCol (-1). The pk is stored per-doc +// (seg.pk) and typed by the index's PkType, so a pk predicate is evaluated inline exactly +// like an INCLUDE column: an EXACT, JOIN-free alternative to the docfilter second-scan +// pushdown for the comparable pk types (integer family + varchar/char). +const IncludePredPkCol = -1 + +type includeCmpKind uint8 + +const ( + incEq includeCmpKind = iota + incNe + incLt + incLe + incGt + incGe + incBetween + incIn + incPrefix + incIsNull + incIsNotNull +) + +// includeOpKinds maps the wire op strings (matching the vector plugins' symbolic ops, plus +// fulltext2's "prefix") to compiled kinds. +var includeOpKinds = map[string]includeCmpKind{ + "=": incEq, "!=": incNe, "<": incLt, "<=": incLe, ">": incGt, ">=": incGe, + "between": incBetween, "in": incIn, + "is_null": incIsNull, "is_not_null": incIsNotNull, + "prefix": incPrefix, +} + +// compiledIncludePred is a parsed, type-bound predicate: operands are decoded to the +// column's comparable form (int64 for the integer family, []byte for varchar/char) once at +// compile, so per-candidate evaluation is allocation-free. It is segment-independent (the +// INCLUDE column types are the same across an index's segments); includePredMembership binds +// it to a segment's includeVal. +type compiledIncludePred struct { + col int + kind includeCmpKind + isStr bool + isUnsigned bool // uint64/bit column: compare as uint64 (int64 wraps values above MaxInt64) + ints []int64 // signed integer-column operands (all int types + uint8/16/32, which fit int64) + uints []uint64 // unsigned-64 column operands + strs [][]byte // varchar/char-column operands (also the prefix) +} + +// compileIncludePredicates parses the JSON spec and type-binds each predicate against the +// index's INCLUDE column types (from a loaded segment) — or against pkType for a predicate +// on the primary key (Col == IncludePredPkCol). Returns nil for an empty spec. +func compileIncludePredicates(specJSON []byte, includeTypes []int32, pkType int32) ([]compiledIncludePred, error) { + if len(specJSON) == 0 { + return nil, nil + } + // UseNumber so a JSON integer decodes to json.Number (exact int64), not float64 (which + // loses precision past 2^53 — real for large pks). + dec := json.NewDecoder(bytes.NewReader(specJSON)) + dec.UseNumber() + var preds []includeFilterPred + if err := dec.Decode(&preds); err != nil { + return nil, err + } + out := make([]compiledIncludePred, 0, len(preds)) + for _, p := range preds { + kind, ok := includeOpKinds[p.Op] + if !ok { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: unknown op %q", p.Op) + } + // Resolve the operand type: the pk (Col == IncludePredPkCol) uses pkType; an INCLUDE + // column uses includeTypes[Col]. + var colType int32 + if p.Col == IncludePredPkCol { + colType = pkType + } else if p.Col >= 0 && p.Col < len(includeTypes) { + colType = includeTypes[p.Col] + } else { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: col %d out of range", p.Col) + } + if !isComparableIncludeType(colType) { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: col %d type not comparable", p.Col) + } + // uint64/bit columns store values that overflow int64 (> MaxInt64), so they need an + // unsigned comparison path — int64 casting would wrap a large value to negative and + // silently exclude a matching row. All other integer types (int8..64, uint8..32) fit + // in int64. isStr takes precedence (varchar/char). + unsigned := colType == int32(types.T_uint64) || colType == int32(types.T_bit) + cp := compiledIncludePred{col: p.Col, kind: kind, isStr: isVarlenaIncludeType(colType), isUnsigned: unsigned} + // Gather the op's operands from the shape-appropriate field(s). + var operands []any + switch kind { + case incIsNull, incIsNotNull: + // no operands + case incBetween: + operands = []any{p.Lo, p.Hi} + case incIn: + operands = p.Vals + default: + operands = []any{p.Val} + } + for _, o := range operands { + switch { + case cp.isStr: + s, ok := includeOperandString(o) + if !ok { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: col %d expects a string operand", p.Col) + } + cp.strs = append(cp.strs, []byte(s)) + case cp.isUnsigned: + u, ok := includeOperandUint64(o) + if !ok { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: col %d expects an unsigned-integer operand", p.Col) + } + cp.uints = append(cp.uints, u) + default: + n, ok := includeOperandInt64(o) + if !ok { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2 include predicate: col %d expects an integer operand", p.Col) + } + cp.ints = append(cp.ints, n) + } + } + out = append(out, cp) + } + return out, nil +} + +// includeOperandString extracts a string operand from a decoded JSON value. +func includeOperandString(v any) (string, bool) { + s, ok := v.(string) + return s, ok +} + +// includeOperandInt64 extracts an int64 operand from a decoded JSON value (json.Number with +// UseNumber, or float64/string defensively). +func includeOperandInt64(v any) (int64, bool) { + switch x := v.(type) { + case json.Number: + if n, err := x.Int64(); err == nil { + return n, true + } + if f, err := x.Float64(); err == nil { // e.g. "1.0" + return int64(f), true + } + return 0, false + case float64: + return int64(x), true + case string: + n, err := strconv.ParseInt(x, 10, 64) + return n, err == nil + default: + return 0, false + } +} + +// isVarlenaIncludeType reports whether an INCLUDE column type is compared as bytes (varchar/ +// char) rather than as an integer. +func isVarlenaIncludeType(t int32) bool { + switch types.T(t) { + case types.T_varchar, types.T_char: + return true + default: + return false + } +} + +// includeOperandUint64 extracts a uint64 operand from a decoded JSON value, for uint64/bit +// columns whose values can exceed MaxInt64 (where includeOperandInt64 would fail or saturate). +func includeOperandUint64(v any) (uint64, bool) { + switch x := v.(type) { + case json.Number: + if u, err := strconv.ParseUint(x.String(), 10, 64); err == nil { + return u, true + } + if f, err := x.Float64(); err == nil && f >= 0 { // e.g. "1.0" + return uint64(f), true + } + return 0, false + case float64: + if x < 0 { + return 0, false + } + return uint64(x), true + case uint64: + return x, true + case int64: + if x < 0 { + return 0, false + } + return uint64(x), true + case string: + u, err := strconv.ParseUint(x, 10, 64) + return u, err == nil + default: + return 0, false + } +} + +// includeUint64 coerces a stored unsigned INCLUDE/pk value to uint64 (decodePk yields uint64 +// for T_uint64/T_bit; the smaller unsigned/signed types are widened losslessly). +func includeUint64(v any) (uint64, bool) { + switch x := v.(type) { + case uint64: + return x, true + case uint32: + return uint64(x), true + case uint16: + return uint64(x), true + case uint8: + return uint64(x), true + case int64: + if x < 0 { + return 0, false + } + return uint64(x), true + case int32: + if x < 0 { + return 0, false + } + return uint64(x), true + default: + return 0, false + } +} + +// isComparableIncludeType reports whether the inline evaluator can compare a value of this +// type — the integer family (normalized to int64) or varchar/char (byte-compare). A pk of an +// unsupported type (uuid/decimal/...) must fall back to the docfilter pushdown, so the +// planner never emits an inline pk predicate on it. +func isComparableIncludeType(t int32) bool { + if isVarlenaIncludeType(t) { + return true + } + switch types.T(t) { + case types.T_int8, types.T_int16, types.T_int32, types.T_int64, + types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64, + types.T_bit: // BIT is uint64-backed; compiledIncludePred handles it on the unsigned path + return true + default: + return false + } +} + +// test evaluates the predicate against one candidate's stored INCLUDE value with SQL +// 3-valued logic: a value comparison against NULL is UNKNOWN → the row is NOT admitted; +// only isnull/isnotnull inspect the NULL flag directly. +func (p *compiledIncludePred) test(v any, isNull bool) bool { + switch p.kind { + case incIsNull: + return isNull + case incIsNotNull: + return !isNull + } + if isNull { + return false // NULL vs value ⇒ UNKNOWN ⇒ excluded + } + if p.isStr { + b := includeBytes(v) + switch p.kind { + case incEq: + return bytes.Equal(b, p.strs[0]) + case incNe: + return !bytes.Equal(b, p.strs[0]) + case incLt: + return bytes.Compare(b, p.strs[0]) < 0 + case incLe: + return bytes.Compare(b, p.strs[0]) <= 0 + case incGt: + return bytes.Compare(b, p.strs[0]) > 0 + case incGe: + return bytes.Compare(b, p.strs[0]) >= 0 + case incBetween: + return bytes.Compare(b, p.strs[0]) >= 0 && bytes.Compare(b, p.strs[1]) <= 0 + case incPrefix: + return bytes.HasPrefix(b, p.strs[0]) + case incIn: + for _, s := range p.strs { + if bytes.Equal(b, s) { + return true + } + } + return false + } + return false + } + if p.isUnsigned { + u, ok := includeUint64(v) + if !ok { + return false + } + switch p.kind { + case incEq: + return u == p.uints[0] + case incNe: + return u != p.uints[0] + case incLt: + return u < p.uints[0] + case incLe: + return u <= p.uints[0] + case incGt: + return u > p.uints[0] + case incGe: + return u >= p.uints[0] + case incBetween: + return u >= p.uints[0] && u <= p.uints[1] + case incIn: + for _, x := range p.uints { + if u == x { + return true + } + } + return false + case incPrefix: + return false // prefix is string-only + } + return false + } + n, ok := includeInt64(v) + if !ok { + return false + } + switch p.kind { + case incEq: + return n == p.ints[0] + case incNe: + return n != p.ints[0] + case incLt: + return n < p.ints[0] + case incLe: + return n <= p.ints[0] + case incGt: + return n > p.ints[0] + case incGe: + return n >= p.ints[0] + case incBetween: + return n >= p.ints[0] && n <= p.ints[1] + case incIn: + for _, x := range p.ints { + if n == x { + return true + } + } + return false + case incPrefix: + return false // prefix is a string-only op; the planner never emits it on an int column + } + return false +} + +// includeBytes coerces a stored varchar/char INCLUDE value to bytes (decodePk yields []byte). +func includeBytes(v any) []byte { + switch x := v.(type) { + case []byte: + return x + case string: + return []byte(x) + default: + return nil + } +} + +// includeInt64 normalizes a stored integer INCLUDE value (any int/uint width, as decodePk +// returns it) to int64 for comparison. A uint64 above math.MaxInt64 wraps — a documented +// v1 limitation; such huge unsigned filter values are not expected. +func includeInt64(v any) (int64, bool) { + switch x := v.(type) { + case int64: + return x, true + case int32: + return int64(x), true + case int16: + return int64(x), true + case int8: + return int64(x), true + case uint64: + return int64(x), true + case uint32: + return int64(x), true + case uint16: + return int64(x), true + case uint8: + return int64(x), true + default: + return 0, false + } +} + +// includePredMembership admits an ord iff every compiled INCLUDE predicate passes against +// its stored value in seg — the in-index prefilter, ANDed into the search's allow-set +// (mkAllow) alongside the docfilter/liveness memberships. +type includePredMembership struct { + seg *Segment + preds []compiledIncludePred +} + +func (m *includePredMembership) Contains(ord int64) bool { + for i := range m.preds { + p := &m.preds[i] + var v any + var isNull bool + if p.col == IncludePredPkCol { + // The primary key is stored per-doc and is never NULL. + v = m.seg.pk(ord) + } else { + var err error + v, isNull, err = m.seg.includeVal(ord, p.col) + if err != nil { + return false // corrupt value ⇒ exclude (safe: a covering fallback JOIN re-checks) + } + } + if !p.test(v, isNull) { + return false + } + } + return true +} diff --git a/pkg/fulltext2/include_predicate_test.go b/pkg/fulltext2/include_predicate_test.go new file mode 100644 index 0000000000000..8c0460c64d6aa --- /dev/null +++ b/pkg/fulltext2/include_predicate_test.go @@ -0,0 +1,211 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "math" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// incIdx builds a 5-doc index where every doc contains the term "x" (so a disjunctive query +// "x" matches all) plus two INCLUDE columns: col0 status(varchar), col1 prio(int64). +func incIdx(t *testing.T) *Index { + docs := []TokenizedDoc{ + {Pk: int64(1), Terms: []string{"x"}, Positions: []int32{0}, Include: []any{[]byte("active"), int64(10)}}, + {Pk: int64(2), Terms: []string{"x"}, Positions: []int32{0}, Include: []any{[]byte("inactive"), int64(20)}}, + {Pk: int64(3), Terms: []string{"x"}, Positions: []int32{0}, Include: []any{[]byte("active"), int64(30)}}, + {Pk: int64(4), Terms: []string{"x"}, Positions: []int32{0}, Include: []any{nil, int64(40)}}, // NULL status + {Pk: int64(5), Terms: []string{"x"}, Positions: []int32{0}, Include: []any{[]byte("archived"), int64(5)}}, + } + incTypes := []int32{int32(types.T_varchar), int32(types.T_int64)} + seg, err := BuildSegmentFromTokenized("inc", int32(types.T_int64), docs, WithIncludeTypes(incTypes)) + require.NoError(t, err) + return NewIndex([]*Segment{seg}, nil) +} + +// runPrefilter compiles specJSON and runs a disjunctive "x" search with the resulting +// INCLUDE prefilter, returning the matched pk set. +func runPrefilter(t *testing.T, idx *Index, specJSON string) []int64 { + preds, err := compileIncludePredicates([]byte(specJSON), idx.includeTypes(), idx.pkType()) + require.NoError(t, err) + res, err := idx.SearchQuery([]byte("x"), true, ParserDefault, BM25, 100, &prefilter{include: preds}) + require.NoError(t, err) + ids := resultIDs(res) + out := make([]int64, len(ids)) + for i, v := range ids { + out[i] = v.(int64) + } + return out +} + +// TestIncludePrefilterOps exercises the in-index INCLUDE prefilter end-to-end (compile → +// membership → search) across the op set, including string prefix, ranges, IN, and 3-valued +// NULL logic. status: 1=active 2=inactive 3=active 4=NULL 5=archived; prio: 1=10..5=5. +func TestIncludePrefilterOps(t *testing.T) { + idx := incIdx(t) + + require.ElementsMatch(t, []int64{1, 3}, runPrefilter(t, idx, `[{"col":0,"op":"=","val":"active"}]`)) + require.ElementsMatch(t, []int64{1, 3, 5}, runPrefilter(t, idx, `[{"col":0,"op":"prefix","val":"a"}]`)) // active/archived + require.ElementsMatch(t, []int64{1, 3, 5}, runPrefilter(t, idx, `[{"col":0,"op":"in","vals":["active","archived"]}]`)) + require.ElementsMatch(t, []int64{2, 5}, runPrefilter(t, idx, `[{"col":0,"op":"!=","val":"active"}]`)) // inactive/archived (NULL excluded) + require.ElementsMatch(t, []int64{4}, runPrefilter(t, idx, `[{"col":0,"op":"is_null"}]`)) + require.ElementsMatch(t, []int64{1, 2, 3, 5}, runPrefilter(t, idx, `[{"col":0,"op":"is_not_null"}]`)) + + // int column: range / comparisons (bare JSON numbers). + require.ElementsMatch(t, []int64{1, 2, 3}, runPrefilter(t, idx, `[{"col":1,"op":"between","lo":10,"hi":30}]`)) + require.ElementsMatch(t, []int64{4}, runPrefilter(t, idx, `[{"col":1,"op":">","val":30}]`)) + require.Empty(t, runPrefilter(t, idx, `[{"col":1,"op":">","val":100}]`)) + + // conjunction: status=active AND prio>=30 → only pk3. + require.ElementsMatch(t, []int64{3}, runPrefilter(t, idx, + `[{"col":0,"op":"=","val":"active"},{"col":1,"op":">=","val":30}]`)) + + // empty array → all 5. + require.Len(t, runPrefilter(t, idx, `[]`), 5) + + // PRIMARY KEY predicates (col -1, the pk sentinel) — evaluated inline against the stored + // pk, no docfilter second-scan. + require.ElementsMatch(t, []int64{2, 3, 4}, runPrefilter(t, idx, `[{"col":-1,"op":"between","lo":2,"hi":4}]`)) + require.ElementsMatch(t, []int64{1, 5}, runPrefilter(t, idx, `[{"col":-1,"op":"in","vals":[1,5]}]`)) + require.ElementsMatch(t, []int64{4, 5}, runPrefilter(t, idx, `[{"col":-1,"op":">","val":3}]`)) + // pk predicate ANDed with an INCLUDE predicate: id>2 AND status=active → {3}. + require.ElementsMatch(t, []int64{3}, runPrefilter(t, idx, + `[{"col":-1,"op":">","val":2},{"col":0,"op":"=","val":"active"}]`)) +} + +// TestIncludePredicateUnit covers the compiled-predicate evaluator directly, incl. the +// NULL 3-valued rule (a value comparison against NULL is UNKNOWN ⇒ not admitted). +func TestIncludePredicateUnit(t *testing.T) { + // varchar eq + p := compiledIncludePred{col: 0, kind: incEq, isStr: true, strs: [][]byte{[]byte("active")}} + require.True(t, p.test([]byte("active"), false)) + require.False(t, p.test([]byte("other"), false)) + require.False(t, p.test(nil, true)) // NULL vs value ⇒ excluded + + // int between + pb := compiledIncludePred{col: 1, kind: incBetween, ints: []int64{10, 20}} + require.True(t, pb.test(int64(15), false)) + require.True(t, pb.test(int32(10), false)) // narrower int type normalized + require.False(t, pb.test(int64(21), false)) + require.False(t, pb.test(nil, true)) + + // isnull / isnotnull + require.True(t, (&compiledIncludePred{kind: incIsNull}).test(nil, true)) + require.False(t, (&compiledIncludePred{kind: incIsNull}).test([]byte("x"), false)) + require.True(t, (&compiledIncludePred{kind: incIsNotNull}).test([]byte("x"), false)) +} + +// TestIncludePredicateUint64 pins the unsigned path: a uint64 column with values ABOVE +// MaxInt64 must compare as uint64. The old int64 path wrapped such a stored value to +// negative (and saturated the operand), silently dropping a matching row. +func TestIncludePredicateUint64(t *testing.T) { + const big = uint64(math.MaxInt64) + 100 // 9223372036854775907, > MaxInt64 + + // compile a pk predicate (col=-1) on a T_uint64 pk with a huge operand: unsigned path, + // operand parsed into uints (not wrapped/saturated). + preds, err := compileIncludePredicates( + []byte(fmt.Sprintf(`[{"col":-1,"op":"=","val":%d}]`, big)), nil, int32(types.T_uint64)) + require.NoError(t, err) + require.Len(t, preds, 1) + require.True(t, preds[0].isUnsigned) + require.Equal(t, []uint64{big}, preds[0].uints) + require.True(t, preds[0].test(big, false)) // exact match (int64 path would miss it) + require.False(t, preds[0].test(big-1, false)) + require.False(t, preds[0].test(uint64(1), false)) + + // '>' at the MaxInt64 boundary: big > MaxInt64 must be TRUE (int64 wrap would say false). + gt := compiledIncludePred{col: 0, kind: incGt, isUnsigned: true, uints: []uint64{uint64(math.MaxInt64)}} + require.True(t, gt.test(big, false)) + require.False(t, gt.test(uint64(5), false)) + + // IN including a huge value. + in := compiledIncludePred{col: 0, kind: incIn, isUnsigned: true, uints: []uint64{1, big}} + require.True(t, in.test(big, false)) + require.True(t, in.test(uint64(1), false)) + require.False(t, in.test(uint64(2), false)) +} + +// TestCompileIncludePredicatesErrors: bad op / out-of-range col / bad int literal are rejected. +func TestCompileIncludePredicatesErrors(t *testing.T) { + it := []int32{int32(types.T_varchar), int32(types.T_int64)} + pk := int32(types.T_int64) + _, err := compileIncludePredicates([]byte(`[{"col":0,"op":"bogus","val":"a"}]`), it, pk) + require.Error(t, err) + _, err = compileIncludePredicates([]byte(`[{"col":9,"op":"=","val":"a"}]`), it, pk) + require.Error(t, err) + _, err = compileIncludePredicates([]byte(`[{"col":1,"op":"=","val":"notanint"}]`), it, pk) + require.Error(t, err) + // a pk predicate on a non-comparable pk type (decimal128) is rejected (planner falls back). + _, err = compileIncludePredicates([]byte(`[{"col":-1,"op":"=","val":1}]`), it, int32(types.T_decimal128)) + require.Error(t, err) + // empty spec → nil, no error. + preds, err := compileIncludePredicates(nil, it, pk) + require.NoError(t, err) + require.Nil(t, preds) +} + +// TestBitPkComparable (B2): a BIT primary key is comparable — the planner peels a pk +// predicate on a BIT pk, so compileIncludePredicates must accept T_bit (via the unsigned +// path) rather than erroring "col -1 type not comparable" (which failed the whole query). +func TestBitPkComparable(t *testing.T) { + require.True(t, isComparableIncludeType(int32(types.T_bit))) + preds, err := compileIncludePredicates([]byte(`[{"col":-1,"op":"=","val":5}]`), nil, int32(types.T_bit)) + require.NoError(t, err) + require.Len(t, preds, 1) + require.True(t, preds[0].isUnsigned) + require.True(t, preds[0].test(uint64(5), false)) + require.False(t, preds[0].test(uint64(6), false)) +} + +// TestResultCarriesInclude pins the covering primitive: a top-k search Result carries the +// doc's INCLUDE column values (in index order, NULL-aware) across the WAND/boolean path +// (heapToResults) and the NL phrase path (boundedTopK) — so a covering query is served from +// the index. Uses the incIdx fixture (col0 status varchar, col1 prio int64). +func TestResultCarriesInclude(t *testing.T) { + idx := incIdx(t) + + byPk := func(res []Result) map[int64][]any { + m := map[int64][]any{} + for _, r := range res { + m[r.Pk.(int64)] = r.Include + } + return m + } + + // Boolean/WAND path (disjunctive "x"). + wand, err := idx.SearchQuery([]byte("x"), true, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + mw := byPk(wand) + require.Equal(t, []any{[]byte("active"), int64(10)}, mw[1]) + require.Equal(t, []any{nil, int64(40)}, mw[4]) // NULL status preserved + + // NL phrase path (boundedTopK). + nl, err := idx.SearchQuery([]byte("x"), false, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + mn := byPk(nl) + require.Equal(t, []any{[]byte("archived"), int64(5)}, mn[5]) + + // An index WITHOUT include columns → Result.Include is nil. + seg, err := BuildSegmentFromTokenized("noinc", int32(types.T_int64), + []TokenizedDoc{{Pk: int64(1), Terms: []string{"x"}, Positions: []int32{0}}}) + require.NoError(t, err) + res, err := NewIndex([]*Segment{seg}, nil).SearchQuery([]byte("x"), true, ParserDefault, BM25, 10, nil) + require.NoError(t, err) + require.Nil(t, res[0].Include) +} diff --git a/pkg/fulltext2/index.go b/pkg/fulltext2/index.go new file mode 100644 index 0000000000000..7cd6d79192650 --- /dev/null +++ b/pkg/fulltext2/index.go @@ -0,0 +1,673 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "iter" + "sort" + "strconv" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// Index is a queryable fulltext v2 index made of several loaded segments — a +// tag=0 base plus tag=1 CDC tail deltas (§4) — with per-pk liveness and a delete +// set. It resolves the two things a lone Segment cannot: +// +// - GLOBAL stats: N and avgDocLen span all segments (a term's idf and BM25's +// length norm must be global, not per-segment), computed once at construction. +// - Liveness: the same pk may land in several segments (UPDATE / reinsert / a +// stale base copy); only the highest-Recency copy is live, and a delete at +// recency d kills copies with recency < d. Dead copies never appear in +// results and never contribute to a phrase's document frequency. +type Index struct { + segments []*Segment + deletes map[any]int64 // normalizeKey(pk) -> recency at/after which older copies are dead + + globalN int64 + globalAvgDocLen float64 + // liveOrd[si] is a per-segment ord-indexed liveness bitmap (built once in resolve), + // so every liveness check — isLive (phrase), livenessMembership (boolean/stream), and + // ReconstructLiveDocs (compact) — is an O(1) bitmap index, allocation-free, with no + // per-candidate keyOf + map lookup. A fully-live segment keeps liveOrd[si]==nil ("all + // live" fast path), so an append-only index costs ZERO resident liveness heap. It is + // the SOLE resident liveness structure: the pk->loc map that resolve() builds to + // derive it is local and discarded (it was O(live docs) of Go heap — the dominant + // load-time floor). Mirrors bm25's ComputeLiveness []Membership. + liveOrd [][]bool +} + +type docLoc struct { + si int + ord int64 +} + +// NewIndex builds an index over the given loaded segments (each carrying its +// Recency) and delete set, computing global stats and the liveness map. deletes +// may be nil. +func NewIndex(segments []*Segment, deletes map[any]int64) *Index { + idx := &Index{segments: segments, deletes: deletes} + idx.resolve() + return idx +} + +// resolve computes the live copy of each pk (highest Recency, not delete-shadowed), +// then the global doc count + average document length; each segment's AvgDocLen is +// set to the global value (§4) so BM25 length-normalizes consistently. +func (idx *Index) resolve() { + type best struct { + rec int64 + loc docLoc + } + top := make(map[any]best) + for si, seg := range idx.segments { + nd := seg.numDocs() + for ord := 0; ord < nd; ord++ { + key := normalizeKey(seg.pk(int64(ord))) + cand := best{seg.Recency, docLoc{si, int64(ord)}} + if cur, ok := top[key]; !ok || cand.rec > cur.rec { + top[key] = cand + } + } + } + + // liveLoc (pk key -> live loc) is LOCAL: it drives the liveOrd bitmap + global stats + // below and is then discarded, so it never becomes part of the resident cache + // footprint (it was O(live docs) of Go-heap map — the biggest load-time allocation). + liveLoc := make(map[any]docLoc, len(top)) + var sumDocLen int64 + for key, b := range top { + if d, ok := idx.deletes[key]; ok && b.rec < d { + continue // shadowed by a later delete + } + liveLoc[key] = b.loc + sumDocLen += int64(idx.segments[b.loc.si].docLen[b.loc.ord]) + } + idx.globalN = int64(len(liveLoc)) + if idx.globalN > 0 { + idx.globalAvgDocLen = float64(sumDocLen) / float64(idx.globalN) + } + for _, seg := range idx.segments { + seg.AvgDocLen = idx.globalAvgDocLen + } + + // Per-segment liveness bitmap (mirrors bm25's ComputeLiveness): a fully-live + // segment keeps liveOrd[si]==nil (the common single-base / append-only case) so it + // costs no O(doc-count) allocation; otherwise mark each live ord true. Every query + // and compact liveness check then indexes this O(1) instead of keyOf+map per candidate. + idx.liveOrd = make([][]bool, len(idx.segments)) + liveCount := make([]int, len(idx.segments)) + for _, loc := range liveLoc { + liveCount[loc.si]++ + } + for si, seg := range idx.segments { + if liveCount[si] < seg.numDocs() { // some dead ord in this segment → need the bitmap + idx.liveOrd[si] = make([]bool, seg.numDocs()) + } + } + for _, loc := range liveLoc { + if b := idx.liveOrd[loc.si]; b != nil { + b[loc.ord] = true + } + } +} + +// includeTypes returns the index's INCLUDE column types (types.T per column), read from the +// first segment that carries them — all segments of an index share one INCLUDE schema. +// nil when the index has no INCLUDE columns. +func (idx *Index) includeTypes() []int32 { + for _, s := range idx.segments { + if len(s.includeTypes) > 0 { + return s.includeTypes + } + } + return nil +} + +// pkType returns the index's primary-key type (types.T), used to type-bind an inline pk +// predicate. All segments share the pk type; 0 when the index has no segments. +func (idx *Index) pkType() int32 { + if len(idx.segments) > 0 { + return idx.segments[0].PkType + } + return 0 +} + +// isLive reports whether (si, ord) is the live copy of its pk, via the per-segment +// liveness bitmap (nil => the segment is fully live). Same semantics as +// livenessMembership; no pk key or map lookup needed. +func (idx *Index) isLive(si int, ord int64) bool { + b := idx.liveOrd[si] + return b == nil || b[ord] +} + +// SearchPhrase runs an NL exact-phrase query across all segments and returns the +// global top-k (score desc; equal scores are order-unspecified). A single term is the +// degenerate one-term phrase. Scoring uses global N + global avgDocLen, and the +// phrase's document frequency is the number of LIVE documents that match (dead +// and delete-shadowed copies are excluded), so idf is exact. +func (idx *Index) SearchPhrase(slots []phraseSlot, algo ScoreAlgo, k int, filter *prefilter) []Result { + if k <= 0 || idx.globalN == 0 || len(slots) == 0 { + return nil + } + // A phrase can match at most globalN live docs, so cap k to it: a huge pushed LIMIT + // (clamped to MaxInt32 upstream) must never size the top-k beyond the live corpus. + if int64(k) > idx.globalN { + k = int(idx.globalN) + } + // Single pass, O(min(k, #matches)) memory (was O(matches): a matched map + a full results slice, + // which OOMs on a low-selectivity phrase that hits most of the corpus). A lone + // phrase has ONE idf² (idfSquared over its live df), and scoreTerm multiplies the + // per-doc factor by that constant, so idf² does not change the top-k order — rank on + // the idf²-free partial score in a bounded top-k, count the (filter-independent) live + // df alongside, and scale the k winners by idf² at the end. isLive is true for exactly + // one (si,ord) per pk, so counting live hits IS the distinct-live df (no dedup set). + tk := newBoundedTopK(k) + df := 0 + for si, seg := range idx.segments { + allow := mkAllow(seg, filter) // WHERE prefilter over this segment's ords (nil = none) + for _, h := range seg.matchPhrase(slots) { + if !idx.isLive(si, h.ord) { + continue + } + df++ // distinct live phrase match (filter-independent, matches the old dfSet) + if allowed(allow, h.ord) { + partial := seg.scoreTerm(algo, float64(h.tf), 1.0, h.ord, idx.globalAvgDocLen) + tk.push(seg.pk(h.ord), partial, seg.decodeInclude(h.ord)) + } + } + } + if tk.len() == 0 { + return nil + } + idf2 := idfSquared(idx.globalN, df) // df = live phrase-matched docs, filter-independent + return tk.resultsDescScaled(float32(idf2)) +} + +// boundedTopK keeps only the highest-scoring k (pk, score) pairs seen via push in O(k) +// memory — a binary min-heap by score, so the current k-th best sits at the root and is +// evicted in O(log k) when a better candidate arrives. It replaces "materialize every +// match, then top-k", which was O(matches). Ties at the k-th place are unspecified (as +// before — no arbitrary pk tie-break; see topKResults). +// +// It is deliberately keyed by the resolved `pk any` (NOT the segment `ord`), unlike the +// ord-keyed FastMaxHeap in wand.go: SearchPhrase accumulates across ALL segments into +// ONE heap, and each segment has its own ord space (ord 5 in seg0 != ord 5 in seg1), so +// only the globally-unique pk is a safe key here. It also defers the single idf² scale to +// resultsDescScaled. Those two properties are why it is not the FastMaxHeap used by the +// per-segment WAND paths — do not "unify" them; keying this by ord reintroduces cross- +// segment ord collisions. +type topKItem struct { + pk any + score float32 + include []any // INCLUDE values carried for a covering query (nil when no INCLUDE columns) +} + +type boundedTopK struct { + k int + h []topKItem +} + +// newBoundedTopK does NOT pre-allocate a cap-k backing array: push grows h lazily and +// never past k items, so the heap is bounded by min(k, #matches) — an absurd pushed +// LIMIT (e.g. 5e8) on a query matching a handful of docs costs a handful of slots, not +// a k-sized (multi-GB) allocation. +func newBoundedTopK(k int) *boundedTopK { return &boundedTopK{k: k} } + +func (b *boundedTopK) len() int { return len(b.h) } + +func (b *boundedTopK) push(pk any, score float32, include []any) { + if b.k <= 0 { + return + } + if len(b.h) < b.k { + b.h = append(b.h, topKItem{pk, score, include}) + for i := len(b.h) - 1; i > 0; { + p := (i - 1) / 2 + if b.h[p].score <= b.h[i].score { + break + } + b.h[p], b.h[i] = b.h[i], b.h[p] + i = p + } + return + } + if score <= b.h[0].score { + return // not better than the current k-th best + } + b.h[0] = topKItem{pk, score, include} + n := len(b.h) + for i := 0; ; { + l, r, s := 2*i+1, 2*i+2, i + if l < n && b.h[l].score < b.h[s].score { + s = l + } + if r < n && b.h[r].score < b.h[s].score { + s = r + } + if s == i { + break + } + b.h[s], b.h[i] = b.h[i], b.h[s] + i = s + } +} + +// resultsDescScaled returns the retained items as Results in score-descending order, +// each partial score multiplied by scale (the idf² deferred out of the ranking). +func (b *boundedTopK) resultsDescScaled(scale float32) []Result { + out := make([]Result, len(b.h)) + for i := range b.h { + out[i] = Result{Pk: b.h[i].pk, Score: b.h[i].score * scale, Include: b.h[i].include} + } + sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score }) + return out +} + +// topKResults returns the top-k results ordered by score descending. Equal scores are +// equally relevant, so their relative order is unspecified (SQL LIMIT over a tied ORDER +// BY key is itself unspecified) — we do NOT impose an arbitrary pk tie-break, which would +// force a sortKey string alloc for every matched doc (the alloc storm that dominated the +// CPU/GC profile of a query matching far more than k docs, e.g. a common single-term NL +// phrase or a boolean union). +// +// Selection is a bounded SoA FastMaxHeap keyed by result index with distance = -score: +// keeping the k smallest distances yields the k highest scores in O(n) with ZERO +// per-candidate allocation. Pop returns largest-distance (lowest-score) first, so +// filling the output back-to-front lands it in score-descending order. +func topKResults(results []Result, k int) []Result { + if k <= 0 || len(results) == 0 { + return nil + } + if k > len(results) { + k = len(results) + } + keysBuf := make([]int64, k) + distsBuf := make([]float32, k) + h := vectorindex.NewFastMaxHeap[float32, int64](k, keysBuf, distsBuf) + for i := range results { + h.Push(int64(i), -results[i].Score) + } + out := make([]Result, k) + for i := k - 1; i >= 0; i-- { + idx, _, ok := h.Pop() + if !ok { // fewer than k live entries (e.g. duplicate pushes were bounded out) + return out[i+1:] + } + out[i] = results[idx] + } + return out +} + +// SearchText tokenizes query with tok and runs an NL exact-phrase search across +// the index. +func (idx *Index) SearchText(query []byte, tok tokenizer.Tokenizer, algo ScoreAlgo, k int, filter *prefilter) ([]Result, error) { + return idx.SearchPhrase(tokenizePhraseSlots(tok, query), algo, k, filter), nil +} + +// globalStats carries the corpus-global scoring inputs (N, average doc length, and +// per-term document frequency) so a boolean query scores every segment on the SAME +// scale. Without it each segment used its LOCAL N/df, so appending a CDC tail shifted +// a doc's score and could drop a globally-top-k doc at the merge. df is summed across +// segments (like bm25's gdf) and memoized per query; it is built fresh per query and +// used sequentially across segments, so the cache needs no lock. +type globalStats struct { + n int64 + avgDocLen float64 + idx *Index + dfCache map[string]int + // phraseDfCache memoizes only the corpus-global phrase COUNT per phrase (an int), + // never the per-segment hit slices — see phraseDf. Retaining the slices would defeat + // the no-LIMIT streaming O(one segment) bound: a low-selectivity phrase over many + // capacity-sized segments would hold O(global matching corpus) before the first + // segment could finish emitting. + phraseDfCache map[string]int +} + +func (idx *Index) newGlobalStats() *globalStats { + return &globalStats{n: idx.globalN, avgDocLen: idx.globalAvgDocLen, idx: idx, + dfCache: make(map[string]int), phraseDfCache: make(map[string]int)} +} + +// phraseHits returns seg's matchPhrase hits WITHOUT retaining them across segments: the +// caller (evalClause) uses the slice transiently to score one segment and drops it, so +// only O(one segment) of phrase hits is live at a time. phraseDf recomputes-and-discards +// its own cross-segment scan for the same reason; the two intentionally do NOT share a +// cache (that sharing is what broke the streaming memory bound). The doubled matchPhrase +// work is the accepted cost of bounded memory on the no-LIMIT streaming path. +func (gs *globalStats) phraseHits(seg *Segment, slots []phraseSlot) []docTf { + return seg.matchPhrase(slots) +} + +// df returns term's corpus-global LIVE document frequency: only ords not superseded by a +// newer copy (recency) or a tombstone are counted. df feeds idf against the LIVE globalN, so +// counting dead PHYSICAL copies (an UPDATE leaves the old copy in a lower-recency segment +// until MERGE reclaims it) would inflate df past N and make a dirty base+tail rank a query +// DIFFERENTLY from the identical corpus after compaction. Enumerating the term's ords and +// testing isLive costs O(df) per query term, memoized so it is paid once — and WAND walks the +// same posting list anyway. +func (gs *globalStats) df(term string) int { + if d, ok := gs.dfCache[term]; ok { + return d + } + d := 0 + for si, seg := range gs.idx.segments { + if pl, ok := seg.lookup(term); ok { + for _, ord := range pl.materializeDocIDs() { + if gs.idx.isLive(si, ord) { + d++ + } + } + } + } + gs.dfCache[term] = d + return d +} + +// idfFor is the term's idf² under the global corpus stats, or the segment-local stats +// when gs is nil (a direct Segment query / single-segment test, where local == global). +func (gs *globalStats) idfFor(seg *Segment, term string, pl *termPostings) float64 { + if gs == nil { + return idfSquared(seg.N, pl.df()) + } + return idfSquared(gs.n, gs.df(term)) +} + +// phraseDf returns the corpus-global LIVE document frequency of a phrase (the number of LIVE +// docs matching the contiguous phrase, summed over segments), memoizing only the COUNT. Each +// segment's matchPhrase slice is counted and immediately discarded (count-and-discard), so the +// df scan holds at most one segment's hits at a time — never the whole corpus. Only live ords +// are counted (mirrors the live term df above and streamPhrase's pass 1), so a dirty tail does +// not inflate the phrase idf relative to the compacted corpus. +func (gs *globalStats) phraseDf(slots []phraseSlot) int { + key := slotsKey(slots) + if d, ok := gs.phraseDfCache[key]; ok { + return d + } + d := 0 + for si, seg := range gs.idx.segments { + for _, h := range seg.matchPhrase(slots) { // counted, then the slice is released this iteration + if gs.idx.isLive(si, h.ord) { + d++ + } + } + } + gs.phraseDfCache[key] = d + return d +} + +// phraseIdfFor is a phrase clause's idf² under the global corpus stats (global N + +// cross-segment phrase df), or the segment-local stats when gs is nil — so a phrase +// clause ranks consistently across a base + CDC tail, like the term/prefix clauses. +func (gs *globalStats) phraseIdfFor(seg *Segment, slots []phraseSlot, localHits int) float64 { + if gs == nil { + return idfSquared(seg.N, localHits) + } + return idfSquared(gs.n, gs.phraseDf(slots)) +} + +// avgdl is the global average doc length, or the segment's own when gs is nil. +func (gs *globalStats) avgdl(seg *Segment) float64 { + if gs == nil { + return seg.avgDocLenOrMean() + } + return gs.avgDocLen +} + +// SearchBoolean runs a parsed boolean-mode query across the index and returns the +// global top-k (score desc; equal scores are order-unspecified). Each segment is scored on the +// GLOBAL corpus stats (globalStats), and its walk admits only the LIVE copy of a pk +// (liveness ANDed into the WHERE prefilter), so a segment's top-k is k live docs on +// the global scale — the merge below is then the exact global top-k even across a +// base + CDC tail. All clause types (term, prefix, phrase, and the disjunctive WAND +// path) score on global stats — a phrase clause uses the cross-segment phrase df. +func (idx *Index) SearchBoolean(q BoolQuery, algo ScoreAlgo, k int, filter *prefilter) ([]Result, error) { + if k <= 0 || idx.globalN == 0 { + return nil, nil + } + gs := idx.newGlobalStats() + matched := make(map[any]Result) + for si, seg := range idx.segments { + allow := andAllow(mkAllow(seg, filter), &livenessMembership{idx: idx, si: si}) + res, err := seg.SearchBoolean(q, algo, k, allow, gs) + if err != nil { + return nil, err + } + for _, r := range res { + matched[normalizeKey(r.Pk)] = r // liveness inside the walk → one live copy per pk + } + } + if len(matched) == 0 { + return nil, nil + } + results := make([]Result, 0, len(matched)) + for _, r := range matched { + results = append(results, r) + } + return topKResults(results, k), nil +} + +// streamBoolean emits every LIVE, WHERE-admitted boolean match (pk, score) through sink +// WITHOUT a global top-K heap or an O(corpus) matched map — the streaming analogue of +// SearchBoolean for the no-LIMIT MUST / MUST-NOT / mixed-phrase path. Liveness is ANDed into +// each segment's prefilter exactly as SearchBoolean does, so a pk that survives in exactly +// one live segment is emitted once. Each segment's evalBoolean is O(segment N) (bounded by +// the segment capacity) and freed before the next, so peak Go heap is O(one segment), NOT +// O(globalN) — this is what makes the streamed boolean shape genuinely bounded rather than +// the old SearchQuery(globalN) fallback that materialized the whole live corpus. Ranking is +// the upstream ORDER BY score node's job (same contract as the other streaming paths). +func (idx *Index) streamBoolean(q BoolQuery, algo ScoreAlgo, filter *prefilter, sink *streamSink) error { + if idx.globalN == 0 { + return nil + } + gs := idx.newGlobalStats() + for si, seg := range idx.segments { + allow := andAllow(mkAllow(seg, filter), &livenessMembership{idx: idx, si: si}) + err := seg.evalBoolean(q, algo, allow, gs, func(ord int, score float32) { + sink.pushPk(seg, int64(ord), score) // pushPk no-ops once the sink has stopped + }) + if err != nil { + return err + } + // Bail before the NEXT segment on an emit error (the expensive per-segment work is + // bounded to one segment). Unlike streamWAND's `for !sink.stopped`, evalBoolean is + // NOT short-circuited mid-segment on abort: a boolean query must fully score every + // clause into the dense array before it can admit any candidate, so that work is + // already done by the time an emit can fail — the only remaining cost is the tail of + // the candidate forEach running as no-op pushes (a cheap bitset walk, no scoring). + // Threading a stop signal through the sink-agnostic evalBoolean to save that tail + // isn't worth complicating the helper it shares with searchBooleanFull. + if sink.err != nil { + return sink.err + } + } + sink.flush() + return sink.err +} + +// SearchBooleanText tokenizes+parses query in boolean mode with tok, then +// evaluates it across the index — the convenience entry mirroring +// MATCH(col) AGAINST('query' IN BOOLEAN MODE) with a fixed tokenizer. +func (idx *Index) SearchBooleanText(query []byte, tok tokenizer.Tokenizer, algo ScoreAlgo, k int) ([]Result, error) { + q, err := ParseBoolean(query, tok) + if err != nil { + return nil, err + } + return idx.SearchBoolean(q, algo, k, nil) // convenience entry: no WHERE prefilter +} + +// ReconstructLiveDocs yields each LIVE document as (pk, ordered terms) from the +// positional postings across all loaded segments — the input to a MERGE compaction +// that folds base + tail into a fresh dead-doc-free base WITHOUT re-tokenizing the +// source. Per-doc term order is recovered from token positions. +// +// Reconstruction is done ONE SEGMENT AT A TIME: a live doc's postings live entirely +// within its own (live) segment, so `buckets` only ever holds the CURRENT segment's +// live docs and is freed before the next — peak Go heap is O(one segment's postings) +// (≤ max_index_capacity), NOT O(the whole live corpus). This matters because MERGE +// (CompactSegments) is deliberately exempt from checkBaseLoadBudget, so an all-at-once +// reconstruction would OOM-kill the CN on exactly the operation meant to reclaim it. +// Docs are yielded in load-order across segments, pk-sorted WITHIN each segment (so the +// output is still deterministic; the fresh base's per-segment doc grouping differs but +// its contents and the resulting query answers do not). On a posting-decode error the +// iterator yields once with a non-nil error and stops. +func (idx *Index) ReconstructLiveDocs(positionFree bool) iter.Seq2[TokenizedDoc, error] { + return func(yield func(TokenizedDoc, error) bool) { + type posTerm struct { + pos int32 + term string + } + for si, seg := range idx.segments { + live := idx.liveOrd[si] + nd := seg.numDocs() + // Bucket ONLY this segment's live ords (keyed by ord, unique within the segment). + buckets := make(map[int64][]posTerm) + for ord := 0; ord < nd; ord++ { + if live == nil || live[ord] { + buckets[int64(ord)] = nil + } + } + if len(buckets) == 0 { + continue + } + err := seg.forEachPosting(func(term string, tp *termPostings) { + docs := tp.materializeDocIDs() // docIDs (block-compressed on load) + if positionFree { + // A position-free segment has no positions: reconstruct each doc's terms + // from tf — emit the term tf times with a synthetic ascending position. + // Order is irrelevant (the position-free rebuild drops positions again), + // but tf is preserved so BM25 ranking is unchanged. + tfs := tp.materializeTfs() + for i, ord := range docs { + if _, ok := buckets[ord]; !ok { + continue + } + for t := int32(0); t < int32(tfs[i]); t++ { + buckets[ord] = append(buckets[ord], posTerm{t, term}) + } + } + return + } + mat := tp.materializePositions() // decode this term's positions once + for i, ord := range docs { + if _, ok := buckets[ord]; !ok { + continue + } + for _, pos := range mat[i] { + buckets[ord] = append(buckets[ord], posTerm{pos, term}) + } + } + }) + if err != nil { + yield(TokenizedDoc{}, err) + return + } + + // Yield this segment's live docs pk-sorted (sortKey computed once each). + type keyed struct { + sk string + ord int64 + } + keys := make([]keyed, 0, len(buckets)) + for ord := range buckets { + keys = append(keys, keyed{sortKey(normalizeKey(seg.pk(ord))), ord}) + } + sort.Slice(keys, func(i, j int) bool { return keys[i].sk < keys[j].sk }) + for _, kd := range keys { + pl := buckets[kd.ord] + sort.SliceStable(pl, func(i, j int) bool { return pl[i].pos < pl[j].pos }) + terms := make([]string, len(pl)) + positions := make([]int32, len(pl)) + for i, pt := range pl { + terms[i] = pt.term + positions[i] = pt.pos // byte position, carried through MERGE + } + buckets[kd.ord] = nil // copied out — let GC reclaim it + // Carry the source doc's INCLUDE values through MERGE so the rebuilt base + // keeps them (a compaction that dropped them would break covering/prefilter). + var include []any + if nIncl := seg.nIncludeCols(); nIncl > 0 { + include = make([]any, nIncl) + for c := 0; c < nIncl; c++ { + v, isNull, ierr := seg.includeVal(kd.ord, c) + if ierr != nil { + yield(TokenizedDoc{}, ierr) + return + } + if !isNull { + include[c] = v + } + } + } + if !yield(TokenizedDoc{Pk: seg.pk(kd.ord), Terms: terms, Positions: positions, Include: include}, nil) { + return + } + } + // buckets goes out of scope → freed before the next segment. + } + } +} + +// NumDocs returns the number of live documents in the index. +func (idx *Index) NumDocs() int64 { return idx.globalN } + +// Free releases the off-heap posting buffers of every segment the index owns. +// Called on cache eviction (Fulltext2Search.Destroy) and by single-shot loaders +// (compact) once done. Build-side segments free nothing (no-op). +func (idx *Index) Free() { + if idx != nil { + freeSegs(idx.segments) + } +} + +// normalizeKey converts a pk to a comparable map key ([]byte -> string) and is the +// key for every pk-keyed map (liveness, deletes, dedup). Keying by the pk VALUE (not +// its String()) makes it INJECTIVE for every comparable pk type via Go == : a +// microsecond-precision DATETIME/TIME/TIMESTAMP (whose String() truncates to whole +// seconds), a raw uuid ([16]byte), a decimal128 struct — all key by their exact value +// and can never collide, so no distinct live document is dropped. Mirrors bm25's +// normalizeKey exactly (the complete port; a %v/String() key was lossy). +func normalizeKey(pk any) any { + if b, ok := pk.([]byte); ok { + return string(b) + } + return pk +} + +// sortKey is a deterministic INJECTIVE string projection of a pk, used to order +// equal-score results (and MERGE output) reproducibly — never as a map key. The +// temporal types are keyed by their underlying int64 because %v/String() truncates +// them to whole seconds, which would make two equal-score DATETIME(6) rows sharing a +// second sort nondeterministically at a LIMIT boundary; every other scalar pk has an +// injective %v (int/decimal exact, uuid canonical, date a distinct day). +func sortKey(pk any) string { + switch v := pk.(type) { + case []byte: + return string(v) + case string: + return v + case types.Datetime: + return strconv.FormatInt(int64(v), 10) + case types.Time: + return strconv.FormatInt(int64(v), 10) + case types.Timestamp: + return strconv.FormatInt(int64(v), 10) + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/pkg/fulltext2/index_test.go b/pkg/fulltext2/index_test.go new file mode 100644 index 0000000000000..cac1c7871c4ab --- /dev/null +++ b/pkg/fulltext2/index_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// TestIdfSquaredClamp pins the df-in-[1,n] clamp: df is summed over physical (dead+live) +// postings while n is live-only, so churn can push df past n. The clamp keeps idf>=0 and +// monotonic instead of squaring a negative log10(n/df) into a positive impact. +func TestIdfSquaredClamp(t *testing.T) { + // df < 1 clamps up to 1: idf = (log10(n))^2. + require.InDelta(t, math.Log10(10)*math.Log10(10), idfSquared(10, 0), 1e-9) + // df == n: term in every live doc -> idf 0. + require.Equal(t, 0.0, idfSquared(10, 10)) + // df > n (dirty base+tail churn): clamped to n -> idf 0, NOT a positive squared-neg-log. + require.Equal(t, 0.0, idfSquared(10, 25)) + // monotonically non-increasing in df across [1, n]. + prev := math.Inf(1) + for df := 1; df <= 10; df++ { + v := idfSquared(10, df) + require.LessOrEqualf(t, v, prev, "idfSquared not monotonic at df=%d", df) + prev = v + } +} + +func buildSeg(t *testing.T, id string, rec int64, docs []Doc) *Segment { + t.Helper() + s, err := BuildSegmentFromDocs(id, int32(types.T_int64), docs, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + s.Recency = rec + return s +} + +func iquery(t *testing.T, idx *Index, q string) []any { + t.Helper() + rs, err := idx.SearchText([]byte(q), tokenizer.NewSimpleTokenizer(), TfIdf, 10, nil) + require.NoError(t, err) + return pkslice(rs) +} + +// TestIndexLiveness: a base segment + a tail that UPDATES pk 2 (new text) and +// adds pk 4, with pk 3 deleted. Queries must see only live copies. +func TestIndexLiveness(t *testing.T) { + base := buildSeg(t, "base", 0, []Doc{ + {int64(1), []byte("alpha beta")}, + {int64(2), []byte("gamma")}, // superseded below + {int64(3), []byte("delta")}, // deleted below + }) + tail := buildSeg(t, "tail", 1, []Doc{ + {int64(2), []byte("omega")}, // new text for pk 2 + {int64(4), []byte("beta")}, + }) + idx := NewIndex([]*Segment{base, tail}, map[any]int64{normalizeKey(int64(3)): 1}) + + require.Equal(t, int64(3), idx.NumDocs()) // pk 1, 2(new), 4 live; pk 3 deleted + + // "beta" is in base:pk1 and tail:pk4 — both live (equal score → order unspecified). + require.ElementsMatch(t, []any{int64(1), int64(4)}, iquery(t, idx, "beta")) + // pk2's OLD text ("gamma") is superseded by the tail → not visible. + require.Empty(t, iquery(t, idx, "gamma")) + // pk2's NEW text ("omega") is the live copy. + require.Equal(t, []any{int64(2)}, iquery(t, idx, "omega")) + // pk3 is deleted. + require.Empty(t, iquery(t, idx, "delta")) + // base-only live doc. + require.Equal(t, []any{int64(1)}, iquery(t, idx, "alpha")) +} + +// TestIndexGlobalStats: df/idf and avgDocLen span segments. A term present once +// in each of two segments has df=2 against the global N. +func TestIndexGlobalStats(t *testing.T) { + s1 := buildSeg(t, "s1", 0, []Doc{{int64(1), []byte("cat")}, {int64(2), []byte("dog")}}) + s2 := buildSeg(t, "s2", 0, []Doc{{int64(3), []byte("cat")}, {int64(4), []byte("bird")}}) + idx := NewIndex([]*Segment{s1, s2}, nil) + + require.Equal(t, int64(4), idx.NumDocs()) + require.EqualValues(t, 1.0, idx.globalAvgDocLen) // every doc has 1 token + // "cat" is in pk1 and pk3 across the two segments (equal score → order unspecified). + require.ElementsMatch(t, []any{int64(1), int64(3)}, iquery(t, idx, "cat")) + // idf uses the GLOBAL df (=2) and N (=4), not a per-segment count. + rs, err := idx.SearchText([]byte("cat"), tokenizer.NewSimpleTokenizer(), TfIdf, 10, nil) + require.NoError(t, err) + want := idfSquared(4, 2) // tf=1 + require.InDelta(t, want, rs[0].Score, 1e-5) +} + +// TestIndexPhraseAcrossSegments: an exact phrase resolves within each segment; +// the live matches across segments merge. +func TestIndexPhraseAcrossSegments(t *testing.T) { + s1 := buildSeg(t, "s1", 0, []Doc{{int64(1), []byte("quick brown fox")}}) + s2 := buildSeg(t, "s2", 0, []Doc{{int64(2), []byte("the quick brown fox runs")}, {int64(3), []byte("brown quick")}}) + idx := NewIndex([]*Segment{s1, s2}, nil) + + // both match with equal TfIdf score → order unspecified. + require.ElementsMatch(t, []any{int64(1), int64(2)}, iquery(t, idx, "quick brown fox")) + require.Empty(t, iquery(t, idx, "fox brown")) // not contiguous anywhere +} diff --git a/pkg/fulltext2/jsonvalue_test.go b/pkg/fulltext2/jsonvalue_test.go new file mode 100644 index 0000000000000..fe50be3f7b9be --- /dev/null +++ b/pkg/fulltext2/jsonvalue_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestJSONValueExactMatch pins the json_value parser's contract to classic fulltext: +// each json leaf value is ONE whole atomic token (no ngram, punctuation/CJK kept), and +// a query operand matches a WHOLE value exactly — never a substring/bag. This is the +// divergence fixed in parity with classic (tokenize.go stores the whole value; sql.go +// queries `word = value`). +func TestJSONValueExactMatch(t *testing.T) { + docs := []Doc{ + {int64(1), []byte(`{"a":"redbluebrownyelloworange","b":"winter"}`)}, + {int64(2), []byte(`{"a":"red","b":"winter"}`)}, + {int64(3), []byte(`{"a":"中文學習教材"}`)}, + {int64(4), []byte(`{"a":"happybirthday.happy-birthday_happybirthday"}`)}, + } + + run := func(t *testing.T, idx *Index) { + bl := func(q string) []any { + rs, err := idx.SearchQuery([]byte(q), true, ParserJSONValue, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(rs) + } + nl := func(q string) []any { + rs, err := idx.SearchQuery([]byte(q), false, ParserJSONValue, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(rs) + } + + // Whole-value exact match — including punctuation and CJK values as ONE token. + require.ElementsMatch(t, []any{int64(1)}, bl("redbluebrownyelloworange")) + require.ElementsMatch(t, []any{int64(3)}, bl("中文學習教材")) + require.ElementsMatch(t, []any{int64(4)}, bl("happybirthday.happy-birthday_happybirthday")) + + // "red" is a WHOLE value only in doc 2 — NOT a substring hit on doc 1. + require.ElementsMatch(t, []any{int64(2)}, bl("red")) + require.ElementsMatch(t, []any{int64(2)}, nl("red")) + + // Distinct-value OR / multi-operand MUST. + require.ElementsMatch(t, []any{int64(1), int64(2)}, bl("winter")) // both docs carry value "winter" + require.ElementsMatch(t, []any{int64(2)}, bl("+red +winter")) // exact red AND exact winter + + // No bag-of-words / no ngram: a substring of a value never matches. + require.Empty(t, bl("bluebrown"), "substring of doc1's value is not a whole value") + require.Empty(t, bl("中文學習"), "substring of doc3's value is not a whole value") + require.Empty(t, bl("happybirthday"), "substring of doc4's value is not a whole value") + } + + seg, err := BuildSegmentFromDocsParser("jv", int32(types.T_int64), docs, ParserJSONValue) + require.NoError(t, err) + t.Run("build-side", func(t *testing.T) { run(t, NewIndex([]*Segment{seg}, nil)) }) + + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("jv", bytes.NewReader(blob)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + t.Run("loaded-side", func(t *testing.T) { run(t, NewIndex([]*Segment{loaded}, nil)) }) +} diff --git a/pkg/fulltext2/load_observer.go b/pkg/fulltext2/load_observer.go new file mode 100644 index 0000000000000..a7908ae13c4d1 --- /dev/null +++ b/pkg/fulltext2/load_observer.go @@ -0,0 +1,197 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/json" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/logutil" +) + +// LoadMissReason identifies the event which caused a cached FULLTEXT2 load. +// It is intentionally a small, bounded vocabulary: these values are suitable +// for diagnostics and must not contain query or primary-key data. +type LoadMissReason string + +const ( + LoadMissProcessStart LoadMissReason = "process_start" + LoadMissTTLExpired LoadMissReason = "ttl_expired" + LoadMissCDCFlush LoadMissReason = "cdc_flush" + LoadMissMerge LoadMissReason = "merge" + LoadMissRebuild LoadMissReason = "rebuild" + LoadMissGenerationChange LoadMissReason = "generation_changed" +) + +// LoadEvent is emitted once for one Fulltext2Search.Load attempt when the +// internal observer is installed or MO_FULLTEXT2_LOAD_TRACE=1 is set. The +// default process has no observer and does not sample any phase timestamps. +type LoadEvent struct { + Index string `json:"index"` + MissReason LoadMissReason `json:"miss_reason"` + BaseGeneration int64 `json:"base_generation"` + TailGeneration int64 `json:"tail_generation"` + BaseBytes int64 `json:"base_bytes"` + TailBytes int64 `json:"tail_bytes"` + InternalSQLTimeMicros int64 `json:"internal_sql_time_us"` + TempFileWriteMicros int64 `json:"temp_file_write_time_us"` + MmapMicros int64 `json:"mmap_time_us"` + ChecksumMicros int64 `json:"checksum_time_us"` + TotalLoadMicros int64 `json:"total_load_time_us"` + SingleflightWaiters int64 `json:"singleflight_waiters"` + LoadSuccess bool `json:"load_success"` + LoadError bool `json:"load_error"` + LoadCancel bool `json:"load_cancel"` +} + +type loadObserverHolder struct { + observe func(LoadEvent) +} + +var loadObserver atomic.Pointer[loadObserverHolder] + +// loadTraceEnabled is read once at process startup. The environment switch is +// diagnostic-only; it is not a SQL/session setting and remains off by default. +var loadTraceEnabled = os.Getenv("MO_FULLTEXT2_LOAD_TRACE") == "1" + +// setLoadObserver is intentionally package-private. Tests and an in-process +// diagnostic harness can install a bounded observer without adding a public +// FULLTEXT2 configuration surface. +func setLoadObserver(observe func(LoadEvent)) func() { + if observe == nil { + loadObserver.Store(nil) + return func() {} + } + loadObserver.Store(&loadObserverHolder{observe: observe}) + return func() { loadObserver.Store(nil) } +} + +func loadObservationEnabled() bool { + return loadObserver.Load() != nil || loadTraceEnabled +} + +func emitLoadEvent(event LoadEvent) { + if h := loadObserver.Load(); h != nil && h.observe != nil { + h.observe(event) + } + if !loadTraceEnabled { + return + } + b, err := json.Marshal(event) + if err == nil { + logutil.Debugf("[ft2-load] %s", b) + } +} + +type loadTrace struct { + mu sync.Mutex + event LoadEvent + start time.Time + phase phaseTimes + ended atomic.Bool +} + +type phaseTimes struct { + internalSQL time.Duration + tempWrite time.Duration + mmap time.Duration + checksum time.Duration +} + +func newLoadTrace(index string, reason LoadMissReason) *loadTrace { + if !loadObservationEnabled() { + return nil + } + return &loadTrace{event: LoadEvent{Index: index, MissReason: reason}, start: time.Now()} +} + +func (t *loadTrace) addInternalSQL(d time.Duration) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.phase.internalSQL += d + } +} + +func (t *loadTrace) addTempWrite(d time.Duration) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.phase.tempWrite += d + } +} + +func (t *loadTrace) addMmap(d time.Duration) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.phase.mmap += d + } +} + +func (t *loadTrace) addChecksum(d time.Duration) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.phase.checksum += d + } +} + +func (t *loadTrace) addBaseBytes(n int64) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.event.BaseBytes += n + } +} + +func (t *loadTrace) addTailBytes(n int64) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.event.TailBytes += n + } +} + +func (t *loadTrace) setGeneration(base, tail int64) { + if t != nil { + t.mu.Lock() + defer t.mu.Unlock() + t.event.BaseGeneration = base + t.event.TailGeneration = tail + } +} + +func (t *loadTrace) finish(err error, canceled bool, waiters int64) { + if t == nil || !t.ended.CompareAndSwap(false, true) { + return + } + t.mu.Lock() + t.event.InternalSQLTimeMicros = t.phase.internalSQL.Microseconds() + t.event.TempFileWriteMicros = t.phase.tempWrite.Microseconds() + t.event.MmapMicros = t.phase.mmap.Microseconds() + t.event.ChecksumMicros = t.phase.checksum.Microseconds() + t.event.TotalLoadMicros = time.Since(t.start).Microseconds() + t.event.SingleflightWaiters = waiters + t.event.LoadError = err != nil && !canceled + t.event.LoadCancel = canceled + t.event.LoadSuccess = err == nil && !canceled + event := t.event + t.mu.Unlock() + emitLoadEvent(event) +} diff --git a/pkg/fulltext2/load_observer_test.go b/pkg/fulltext2/load_observer_test.go new file mode 100644 index 0000000000000..a67518817a4b5 --- /dev/null +++ b/pkg/fulltext2/load_observer_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/stretchr/testify/require" +) + +func TestLoadTraceEmitsExactlyOnce(t *testing.T) { + var mu sync.Mutex + var events []LoadEvent + restore := setLoadObserver(func(event LoadEvent) { + mu.Lock() + events = append(events, event) + mu.Unlock() + }) + defer restore() + + trace := newLoadTrace("db.store", LoadMissCDCFlush) + trace.addInternalSQL(2 * time.Millisecond) + trace.addTempWrite(3 * time.Millisecond) + trace.addMmap(4 * time.Millisecond) + trace.addChecksum(5 * time.Millisecond) + trace.addBaseBytes(11) + trace.addTailBytes(7) + trace.setGeneration(12, 34) + trace.finish(nil, false, 2) + trace.finish(errors.New("late error"), false, 9) + + mu.Lock() + defer mu.Unlock() + require.Len(t, events, 1) + event := events[0] + require.Equal(t, "db.store", event.Index) + require.Equal(t, LoadMissCDCFlush, event.MissReason) + require.Equal(t, int64(12), event.BaseGeneration) + require.Equal(t, int64(34), event.TailGeneration) + require.Equal(t, int64(11), event.BaseBytes) + require.Equal(t, int64(7), event.TailBytes) + require.Equal(t, int64(2), event.SingleflightWaiters) + require.True(t, event.LoadSuccess) + require.False(t, event.LoadError) + require.False(t, event.LoadCancel) + require.GreaterOrEqual(t, event.TotalLoadMicros, int64(0)) +} + +func TestLoadTraceCancellationClassification(t *testing.T) { + var got LoadEvent + restore := setLoadObserver(func(event LoadEvent) { got = event }) + defer restore() + trace := newLoadTrace("store", LoadMissTTLExpired) + trace.finish(contextCanceledError{}, true, 0) + require.True(t, got.LoadCancel) + require.False(t, got.LoadError) + require.False(t, got.LoadSuccess) +} + +func TestLoadCancellationErrorRecognizesProductionErrors(t *testing.T) { + tests := []struct { + name string + err error + }{ + {name: "context canceled", err: context.Canceled}, + {name: "context deadline", err: context.DeadlineExceeded}, + {name: "query interrupted", err: moerr.NewQueryInterrupted(context.Background())}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.True(t, isLoadCancellationError(tt.err)) + }) + } + require.False(t, isLoadCancellationError(errors.New("storage failure"))) +} + +func TestFulltext2SearchFinishesPendingTraceAfterWaiterSample(t *testing.T) { + var got LoadEvent + restore := setLoadObserver(func(event LoadEvent) { got = event }) + defer restore() + s := NewFulltext2Search(TableConfig{DbName: "db", IndexTable: "store"}) + s.pendingTrace = newLoadTrace("store", LoadMissGenerationChange) + s.pendingLoadErr = nil + s.SetLoadWaiters(4) + s.FinishLoadObservation() + require.Equal(t, int64(4), got.SingleflightWaiters) + require.True(t, got.LoadSuccess) + s.FinishLoadObservation() +} + +type contextCanceledError struct{} + +func (contextCanceledError) Error() string { return "canceled" } diff --git a/pkg/fulltext2/load_reason.go b/pkg/fulltext2/load_reason.go new file mode 100644 index 0000000000000..77c97f1b84e15 --- /dev/null +++ b/pkg/fulltext2/load_reason.go @@ -0,0 +1,335 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "sync" + "sync/atomic" + "time" + + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" +) + +const ( + loadReasonTTL = 15 * time.Minute + loadReasonSize = 1024 + loadGenerationTTL = 15 * time.Minute + loadGenerationSize = 1024 +) + +type pendingLoadReason struct { + reason LoadMissReason + generation uint64 + at time.Time +} + +// loadGeneration identifies one cache-load attempt and the invalidation epoch +// it observed. The invalidation epoch is monotonic per index; the attempt is +// unique per index so an older replacement load cannot publish over a newer +// one even when both observe the same invalidation. +type loadGeneration struct { + index string + invalidation uint64 + attempt uint64 + owner uint64 +} + +type loadGenerationState struct { + invalidation uint64 + attempt uint64 + active uint32 + lastUsed time.Time +} + +var loadGenerations = struct { + sync.Mutex + m map[string]loadGenerationState +}{m: make(map[string]loadGenerationState)} + +var nextLoadOwner atomic.Uint64 + +// pendingLoadReasons bridges an invalidation event and the next cache miss. A +// failed load leaves the cause available for its retry; a successful load +// consumes only the entry observed by that load. It is bounded and expiring +// because an index may be invalidated without being queried again. The +// registry is touched only on invalidation/load paths, never by a warm search. +var pendingLoadReasons = struct { + sync.Mutex + m map[string]pendingLoadReason +}{m: make(map[string]pendingLoadReason)} + +var reusableLoadLifecycleOnce sync.Once + +func ensureReusableLoadLifecycle() { + reusableLoadLifecycleOnce.Do(func() { + veccache.RegisterLifecycleHook(func(shutdown bool) { + if shutdown { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + clearLoadGenerationRegistry() + return + } + now := time.Now() + loadedBasePool.evict(now) + loadedTailPool.evict(now) + }) + }) +} + +// loadReasonKey is the bounded registry key used by cache invalidation hooks. +// The database qualifier avoids conflating identically named hidden stores in +// different databases while keeping the key free of query or primary-key data. +func loadReasonKey(db, index string) string { + if db == "" { + return index + } + return db + "." + index +} + +func rememberLoadReason(index string, reason LoadMissReason) { + now := time.Now() + pendingLoadReasons.Lock() + defer pendingLoadReasons.Unlock() + loadGenerations.Lock() + defer loadGenerations.Unlock() + state := loadGenerations.m[index] + state.invalidation++ + state.lastUsed = now + loadGenerations.m[index] = state + pruneLoadGenerationsLocked(now) + rememberLoadReasonLocked(index, reason, state.invalidation) +} + +func pruneLoadGenerationsLocked(now time.Time) { + for index, state := range loadGenerations.m { + if state.active == 0 && !state.lastUsed.IsZero() && now.Sub(state.lastUsed) >= loadGenerationTTL { + delete(loadGenerations.m, index) + } + } + for len(loadGenerations.m) > loadGenerationSize { + var oldestIndex string + var oldest time.Time + for index, state := range loadGenerations.m { + if state.active != 0 { + continue + } + if oldestIndex == "" || state.lastUsed.Before(oldest) { + oldestIndex, oldest = index, state.lastUsed + } + } + if oldestIndex == "" { + return + } + delete(loadGenerations.m, oldestIndex) + } +} + +func rememberLoadReasonLocked(index string, reason LoadMissReason, generation uint64) { + if !loadObservationEnabled() || index == "" || reason == "" || reason == LoadMissReason("process_shutdown") { + return + } + now := time.Now() + for k, v := range pendingLoadReasons.m { + if now.Sub(v.at) >= loadReasonTTL { + delete(pendingLoadReasons.m, k) + } + } + if len(pendingLoadReasons.m) >= loadReasonSize { + var oldestKey string + var oldest time.Time + for k, v := range pendingLoadReasons.m { + if oldestKey == "" || v.at.Before(oldest) { + oldestKey, oldest = k, v.at + } + } + if oldestKey != "" { + delete(pendingLoadReasons.m, oldestKey) + } + } + pendingLoadReasons.m[index] = pendingLoadReason{reason: reason, generation: generation, at: now} +} + +func peekLoadReason(index string) (LoadMissReason, uint64) { + if !loadObservationEnabled() { + return "", 0 + } + now := time.Now() + pendingLoadReasons.Lock() + defer pendingLoadReasons.Unlock() + v, ok := pendingLoadReasons.m[index] + if !ok { + return "", 0 + } + if now.Sub(v.at) >= loadReasonTTL { + delete(pendingLoadReasons.m, index) + return "", 0 + } + return v.reason, v.generation +} + +func consumeLoadReason(index string, generation uint64) { + if index == "" || generation == 0 { + return + } + pendingLoadReasons.Lock() + defer pendingLoadReasons.Unlock() + if v, ok := pendingLoadReasons.m[index]; ok && v.generation == generation { + delete(pendingLoadReasons.m, index) + } +} + +func consumeLoadReasonIfCurrent(index string, reasonGeneration uint64, current loadGeneration) bool { + if index == "" || reasonGeneration == 0 { + return false + } + pendingLoadReasons.Lock() + defer pendingLoadReasons.Unlock() + loadGenerations.Lock() + defer loadGenerations.Unlock() + state, ok := loadGenerations.m[current.index] + if !ok || state.invalidation != current.invalidation || state.attempt != current.attempt { + return false + } + if v, ok := pendingLoadReasons.m[index]; ok && v.generation == reasonGeneration { + delete(pendingLoadReasons.m, index) + state.lastUsed = time.Now() + loadGenerations.m[current.index] = state + return true + } + return false +} + +func beginLoadGeneration(index string) loadGeneration { + now := time.Now() + loadGenerations.Lock() + defer loadGenerations.Unlock() + pruneLoadGenerationsLocked(now) + state := loadGenerations.m[index] + state.attempt++ + state.active++ + state.lastUsed = now + loadGenerations.m[index] = state + return loadGeneration{ + index: index, + invalidation: state.invalidation, + attempt: state.attempt, + owner: nextLoadOwner.Add(1), + } +} + +func endLoadGeneration(g loadGeneration) { + if g.index == "" || g.attempt == 0 { + return + } + now := time.Now() + loadGenerations.Lock() + defer loadGenerations.Unlock() + state, ok := loadGenerations.m[g.index] + if !ok { + return + } + if state.active > 0 { + state.active-- + } + state.lastUsed = now + loadGenerations.m[g.index] = state + pruneLoadGenerationsLocked(now) +} + +func loadGenerationCurrent(g loadGeneration) bool { + if g.index == "" || g.attempt == 0 { + return true + } + loadGenerations.Lock() + defer loadGenerations.Unlock() + state, ok := loadGenerations.m[g.index] + if !ok || state.invalidation != g.invalidation || state.attempt != g.attempt { + return false + } + state.lastUsed = time.Now() + loadGenerations.m[g.index] = state + return true +} + +func clearLoadGeneration(index string) { + if index == "" { + return + } + pendingLoadReasons.Lock() + loadGenerations.Lock() + defer func() { + loadGenerations.Unlock() + pendingLoadReasons.Unlock() + }() + delete(pendingLoadReasons.m, index) + state, ok := loadGenerations.m[index] + if !ok { + return + } + if state.active == 0 { + delete(loadGenerations.m, index) + return + } + state.invalidation++ + state.lastUsed = time.Now() + loadGenerations.m[index] = state +} + +func clearLoadGenerationRegistry() { + pendingLoadReasons.Lock() + loadGenerations.Lock() + pendingLoadReasons.m = make(map[string]pendingLoadReason) + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + pendingLoadReasons.Unlock() +} + +func clearReusableLoadGeneration(cfg TableConfig) { + index := loadReasonKey(cfg.DbName, cfg.IndexTable) + clearLoadGeneration(index) + clearReusableLoadPools(cfg) +} + +func clearReusableLoadPools(cfg TableConfig) { + index := loadReasonKey(cfg.DbName, cfg.IndexTable) + loadedBasePool.clearIndex(index) + loadedTailPool.clear(index) +} + +// invalidateLoadGeneration records why the next load will miss and clears +// reusable immutable state only when the base itself is known to have changed. +// It runs from the generic cache's invalidation hook, so the existing cache +// lifecycle remains the only integration surface for other index algorithms. +func invalidateLoadGeneration(cfg TableConfig, reason LoadMissReason) { + index := loadReasonKey(cfg.DbName, cfg.IndexTable) + rememberLoadReason(index, reason) + switch reason { + case LoadMissMerge, LoadMissRebuild, LoadMissReason("process_shutdown"): + if reason == LoadMissReason("process_shutdown") { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + } else { + clearReusableLoadPools(cfg) + } + case LoadMissTTLExpired, LoadMissGenerationChange: + // Keep the current index's immutable base/tail reusable across CDC and + // TTL refreshes, but sweep stale entries belonging to indexes that have + // not been queried recently. This gives the process-global pools a + // bounded idle lifetime without making every CDC refresh cold-load its + // own base. + loadedBasePool.evict(time.Now()) + loadedTailPool.evict(time.Now()) + } +} diff --git a/pkg/fulltext2/load_reason_test.go b/pkg/fulltext2/load_reason_test.go new file mode 100644 index 0000000000000..262f606848970 --- /dev/null +++ b/pkg/fulltext2/load_reason_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLoadReasonRegistryDisabledObserverIsNoOp(t *testing.T) { + cleanup := setLoadObserver(nil) + defer cleanup() + key := loadReasonKey("db", "store") + rememberLoadReason(key, LoadMissCDCFlush) + reason, generation := peekLoadReason(key) + require.Empty(t, reason) + require.Zero(t, generation) +} + +func TestLoadReasonRegistryStoresOnlyWhenEnabled(t *testing.T) { + cleanup := setLoadObserver(func(LoadEvent) {}) + defer cleanup() + key := loadReasonKey("db", "store") + rememberLoadReason(key, LoadMissCDCFlush) + reason, generation := peekLoadReason(key) + require.Equal(t, LoadMissCDCFlush, reason) + require.NotZero(t, generation) + consumeLoadReason(key, generation) + reason, generation = peekLoadReason(key) + require.Empty(t, reason) + require.Zero(t, generation) +} + +func TestLoadReasonRegistryBoundsExpiryAndInvalidation(t *testing.T) { + cleanupObserver := setLoadObserver(func(LoadEvent) {}) + defer cleanupObserver() + + pendingLoadReasons.Lock() + previous := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = previous + pendingLoadReasons.Unlock() + }) + + require.Equal(t, "index", loadReasonKey("", "index")) + pendingLoadReasons.Lock() + pendingLoadReasons.m["expired"] = pendingLoadReason{reason: LoadMissTTLExpired, at: time.Now().Add(-loadReasonTTL - time.Second)} + pendingLoadReasons.m["oldest"] = pendingLoadReason{reason: LoadMissCDCFlush, at: time.Now().Add(-time.Second)} + for i := 0; i < loadReasonSize-1; i++ { + pendingLoadReasons.m["seed-"+string(rune(i))] = pendingLoadReason{reason: LoadMissMerge, at: time.Now()} + } + pendingLoadReasons.Unlock() + + rememberLoadReason("overflow", LoadMissRebuild) + reason, generation := peekLoadReason("expired") + require.Empty(t, reason) + require.Zero(t, generation) + reason, generation = peekLoadReason("oldest") + require.Empty(t, reason) + require.Zero(t, generation) + reason, generation = peekLoadReason("overflow") + require.Equal(t, LoadMissRebuild, reason) + consumeLoadReason("overflow", generation) + reason, generation = peekLoadReason("missing") + require.Empty(t, reason) + require.Zero(t, generation) + + cfg := TableConfig{DbName: "db", IndexTable: "store"} + invalidateLoadGeneration(cfg, LoadMissCDCFlush) + reason, generation = peekLoadReason("db.store") + require.Equal(t, LoadMissCDCFlush, reason) + consumeLoadReason("db.store", generation) + invalidateLoadGeneration(cfg, LoadMissReason("process_shutdown")) + reason, generation = peekLoadReason("db.store") + require.Empty(t, reason) + require.Zero(t, generation) +} + +func TestLoadReasonRegistryDoesNotConsumeNewerInvalidationAfterReasonReplacement(t *testing.T) { + cleanupObserver := setLoadObserver(func(LoadEvent) {}) + defer cleanupObserver() + + pendingLoadReasons.Lock() + previous := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = previous + pendingLoadReasons.Unlock() + }) + + key := loadReasonKey("db", "store") + rememberLoadReason(key, LoadMissCDCFlush) + firstReason, firstGeneration := peekLoadReason(key) + require.Equal(t, LoadMissCDCFlush, firstReason) + rememberLoadReason(key, LoadMissRebuild) + consumeLoadReason(key, firstGeneration) + secondReason, secondGeneration := peekLoadReason(key) + require.Equal(t, LoadMissRebuild, secondReason) + consumeLoadReason(key, secondGeneration) +} diff --git a/pkg/fulltext2/membership.go b/pkg/fulltext2/membership.go new file mode 100644 index 0000000000000..69ae3605f05d5 --- /dev/null +++ b/pkg/fulltext2/membership.go @@ -0,0 +1,203 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// Membership is a doc-ord allow-set consulted during search for WHERE-clause +// prefiltering — the fulltext2 analogue of bm25/wand.Membership and cuVS filtered +// search. A nil Membership means "allow all" (no prefilter). It is built PER QUERY +// (the loaded Index is shared/cached across concurrent queries, so per-query filter +// state must never live on the Index) and threaded as a parameter into the segment +// searches, where it prunes candidates BEFORE they enter the top-k — so a pushed +// LIMIT bounds the FILTERED result, not the raw match set. +type Membership interface { + Contains(ord int64) bool +} + +// docFilterMembership resolves a pk-based docfilter.MembershipFilter (built in C from +// the pushed-down WHERE pks) against ONE segment's ord→pk dictionary: doc ord is +// allowed iff its pk bytes pass the filter. Mirrors bm25's docFilterMembership; +// fulltext2 shares the byte-identical pk codec, so the encoded probe bytes match what +// docfilter.Build hashed on the source side. For integer PKs this is the C int64 +// cbitmap (exact); for other PKs a bloom (its false positives are removed by the +// downstream join to the filtered source). +type docFilterMembership struct { + seg *Segment + f docfilter.MembershipFilter + scratch [8]byte // reused encode buffer for the hot integer-PK path (Test copies out) +} + +func (d *docFilterMembership) Contains(ord int64) bool { + if ord < 0 || ord >= int64(d.seg.numDocs()) { + return false + } + // pk(ord) decodes on demand on a loaded segment (a small box on this WHERE-prefilter + // hot path); the docmap pk encoding differs from the docfilter probe encoding for some + // types (uuid raw vs canonical), so the decoded value must be re-encoded below. + v := d.seg.pk(ord) + // Contains runs once per WAND candidate on the block-max walk hot path. For the + // common integer PKs, encode straight into the reused scratch buffer (byte-identical + // to packUint*) rather than allocating via encodePk per call; Test reads the bytes + // synchronously and does not retain them, so reuse is safe. + var raw []byte + switch types.T(d.seg.PkType) { + case types.T_int64: + binary.LittleEndian.PutUint64(d.scratch[:], uint64(v.(int64))) + raw = d.scratch[:8] + case types.T_uint64: + binary.LittleEndian.PutUint64(d.scratch[:], v.(uint64)) + raw = d.scratch[:8] + case types.T_int32: + binary.LittleEndian.PutUint32(d.scratch[:4], uint32(v.(int32))) + raw = d.scratch[:4] + case types.T_uint32: + binary.LittleEndian.PutUint32(d.scratch[:4], v.(uint32)) + raw = d.scratch[:4] + case types.T_uuid: + // The membership filter hashes each source uuid as its RAW 16 bytes; probe with + // the same raw bytes, NOT encodePk (the 36-char canonical string), which would + // never hit the same bloom cell and reject every candidate. + u := v.(types.Uuid) + raw = u[:] + default: + var err error + if raw, err = encodePk(d.seg.PkType, v); err != nil { + return false + } + } + return d.f.Test(raw) +} + +// prefilter bundles the per-query WHERE prefilters threaded into every segment search: the +// pk-based docfilter (WHERE pushdown resolved in C) AND compiled INCLUDE-column predicates +// (evaluated in Go against the stored per-doc value). Either may be empty; a nil *prefilter +// means "allow all". Built once per query in Fulltext2Search.Search and bound to each +// segment by mkAllow. It must never live on the shared Index (per-query state). +type prefilter struct { + docFilter docfilter.MembershipFilter + include []compiledIncludePred + // scoreRange is the pushed-down relevance interval. Unlike the membership + // filters above it cannot prune before scoring; it is applied at the point a + // scored document becomes a result (streamSink or runTopK). + scoreRange *ScoreRange +} + +// ScoreRange is a relevance interval pushed into a FULLTEXT2 search. Both ends +// are optional and retain SQL comparison inclusivity. The planner widens bounds +// outward in float32 space before passing them here, so this filter cannot drop +// a row that the SQL predicate would keep. +type ScoreRange struct { + Min float32 `json:"min,omitempty"` + Max float32 `json:"max,omitempty"` + HasMin bool `json:"has_min,omitempty"` + HasMax bool `json:"has_max,omitempty"` + MinInclusive bool `json:"min_inc,omitempty"` + MaxInclusive bool `json:"max_inc,omitempty"` +} + +// contains reports whether score is inside r. A nil range allows every score. +func (r *ScoreRange) contains(score float32) bool { + if r == nil { + return true + } + if r.HasMin { + if r.MinInclusive { + if score < r.Min { + return false + } + } else if score <= r.Min { + return false + } + } + if r.HasMax { + if r.MaxInclusive { + if score > r.Max { + return false + } + } else if score >= r.Max { + return false + } + } + return true +} + +// mkAllow returns a per-segment WHERE-prefilter Membership over seg's ords (the docfilter +// AND the INCLUDE predicates), or nil when there is no filter (the "allow all" fast path). +func mkAllow(seg *Segment, p *prefilter) Membership { + if p == nil { + return nil + } + var m Membership + if p.docFilter != nil { + m = &docFilterMembership{seg: seg, f: p.docFilter} + } + if len(p.include) > 0 { + m = andAllow(m, &includePredMembership{seg: seg, preds: p.include}) + } + return m +} + +// allowed reports whether ord passes the membership (nil = allow all) — the one-liner +// the segment searches use at each admit point. +func allowed(m Membership, ord int64) bool { return m == nil || m.Contains(ord) } + +// livenessMembership admits ord in segment si iff it is the LIVE copy of its pk +// (highest-Recency, not delete-shadowed). Threaded into the per-segment boolean walk +// so a segment's top-k is k LIVE docs — a dead copy (superseded by a later CDC +// segment, or deleted) never enters the heap, so the cross-segment merge can't +// under-fill by later dropping it. Mirrors bm25's per-segment liveness Membership. +type livenessMembership struct { + idx *Index + si int +} + +func (l *livenessMembership) Contains(ord int64) bool { + // Bounds-check on BOTH paths: never admit an out-of-range ord (the scoring that + // follows indexes seg.pks[ord] and would panic). + if ord < 0 || ord >= int64(l.idx.segments[l.si].numDocs()) { + return false + } + b := l.idx.liveOrd[l.si] + if b == nil { + return true // fully-live segment (no dead copies) — the fast path + } + return b[ord] +} + +// andMembership is the conjunction of two memberships (either may be nil = allow all). +type andMembership struct{ a, b Membership } + +func (m andMembership) Contains(ord int64) bool { + return (m.a == nil || m.a.Contains(ord)) && (m.b == nil || m.b.Contains(ord)) +} + +// andAllow ANDs two memberships, collapsing nils (used to AND the WHERE prefilter with +// per-segment liveness). nil AND x = x. +func andAllow(a, b Membership) Membership { + switch { + case a == nil: + return b + case b == nil: + return a + default: + return andMembership{a, b} + } +} diff --git a/pkg/fulltext2/membership_test.go b/pkg/fulltext2/membership_test.go new file mode 100644 index 0000000000000..c3c0c870fa231 --- /dev/null +++ b/pkg/fulltext2/membership_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// TestPrefilterMembership exercises the WHERE-clause membership prefilter end to end +// against a real C docfilter (the same bytes the runtime-filter pushdown ships): a +// candidate doc is admitted iff its pk passes the filter, at every admit point — the +// WAND walk (disjunctive), the full boolean evaluator (MUST), and NL phrase — and a +// nil filter is unchanged. It also pins the ord→pk encode in docFilterMembership. +func TestPrefilterMembership(t *testing.T) { + mp := mpool.MustNewZero() + b := NewBuilder("pf", int32(types.T_int64)) + for i := int64(0); i < 10; i++ { + feed(t, b, i, "x", "y") // "x" and "y" occur in every doc + } + seg, err := b.Finish() + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + // Membership over the EVEN pks {0,2,4,6,8}, built exactly as the pushdown does. + vec := vector.NewVec(types.New(types.T_int64, 8, 0)) + for i := int64(0); i < 10; i += 2 { + require.NoError(t, vector.AppendFixed(vec, i, false, mp)) + } + fbytes, err := docfilter.Build(vec) + require.NoError(t, err) + filter, err := docfilter.New(fbytes) + require.NoError(t, err) + defer filter.Free() + + // docFilterMembership.Contains: even ords pass, odd rejected (pk-encode is correct). + dfm := &docFilterMembership{seg: seg, f: filter} + for i := int64(0); i < 10; i++ { + require.Equalf(t, i%2 == 0, dfm.Contains(i), "ord %d", i) + } + + even := []any{int64(0), int64(2), int64(4), int64(6), int64(8)} + + // WAND path (pure disjunction "x") with the filter → only even pks. + rWand, err := idx.SearchQuery([]byte("x"), true, ParserDefault, BM25, 100, &prefilter{docFilter: filter}) + require.NoError(t, err) + require.ElementsMatch(t, even, resultIDs(rWand)) + + // Full boolean evaluator (MUST "+x") with the filter → only even pks. + rBool, err := idx.SearchQuery([]byte("+x"), true, ParserDefault, BM25, 100, &prefilter{docFilter: filter}) + require.NoError(t, err) + require.ElementsMatch(t, even, resultIDs(rBool)) + + // NL phrase path ("x") with the filter → only even pks. + rNL, err := idx.SearchQuery([]byte("x"), false, ParserDefault, BM25, 100, &prefilter{docFilter: filter}) + require.NoError(t, err) + require.ElementsMatch(t, even, resultIDs(rNL)) + + // nil filter is unchanged: all 10 docs. + rAll, err := idx.SearchQuery([]byte("x"), true, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + require.Len(t, rAll, 10) + + // The filter also correctly bounds a pushed LIMIT to the FILTERED set: top-2 over + // the even docs returns 2 even pks (not 2 of all 10 then post-filtered to fewer). + rLimit, err := idx.SearchQuery([]byte("x"), true, ParserDefault, BM25, 2, &prefilter{docFilter: filter}) + require.NoError(t, err) + require.Len(t, rLimit, 2) + for _, pk := range resultIDs(rLimit) { + require.Contains(t, even, pk) + } +} + +// TestPhraseDfFilterIndependent pins re-review [0]: an NL phrase's idf df is the +// CORPUS phrase df (all live matches), not the WHERE-filtered subset, so a matched +// doc's score must NOT change when an unrelated WHERE prefilter is applied. +func TestPhraseDfFilterIndependent(t *testing.T) { + mp := mpool.MustNewZero() + b := NewBuilder("df", int32(types.T_int64)) + for i := int64(0); i < 4; i++ { + feed(t, b, i, "alpha") // 4 docs contain the phrase + } + for i := int64(4); i < 8; i++ { + feed(t, b, i, "other") + } + seg, err := b.Finish() + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + scoreOf := func(rs []Result, pk int64) float64 { + for _, r := range rs { + if r.Pk.(int64) == pk { + return float64(r.Score) + } + } + t.Fatalf("pk %d not in results", pk) + return 0 + } + unfiltered := idx.SearchPhrase(phr("alpha"), BM25, 100, nil) + require.Len(t, unfiltered, 4) + s0 := scoreOf(unfiltered, 0) + + // WHERE prefilter allowing only {0,1} — df must stay 4 (corpus), so doc 0's score + // is unchanged; before the fix df shrank to 2 and the score differed. + vec := vector.NewVec(types.New(types.T_int64, 8, 0)) + require.NoError(t, vector.AppendFixed(vec, int64(0), false, mp)) + require.NoError(t, vector.AppendFixed(vec, int64(1), false, mp)) + fb, err := docfilter.Build(vec) + require.NoError(t, err) + filter, err := docfilter.New(fb) + require.NoError(t, err) + defer filter.Free() + + filtered := idx.SearchPhrase(phr("alpha"), BM25, 100, &prefilter{docFilter: filter}) + require.Len(t, filtered, 2, "only the allowed docs are returned") + require.InDelta(t, s0, scoreOf(filtered, 0), 1e-5, + "doc 0's score must be filter-independent (corpus df, not filtered count)") +} + +func TestScoreRangeContains(t *testing.T) { + var nilRange *ScoreRange + require.True(t, nilRange.contains(0)) + + r := &ScoreRange{Min: 1, HasMin: true, MinInclusive: true, Max: 3, HasMax: true} + require.True(t, r.contains(1)) + require.True(t, r.contains(2)) + require.False(t, r.contains(0.99)) + require.False(t, r.contains(3), "exclusive upper bound") + + r.MaxInclusive = true + require.True(t, r.contains(3)) + r.MinInclusive = false + require.False(t, r.contains(1), "exclusive lower bound") +} diff --git a/pkg/fulltext2/mmap_other.go b/pkg/fulltext2/mmap_other.go new file mode 100644 index 0000000000000..4b246603b7ca2 --- /dev/null +++ b/pkg/fulltext2/mmap_other.go @@ -0,0 +1,33 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !darwin && !linux + +package fulltext2 + +import ( + "io" + "os" +) + +// mmapReadOnly fallback (non-unix): read the whole file into a Go slice. No page +// cache benefit, but the load/query API is identical; munmap is a no-op. +func mmapReadOnly(f *os.File) ([]byte, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, err + } + return io.ReadAll(f) +} + +func munmap(b []byte) error { return nil } diff --git a/pkg/fulltext2/mmap_test.go b/pkg/fulltext2/mmap_test.go new file mode 100644 index 0000000000000..ba99ee3952b89 --- /dev/null +++ b/pkg/fulltext2/mmap_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "os" + "sort" + "sync" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestMmapLoad validates the base-segment mmap path (the storage LoadFromStorage +// shape, without a cluster): serialize a segment to a file, mmap it, decode over +// the mapping — FST + positions are views into the shared mmap, docIDs/tfs expand +// off-heap — then search (ranking + phrase), and confirm Free() munmaps and deletes +// the file. +func TestMmapLoad(t *testing.T) { + b := NewBuilder("mm", int32(types.T_int64)) + feed(t, b, int64(0), "alpha", "beta", "alpha") // "alpha beta" phrase present + feed(t, b, int64(1), "beta", "gamma") + feed(t, b, int64(2), "alpha", "gamma", "beta") + built, err := b.Finish() + require.NoError(t, err) + blob, err := built.Serialize() + require.NoError(t, err) + + f, err := os.CreateTemp("", "ft2mmtest") + require.NoError(t, err) + path := f.Name() + _, err = f.Write(blob) + require.NoError(t, err) + require.NoError(t, f.Sync()) + data, err := mmapReadOnly(f) + f.Close() + require.NoError(t, err) + + s := &Segment{Id: "mm", mmapData: data, mmapPath: path} + require.NoError(t, s.decodeSegment(data)) + require.NotNil(t, s.mmapData, "segment holds the mmap") + + // Every term: docIDs off-heap match; positions (posRaw into the mmap) materialize + // to exactly the build-side positions. + for _, term := range built.sortedTerms { + want, ok := built.Lookup(term) + require.True(t, ok, term) + got, ok := s.LookupLoaded(term) + require.True(t, ok, term) + require.Equal(t, want.docIDs, got.materializeDocIDs(), term) + gp := got.materializePositions() + require.Equal(t, len(want.positions), len(gp), term) + for i := range want.positions { + require.Equalf(t, want.positions[i], gp[i], "%s positions[%d]", term, i) + } + } + + idx := NewIndex([]*Segment{s}, nil) + // ranking (WAND, docIDs off-heap): "beta" hits all three docs. + require.Len(t, idx.SearchPhrase(phr("beta"), BM25, 10, nil), 3) + // phrase (positions from the mmap): only doc 0 has contiguous "alpha beta". + ph := idx.SearchPhrase(phr("alpha", "beta"), BM25, 10, nil) + require.Len(t, ph, 1) + require.Equal(t, int64(0), ph[0].Pk) + + // Free munmaps and deletes the owned file. + idx.Free() + require.Nil(t, s.mmapData) + _, statErr := os.Stat(path) + require.True(t, os.IsNotExist(statErr), "Free must delete the mmap-owned file") +} + +// TestConcurrentBlockDecode proves the loaded-segment design invariant: docID/tf +// blocks live in ONE shared, read-only mmap, and each WAND cursor decodes into its +// OWN per-cursor buffer — so many queries can hit the same segment concurrently with +// no lock and no race. Terms span many blocks (N/BlockSize each) so skipTo crosses +// block boundaries, and results must match a single-threaded reference exactly. Run +// under -race, this is the guard against a shared/mutable block cache. +func TestConcurrentBlockDecode(t *testing.T) { + const N = 4000 // ~31 blocks for "alpha" + b := NewBuilder("cc", int32(types.T_int64)) + for i := 0; i < N; i++ { + toks := []string{"alpha"} + if i%2 == 0 { + toks = append(toks, "beta") + } + if i%3 == 0 { + toks = append(toks, "gamma") + } + feed(t, b, int64(i), toks...) + } + built, err := b.Finish() + require.NoError(t, err) + blob, err := built.Serialize() + require.NoError(t, err) + + f, err := os.CreateTemp("", "cctest") + require.NoError(t, err) + path := f.Name() + _, err = f.Write(blob) + require.NoError(t, err) + require.NoError(t, f.Sync()) + data, err := mmapReadOnly(f) + f.Close() + require.NoError(t, err) + s := &Segment{Id: "cc", mmapData: data, mmapPath: path} + require.NoError(t, s.decodeSegment(data)) + idx := NewIndex([]*Segment{s}, nil) + defer idx.Free() + + // Reference (single-threaded): a disjunctive WAND top-k over all three terms. + pkset := func(rs []Result) []int64 { + out := make([]int64, len(rs)) + for i, r := range rs { + out[i] = r.Pk.(int64) + } + sort.Slice(out, func(a, b int) bool { return out[a] < out[b] }) + return out + } + ref, err := idx.SearchQuery([]byte("alpha beta gamma"), true, "default", BM25, 100, nil) + require.NoError(t, err) + require.NotEmpty(t, ref) + want := pkset(ref) + + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for it := 0; it < 40; it++ { + got, e := idx.SearchQuery([]byte("alpha beta gamma"), true, "default", BM25, 100, nil) + require.NoError(t, e) + require.Equal(t, want, pkset(got)) // same top-k SET (ties aside) + } + }() + } + wg.Wait() +} diff --git a/pkg/fulltext2/mmap_unix.go b/pkg/fulltext2/mmap_unix.go new file mode 100644 index 0000000000000..92be18430a04f --- /dev/null +++ b/pkg/fulltext2/mmap_unix.go @@ -0,0 +1,47 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin || linux + +package fulltext2 + +import ( + "os" + "syscall" +) + +// mmapReadOnly maps the whole file into a shared, read-only region. One mapping is +// created per cached base segment at load and shared by all concurrent queries — +// reads are plain memory loads (no lock), the kernel serializes page faults, and +// the OS page cache (reclaimable, unlike our off-heap C-malloc) is the residency +// manager. Unmapped by munmap under the cache's eviction write-lock (no reader in +// flight). Returns nil for a zero-length file. +func mmapReadOnly(f *os.File) ([]byte, error) { + fi, err := f.Stat() + if err != nil { + return nil, err + } + if fi.Size() == 0 { + return nil, nil + } + return syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) +} + +// munmap releases a mapping from mmapReadOnly (no-op on nil/empty). +func munmap(b []byte) error { + if len(b) == 0 { + return nil + } + return syscall.Munmap(b) +} diff --git a/pkg/fulltext2/multiseg_test.go b/pkg/fulltext2/multiseg_test.go new file mode 100644 index 0000000000000..5e3dab8b392d5 --- /dev/null +++ b/pkg/fulltext2/multiseg_test.go @@ -0,0 +1,217 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestTemporalPkNoCollision pins the value-keyed liveness fix: two DATETIME pks that +// differ only in microseconds render to the SAME %v/String() (which truncates to +// whole seconds), so a string-keyed liveness map would collide them and silently drop +// one live doc. normalizeKey keys by the pk VALUE, so both stay distinct. +func TestTemporalPkNoCollision(t *testing.T) { + const sec = int64(63745056000) // arbitrary second count + dt1 := types.Datetime(sec*1_000_000 + 1) // ...000001 microseconds + dt2 := types.Datetime(sec*1_000_000 + 2) // ...000002 microseconds + require.Equal(t, fmt.Sprintf("%v", dt1), fmt.Sprintf("%v", dt2), + "precondition: the two pks DO collide under %%v (String drops microseconds)") + + b := NewBuilder("dt", int32(types.T_datetime)) + feed(t, b, dt1, "alpha") + feed(t, b, dt2, "alpha") + seg, err := b.Finish() + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + require.Equal(t, int64(2), idx.globalN, "both microsecond-distinct pks must be live") + require.Len(t, idx.SearchPhrase(phr("alpha"), BM25, 10, nil), 2, + "both docs must be returned (neither dropped by a lossy key)") +} + +// docWords are 24 docs with skewed term frequencies (alpha common, beta medium, +// gamma/delta rarer) and varying lengths, so idf and the BM25 length-norm both +// matter — a boolean ranking is only correct if scored on GLOBAL corpus stats. +func docWords() [][]string { + out := make([][]string, 24) + for i := 0; i < 24; i++ { + w := []string{"alpha"} // alpha in every doc + if i%2 == 0 { + w = append(w, "beta", "beta") + } + if i%3 == 0 { + w = append(w, "gamma") + } + if i%4 == 0 { + w = append(w, "delta", "delta", "delta") + } + if i%5 == 0 { + w = append(w, "beta") + } + out[i] = w + } + return out +} + +func buildFtSeg(t *testing.T, id string, rec int64, lo, hi int) *Segment { + b := NewBuilder(id, int32(types.T_int64)) + words := docWords() + for i := lo; i < hi; i++ { + feed(t, b, int64(i), words[i]...) + } + s, err := b.Finish() + require.NoError(t, err) + s.Recency = rec + return s +} + +// TestMultiSegmentGlobalStats pins the #5 fix: a boolean query over docs SPLIT across +// a base + CDC-tail (three segments) must return the IDENTICAL ranking + scores as the +// same docs in ONE segment. The global corpus stats (N, avgDocLen, per-term df summed +// across segments) make cross-segment scores comparable; before the fix each segment +// used its LOCAL N/df, so the split index scored — and thus ranked / truncated — docs +// differently from the single-segment truth. +func TestMultiSegmentGlobalStats(t *testing.T) { + // Reference: all 24 docs in one segment. + ref := NewIndex([]*Segment{buildFtSeg(t, "ref", 0, 0, 24)}, nil) + // Split: disjoint pk ranges across three segments (a base + two CDC tails), each + // with its own (smaller) local N/df. + split := NewIndex([]*Segment{ + buildFtSeg(t, "a", 0, 0, 10), + buildFtSeg(t, "b", 1, 10, 18), + buildFtSeg(t, "c", 2, 18, 24), + }, nil) + + scoreByPk := func(rs []Result) map[int64]float64 { + m := make(map[int64]float64, len(rs)) + for _, r := range rs { + m[r.Pk.(int64)] = float64(r.Score) + } + return m + } + for _, q := range []string{"alpha beta gamma delta", "beta gamma", "+alpha beta", "alpha delta"} { + // Full result (k large): the split index must return the SAME set of pks, each + // with the SAME score as the single-segment truth — the direct proof that the + // per-segment idf drift is gone (a dropped or rescored doc would diverge here). + want, err := ref.SearchQuery([]byte(q), true, ParserDefault, BM25, 1000, nil) + require.NoError(t, err) + got, err := split.SearchQuery([]byte(q), true, ParserDefault, BM25, 1000, nil) + require.NoError(t, err) + ws, gsc := scoreByPk(want), scoreByPk(got) + require.Equalf(t, len(ws), len(gsc), "q=%q: same matched-doc count", q) + for pk, s := range ws { + g, ok := gsc[pk] + require.Truef(t, ok, "q=%q: pk %d present in split", q, pk) + require.InDeltaf(t, s, g, 1e-5, "q=%q pk %d: global score matches single-segment", q, pk) + } + // Bounded top-k: the ranked SCORE sequence is identical (which tie member fills + // the k-th slot is arbitrary — the documented WAND tie caveat — but the scores at + // every rank, and the count, must match; a truncation loss would shorten got). + for _, k := range []int{3, 5, 12} { + w, err := ref.SearchQuery([]byte(q), true, ParserDefault, BM25, k, nil) + require.NoError(t, err) + g, err := split.SearchQuery([]byte(q), true, ParserDefault, BM25, k, nil) + require.NoError(t, err) + require.Lenf(t, g, len(w), "q=%q k=%d: same count (no truncation loss)", q, k) + for i := range w { + require.InDeltaf(t, w[i].Score, g[i].Score, 1e-5, "q=%q k=%d rank=%d: score", q, k, i) + } + } + } +} + +// TestMultiSegmentPhraseStats pins the ② fix: a boolean-mode PHRASE clause scores on +// the cross-segment (global) phrase df, so a phrase query over a base + CDC-tail split +// ranks identically to the same docs in one segment (before the fix the phrase clause +// used per-segment idf and diverged). +func TestMultiSegmentPhraseStats(t *testing.T) { + words := func(i int) []string { + w := []string{"filler", "text"} + if i%3 == 0 { + w = append(w, "quick", "brown", "fox") // contiguous "quick brown" phrase + } + if i%2 == 0 { + w = append(w, "quick") // 'quick' alone — not the phrase + } + return w + } + build := func(id string, rec int64, lo, hi int) *Segment { + b := NewBuilder(id, int32(types.T_int64)) + for i := lo; i < hi; i++ { + feed(t, b, int64(i), words(i)...) + } + s, err := b.Finish() + require.NoError(t, err) + s.Recency = rec + return s + } + ref := NewIndex([]*Segment{build("ref", 0, 0, 18)}, nil) + split := NewIndex([]*Segment{build("a", 0, 0, 9), build("b", 1, 9, 18)}, nil) + + byPk := func(rs []Result) map[int64]float64 { + m := make(map[int64]float64, len(rs)) + for _, r := range rs { + m[r.Pk.(int64)] = float64(r.Score) + } + return m + } + q := `"quick brown"` // quoted => boolean-mode phrase clause + want, err := ref.SearchQuery([]byte(q), true, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + got, err := split.SearchQuery([]byte(q), true, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + ws, gsc := byPk(want), byPk(got) + require.NotEmpty(t, ws) + require.Equal(t, len(ws), len(gsc), "same phrase-matched set") + for pk, s := range ws { + g, ok := gsc[pk] + require.Truef(t, ok, "pk %d present in split", pk) + require.InDeltaf(t, s, g, 1e-5, "pk %d: global phrase idf matches single-segment", pk) + } +} + +// TestMultiSegmentLiveness pins the liveness-inside-the-walk half of #5: when a pk is +// UPDATEd (a later, higher-Recency segment holds a fresh copy), a boolean query must +// score/return only the LIVE copy — the dead copy in the older segment must never +// enter a segment's top-k (which would let the merge under-fill by dropping it). +func TestMultiSegmentLiveness(t *testing.T) { + base := NewBuilder("base", int32(types.T_int64)) + feed(t, base, int64(1), "alpha", "alpha") // pk1: old text = alpha + feed(t, base, int64(2), "alpha") // pk2: alpha (never updated) + baseSeg, err := base.Finish() + require.NoError(t, err) + baseSeg.Recency = 0 + + tail := NewBuilder("tail", int32(types.T_int64)) + feed(t, tail, int64(1), "beta", "beta") // pk1 UPDATEd: new text = beta + tailSeg, err := tail.Finish() + require.NoError(t, err) + tailSeg.Recency = 1 + + idx := NewIndex([]*Segment{baseSeg, tailSeg}, nil) + + ids := func(q string) []any { + r, err := idx.SearchQuery([]byte(q), true, ParserDefault, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(r) + } + // pk1's OLD alpha copy is dead → alpha matches only pk2; pk1's live copy is beta. + require.ElementsMatch(t, []any{int64(2)}, ids("alpha")) + require.ElementsMatch(t, []any{int64(1)}, ids("beta")) +} diff --git a/pkg/fulltext2/parser_test.go b/pkg/fulltext2/parser_test.go new file mode 100644 index 0000000000000..e045b17c3ae18 --- /dev/null +++ b/pkg/fulltext2/parser_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Parser support: the ngram (default), gojieba, and json parsers over a fulltext2 +// index. CJK is the focus — NL exact-phrase over overlapping 3-grams is unreliable, +// so ngram/json NL routes CJK through classic fulltext's redundant-ngram removal +// (bag-of-words), while gojieba word-segmentation keeps exact-phrase. +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// zhCorpus indexes five CJK docs under parser and returns a queryable Index. +// +// 0: 遠東兒童中文學習教材 (contains 中文學習, 兒童) +// 1: 中文短篇小說適合初學者 (contains 中文, no 學習/兒童) +// 2: 兒童學習樂趣 (contains 兒童, 學習) +// 3: 教學指引與生字卡 (none of the above) +// 4: 中文學習中文學習 (contains 中文學習 twice) +func zhCorpus(t *testing.T, parser string) *Index { + docs := []Doc{ + {int64(0), []byte("遠東兒童中文學習教材")}, + {int64(1), []byte("中文短篇小說適合初學者")}, + {int64(2), []byte("兒童學習樂趣")}, + {int64(3), []byte("教學指引與生字卡")}, + {int64(4), []byte("中文學習中文學習")}, + } + seg, err := BuildSegmentFromDocsParser("zh", int32(types.T_int64), docs, parser) + require.NoError(t, err) + return NewIndex([]*Segment{seg}, nil) +} + +func nlIDs(t *testing.T, idx *Index, parser, q string) []any { + res, err := idx.SearchQuery([]byte(q), false, parser, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(res) +} + +func boolIDs(t *testing.T, idx *Index, parser, q string) []any { + res, err := idx.SearchQuery([]byte(q), true, parser, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(res) +} + +func resultIDs(res []Result) []any { + out := make([]any, len(res)) + for i, r := range res { + out[i] = r.Pk + } + return out +} + +func requireJieba(t *testing.T) { + if _, err := DocTokenizer(ParserGojieba); err != nil { + t.Skipf("gojieba dictionary unavailable: %v", err) + } +} + +// --- ngram (default) parser: CJK NL is bag-of-words over redundant-removed 3-grams --- + +func TestNgramNLChinese(t *testing.T) { + idx := zhCorpus(t, ParserNgram) + // 中文學習 -> ngrams 中文學, 文學習 (bag-of-words); docs 0 and 4 contain both. + require.ElementsMatch(t, []any{int64(0), int64(4)}, nlIDs(t, idx, ParserNgram, "中文學習")) + // 兒童 (<3 chars) -> prefix 兒童* ; matches 3-grams starting 兒童 in docs 0 and 2. + require.ElementsMatch(t, []any{int64(0), int64(2)}, nlIDs(t, idx, ParserNgram, "兒童")) + // unknown compound -> no hit. + require.Empty(t, nlIDs(t, idx, ParserNgram, "西班牙語")) +} + +func TestNgramBooleanChinese(t *testing.T) { + idx := zhCorpus(t, ParserNgram) + // +中文 +學習 : both compounds' ngrams present -> docs 0, 4 (2 has 學習 but not 中文). + require.ElementsMatch(t, []any{int64(0), int64(4)}, boolIDs(t, idx, ParserNgram, "+中文 +學習")) + // +兒童 -中文 : 兒童 present, 中文 absent -> only doc 2 (doc 0 has both). + require.ElementsMatch(t, []any{int64(2)}, boolIDs(t, idx, ParserNgram, "+兒童 -中文")) +} + +func TestNgramLatinStaysPhrase(t *testing.T) { + // Latin under ngram keeps fulltext2 exact-phrase NL. + docs := []Doc{ + {int64(0), []byte("the quick brown fox")}, + {int64(1), []byte("quick red fox")}, + {int64(2), []byte("brown fox jumps")}, + } + seg, err := BuildSegmentFromDocsParser("en", int32(types.T_int64), docs, ParserNgram) + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + // "brown fox" is contiguous only in docs 0 and 2 (not doc 1: quick red fox). + require.ElementsMatch(t, []any{int64(0), int64(2)}, nlIDs(t, idx, ParserNgram, "brown fox")) +} + +// --- gojieba parser: word segmentation, exact-phrase NL works for CJK --- + +func TestGojiebaNLChinese(t *testing.T) { + requireJieba(t) + idx := zhCorpus(t, ParserGojieba) + // jieba segments 中文學習 into words; exact-phrase matches docs 0 and 4. + require.ElementsMatch(t, []any{int64(0), int64(4)}, nlIDs(t, idx, ParserGojieba, "中文學習")) + // 兒童 is a jieba word present in docs 0 and 2. + require.ElementsMatch(t, []any{int64(0), int64(2)}, nlIDs(t, idx, ParserGojieba, "兒童")) +} + +func TestGojiebaBooleanChinese(t *testing.T) { + requireJieba(t) + idx := zhCorpus(t, ParserGojieba) + require.ElementsMatch(t, []any{int64(0), int64(4)}, boolIDs(t, idx, ParserGojieba, "+中文 +學習")) + require.ElementsMatch(t, []any{int64(2)}, boolIDs(t, idx, ParserGojieba, "+兒童 -中文")) +} + +// --- json parser: values flattened to text, then indexed/queried as ngram --- + +func TestJSONParser(t *testing.T) { + docs := []Doc{ + {int64(0), []byte(`{"a":1, "b":"red apple"}`)}, + {int64(1), []byte(`{"a":2, "b":"中文學習教材"}`)}, + {int64(2), []byte(`{"a":3, "b":"red blue"}`)}, + } + seg, err := BuildSegmentFromDocsParser("js", int32(types.T_int64), docs, ParserJSON) + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + // Latin value token, boolean. + require.ElementsMatch(t, []any{int64(0), int64(2)}, boolIDs(t, idx, ParserJSON, "red")) + // CJK value: 中文學習 -> ngram bag-of-words, only doc 1. + require.ElementsMatch(t, []any{int64(1)}, nlIDs(t, idx, ParserJSON, "中文學習")) +} diff --git a/pkg/fulltext2/phrase_cursor_test.go b/pkg/fulltext2/phrase_cursor_test.go new file mode 100644 index 0000000000000..ca48d24452394 --- /dev/null +++ b/pkg/fulltext2/phrase_cursor_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "fmt" + "math" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestPhraseCursorMatchesFallback proves the block-cursor phrase intersection +// (matchPhraseCursor) returns EXACTLY what the materialize fallback returns, across many +// blocks (> BlockSize docs, so the cursor crosses block boundaries, skips non-overlapping +// blocks, and decodes positions per block) — on both a build-side and a serialized/loaded +// segment. The fallback is the trusted oracle; any divergence is a cursor bug. +func TestPhraseCursorMatchesFallback(t *testing.T) { + phrases := []string{ + "中文學習", "學習教材", "遠東兒童", "兒童中文", "教學指引", + "中文學習教材", "生字卡片", "初學者適合", "短篇小說", "樂趣無窮", + } + // 500 docs (~4 blocks). Each doc concatenates a deterministic rotating subset of the + // phrases (some repeated → tf>1), so phrases appear in scattered, block-spanning docs. + docs := make([]Doc, 0, 500) + for i := 0; i < 500; i++ { + var b bytes.Buffer + for k := 0; k < 3; k++ { + b.WriteString(phrases[(i+k*3)%len(phrases)]) + b.WriteByte(' ') + } + if i%5 == 0 { // repeat one phrase → occurrence count > 1 + b.WriteString(phrases[i%len(phrases)]) + } + docs = append(docs, Doc{int64(i + 1), b.Bytes()}) + } + + seg, err := BuildSegmentFromDocsParser("cur", int32(types.T_int64), docs, ParserNgram) + require.NoError(t, err) + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("cur", bytes.NewReader(blob)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + // Multi-trigram queries (each decomposes to >= 2 exact slots → the cursor path). + queries := []string{ + "中文學習", "學習教材", "中文學習教材", "遠東兒童中文", "教學指引", + "初學者適合", "短篇小說樂趣", "生字卡片", "兒童中文學習", "不存在的詞語", + } + key := func(h []docTf) []string { + out := make([]string, len(h)) + for i, d := range h { + out[i] = fmt.Sprintf("%d:%d", d.ord, d.tf) + } + return out + } + + for _, name := range []string{"build", "loaded"} { + sg := seg + if name == "loaded" { + sg = loaded + } + t.Run(name, func(t *testing.T) { + for _, q := range queries { + slots, err := phraseSlots(q, ParserNgram) + require.NoError(t, err) + if len(slots) < 2 { + continue // cursor path is multi-slot only + } + cur := sg.matchPhraseCursor(slots) + fb := sg.matchPhraseFallback(slots) + require.ElementsMatch(t, key(fb), key(cur), "query %q (ord:tf must match the oracle)", q) + } + }) + } +} + +func TestPhraseCursorDefersPositionsUntilRequested(t *testing.T) { + data, err := syntheticCorpus(t).Serialize() + require.NoError(t, err) + loaded, err := Deserialize("lazy-pos", bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + tp, ok := loaded.lookup("alpha") + require.True(t, ok) + c := newPhraseCursor(tp, 0) + defer releasePhraseCursors([]*phraseCursor{c}) + + require.Equal(t, -1, c.curDocBlk) + require.Equal(t, -1, c.curPosBlk) + first := c.doc() + require.NotEqual(t, int64(math.MaxInt64), first) + require.Equal(t, 0, c.curDocBlk) + require.Equal(t, -1, c.curPosBlk, "reading doc IDs must not decode positions") + + c.skipTo(first + 1) + require.Equal(t, -1, c.curPosBlk, "skipTo must not decode positions") + if !c.atEnd() { + _ = c.positions() + require.Equal(t, c.idx/BlockSize, c.curPosBlk) + } +} diff --git a/pkg/fulltext2/phrase_exact_test.go b/pkg/fulltext2/phrase_exact_test.go new file mode 100644 index 0000000000000..db5e5d2d30733 --- /dev/null +++ b/pkg/fulltext2/phrase_exact_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestExactPhraseSemantics pins fulltext2's NL and boolean-phrase EXACT-MATCH contract — +// the regression that previously escaped every unit test because the old tests used only +// English (where the ngram tokenizer degenerates to whole words, so a CJK-specific bug is +// invisible). It exercises the production query path (SearchQuery) so it covers the query +// tokenization + byte-position positional phrase matcher end to end. +// +// - NL multi-term is an EXACT ORDERED phrase: a query matches a doc ONLY where the +// terms appear in order and adjacent — NOT reversed, NOT with a word between. The +// old CJK "bag-of-words" path matched all three (a false positive on reversed / +// non-adjacent docs); a naive raw-ngram phrase matched none of the substring cases. +// - A query that is a contiguous SUBSTRING of a longer doc matches (中文學習 ⊂ +// 中文學習教材) — the case the raw-ngram phrase missed because the query's trailing +// partial n-grams are absent from the longer doc's FST. +// - Byte positions are consistent across the build-side AND the serialized/loaded +// segment (so a query works the same after a restart/merge). +func TestExactPhraseSemantics(t *testing.T) { + docs := []Doc{ + {int64(1), []byte("全文 索引")}, // in order, adjacent + {int64(2), []byte("索引 全文")}, // reversed + {int64(3), []byte("全文 系统 索引")}, // in order but NOT adjacent + {int64(4), []byte("中文學習教材")}, // superset of 中文學習 + {int64(5), []byte("quick brown fox")}, // English, in order + {int64(6), []byte("brown quick")}, // English, reversed + {int64(7), []byte("中文學校")}, // shares trigram 中文學 with 中文學習, NOT a superset + {int64(8), []byte("語文學習")}, // shares trigram 文學習 with 中文學習, NOT a superset + } + + run := func(t *testing.T, idx *Index) { + nl := func(q string) []any { + rs, err := idx.SearchQuery([]byte(q), false, ParserNgram, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(rs) + } + bl := func(q string) []any { + rs, err := idx.SearchQuery([]byte(q), true, ParserNgram, BM25, 100, nil) + require.NoError(t, err) + return resultIDs(rs) + } + + // NL exact ordered phrase — matches ONLY the in-order-adjacent doc. + require.ElementsMatch(t, []any{int64(1)}, nl("全文 索引"), "CJK NL: exact ordered phrase only") + require.ElementsMatch(t, []any{int64(5)}, nl("quick brown"), "English NL: order-sensitive") + require.ElementsMatch(t, []any{int64(6)}, nl("brown quick"), "English NL: reversed is a different phrase") + require.Empty(t, nl("索引 系统"), "not adjacent in doc 3 in this order") + + // Contiguous substring of a longer doc matches — but NOT docs that merely share + // a trigram (中文學校 shares 中文學, 語文學習 shares 文學習): there is NO bag-of-words. + require.ElementsMatch(t, []any{int64(4)}, nl("中文學習"), "CJK substring, not trigram-overlap docs") + require.ElementsMatch(t, []any{int64(4), int64(7)}, nl("中文學"), "中文學 is a substring of docs 4 and 7") + require.ElementsMatch(t, []any{int64(4), int64(7)}, nl("中文")) + + // Boolean: a SINGLE bare CJK compound operand is an EXACT phrase, NOT an OR of its + // ngrams — so +中文學習 matches only the superset doc, never the trigram-overlap docs. + require.ElementsMatch(t, []any{int64(4)}, bl("+中文學習"), "boolean bare CJK operand: exact phrase, not bag") + require.ElementsMatch(t, []any{int64(4)}, bl("中文學習"), "boolean bare CJK operand (SHOULD): exact phrase, not bag") + // Boolean explicit "phrase" is exact-ordered; distinct bare operands OR together. + require.ElementsMatch(t, []any{int64(1)}, bl(`"全文 索引"`), "boolean explicit phrase: exact") + require.ElementsMatch(t, []any{int64(1), int64(2), int64(3)}, bl("全文 索引"), "boolean two bare operands: OR") + } + + seg, err := BuildSegmentFromDocsParser("ex", int32(types.T_int64), docs, ParserNgram) + require.NoError(t, err) + t.Run("build-side", func(t *testing.T) { run(t, NewIndex([]*Segment{seg}, nil)) }) + + // Serialize → deserialize, so the loaded (mmap posRaw) byte positions are exercised too. + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("ex", bytes.NewReader(blob)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + t.Run("loaded-side", func(t *testing.T) { run(t, NewIndex([]*Segment{loaded}, nil)) }) +} diff --git a/pkg/fulltext2/phrase_perf_bench_test.go b/pkg/fulltext2/phrase_perf_bench_test.go new file mode 100644 index 0000000000000..6a9774160f14f --- /dev/null +++ b/pkg/fulltext2/phrase_perf_bench_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Perf follow-up Phase 0 — BASELINE measurement (no engine change) for the two phrase +// memory concerns: +// +// A. pooled phrase-buffer position retention / per-block make() churn +// (segment.go fillBlockPositions), surfaced under concurrent phrase load. +// B. no-LIMIT phrase materializing globalN results (stream.go StreamQuery falls back +// to SearchQuery(globalN)) vs the disjunctive path that streams heap-free. +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// benchPhraseIndex builds a LOADED (serialize→deserialize, so the phrase cursor takes +// the posRaw/fillBlockPositions decode path, not the build-side flat view) single-segment +// index of nDocs docs, each holding `repeats` copies of the phrase "alpha beta" — so every +// term has `repeats` positions per doc (drives the per-doc position arrays) and the phrase +// matches every doc. +func benchPhraseIndex(b *testing.B, nDocs, repeats int) *Index { + b.Helper() + bb := NewBuilder("bench", int32(types.T_int64)) + for d := 0; d < nDocs; d++ { + pos := int32(0) + pk := int64(d) + for r := 0; r < repeats; r++ { + if err := bb.Add("alpha", pos, pk); err != nil { + b.Fatal(err) + } + pos += 6 + if err := bb.Add("beta", pos, pk); err != nil { + b.Fatal(err) + } + pos += 5 + } + } + seg, err := bb.Finish() + if err != nil { + b.Fatal(err) + } + blob, err := seg.Serialize() + if err != nil { + b.Fatal(err) + } + loaded, err := Deserialize(seg.Id, bytes.NewReader(blob)) + if err != nil { + b.Fatal(err) + } + return NewIndex([]*Segment{loaded}, nil) +} + +// BenchmarkPhraseSearchConcurrent — concern A. Repeated NL-phrase top-k under parallel +// load; the alloc/op figure captures the phrase-cursor position churn + pooled-buffer +// behavior. High `repeats` ⇒ long per-doc position lists. +func BenchmarkPhraseSearchConcurrent(b *testing.B) { + idx := benchPhraseIndex(b, 2000, 60) + pat := []byte("alpha beta") + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + res, err := idx.SearchQuery(pat, false /*NL phrase*/, ParserDefault, BM25, 10, nil) + if err != nil { + b.Fatal(err) + } + _ = res + } + }) +} + +// BenchmarkPhraseSparseIntersection isolates doc alignment. Alpha occurs only in even +// docs and beta only in odd docs, so the two large posting lists never share a doc and +// no positional block should be decoded by the lazy phrase cursor. +func BenchmarkPhraseSparseIntersection(b *testing.B) { + bb := NewBuilder("phrase-sparse", int32(types.T_int64)) + for d := 0; d < 50000; d++ { + term := "alpha" + if d&1 != 0 { + term = "beta" + } + if err := bb.Add(term, 0, int64(d)); err != nil { + b.Fatal(err) + } + } + seg, err := bb.Finish() + if err != nil { + b.Fatal(err) + } + blob, err := seg.Serialize() + if err != nil { + b.Fatal(err) + } + loaded, err := Deserialize("phrase-sparse", bytes.NewReader(blob)) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = loaded.dict.Close() }) + idx := NewIndex([]*Segment{loaded}, nil) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + res, err := idx.SearchQuery([]byte("alpha beta"), false, ParserDefault, BM25, 10, nil) + if err != nil { + b.Fatal(err) + } + if len(res) != 0 { + b.Fatalf("expected no matches, got %d", len(res)) + } + } +} + +// BenchmarkPhraseStreamNoLimit — concern B, the materializing fallback: a no-LIMIT NL +// phrase that matches ALL docs, so StreamQuery goes through SearchQuery(globalN). +func BenchmarkPhraseStreamNoLimit(b *testing.B) { + idx := benchPhraseIndex(b, 20000, 3) + pat := []byte("alpha beta") + emit := func(o *vectorindex.SearchOutput) error { + PutColumnBuffer(o.Keys) // recycle like the real consumer + return nil + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := idx.StreamQuery(pat, false /*NL phrase → materializing fallback*/, ParserDefault, BM25, nil, false, emit); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkDisjunctionStreamNoLimit — concern B CONTROL: the SAME pattern + corpus as a +// boolean OR (disjunctive), which streams heap-free (streamWAND, no globalN slice). The +// gap between this and BenchmarkPhraseStreamNoLimit is the materialization cost Phase 2 +// targets. +func BenchmarkDisjunctionStreamNoLimit(b *testing.B) { + idx := benchPhraseIndex(b, 20000, 3) + pat := []byte("alpha beta") + emit := func(o *vectorindex.SearchOutput) error { + PutColumnBuffer(o.Keys) + return nil + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := idx.StreamQuery(pat, true /*boolean OR → heap-free stream*/, ParserDefault, BM25, nil, false, emit); err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/fulltext2/pkcodec.go b/pkg/fulltext2/pkcodec.go new file mode 100644 index 0000000000000..9ee0607b003e6 --- /dev/null +++ b/pkg/fulltext2/pkcodec.go @@ -0,0 +1,236 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// pk codec — the SINGLE SOURCE OF TRUTH for how a primary key of a given types.T is +// stored and moved through fulltext2. It is little-endian and used by the docmap, +// CDC log, delete frames, and the streaming ColumnBuffer. +// +// ADDING A PK TYPE: every switch in this file must gain the new case together — +// fixedPkByteWidth (width, or false for varlena) +// encodePk (concrete value → content bytes) +// decodePk (content bytes → boxed value) +// encodePkLen (has fast cases; falls back to encodePk otherwise) +// AppendColumnBuffer (content bytes → typed vector, box-free) +// TestPkCodecAllTypesRoundTrip loops every supported type through the codec so a +// missing case fails loudly rather than shipping (a missing T_bit case once stalled +// CDC — see the BIT-pk fix). +// +// NOTE: byte-for-byte identical to pkg/bm25/wand's pk codec — both engines store +// any-typed source PKs the same way. Extract to a shared pk-codec package once bm25 +// and fulltext2 live in the same tree (rule of three). + +// fixedPkByteWidth returns the exact stored byte width of a fixed-width pk type, and +// false for variable-width types (varchar/blob/uuid/json…). Used to validate a docmap's +// per-pk length prefix at load and to guard decodePk's fixed-width reads. +func fixedPkByteWidth(pkType int32) (int, bool) { + switch types.T(pkType) { + case types.T_int8, types.T_uint8: + return 1, true + case types.T_int16, types.T_uint16: + return 2, true + case types.T_int32, types.T_uint32, types.T_date: + return 4, true + case types.T_int64, types.T_uint64, types.T_bit, + types.T_datetime, types.T_time, types.T_timestamp, types.T_decimal64: + return 8, true + case types.T_decimal128: + return 16, true + default: + return 0, false + } +} + +func encodePk(pkType int32, v any) ([]byte, error) { + switch types.T(pkType) { + case types.T_int64: + return packUint64(uint64(v.(int64))), nil + case types.T_uint64: + return packUint64(v.(uint64)), nil + case types.T_bit: + // BIT is backed by uint64 (types.MustFixedColWithTypeCheck[uint64]); encode + // like T_uint64. build_ddl accepts a BIT primary key, so the codec must too — + // otherwise the first CDC flush errors and stalls index maintenance. + return packUint64(v.(uint64)), nil + case types.T_int32: + return packUint32(uint32(v.(int32))), nil + case types.T_uint32: + return packUint32(v.(uint32)), nil + case types.T_int16: + return packUint16(uint16(v.(int16))), nil + case types.T_uint16: + return packUint16(v.(uint16)), nil + case types.T_int8: + return []byte{byte(v.(int8))}, nil + case types.T_uint8: + return []byte{v.(uint8)}, nil + case types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json: + return asBytes(v), nil + case types.T_uuid: + switch x := v.(type) { + case string: + return []byte(x), nil + case types.Uuid: + return []byte(x.String()), nil + case []byte: + return append([]byte(nil), x...), nil + default: + return nil, moerr.NewInternalErrorNoCtxf("fulltext2: uuid pk unexpected go type %T", v) + } + case types.T_date: + return packUint32(uint32(int32(v.(types.Date)))), nil + case types.T_datetime: + return packUint64(uint64(int64(v.(types.Datetime)))), nil + case types.T_time: + return packUint64(uint64(int64(v.(types.Time)))), nil + case types.T_timestamp: + return packUint64(uint64(int64(v.(types.Timestamp)))), nil + case types.T_decimal64: + return packUint64(uint64(v.(types.Decimal64))), nil + case types.T_decimal128: + d := v.(types.Decimal128) + b := make([]byte, 16) + binary.LittleEndian.PutUint64(b[0:8], d.B0_63) + binary.LittleEndian.PutUint64(b[8:16], d.B64_127) + return b, nil + default: + return nil, moerr.NewInternalErrorNoCtxf("fulltext2: unsupported pk type %d", pkType) + } +} + +func decodePk(pkType int32, b []byte) (any, error) { + // Fixed-width types index into b with LittleEndian.Uint{16,32,64}, which panic on + // a short slice. A corrupt docmap can store a wrong length prefix, so verify the + // slice is wide enough and return a clean error instead of crashing the query + // goroutine. (decodeDocmap also width-validates at load; this is belt-and-suspenders.) + if w, fixed := fixedPkByteWidth(pkType); fixed && len(b) < w { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2: corrupt pk: type %d needs %d bytes, got %d", pkType, w, len(b)) + } + switch types.T(pkType) { + case types.T_int64: + return int64(binary.LittleEndian.Uint64(b)), nil + case types.T_uint64: + return binary.LittleEndian.Uint64(b), nil + case types.T_bit: + return binary.LittleEndian.Uint64(b), nil + case types.T_int32: + return int32(binary.LittleEndian.Uint32(b)), nil + case types.T_uint32: + return binary.LittleEndian.Uint32(b), nil + case types.T_int16: + return int16(binary.LittleEndian.Uint16(b)), nil + case types.T_uint16: + return binary.LittleEndian.Uint16(b), nil + case types.T_int8: + return int8(b[0]), nil + case types.T_uint8: + return b[0], nil + case types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json: + return append([]byte(nil), b...), nil + case types.T_uuid: + u, err := types.ParseUuid(string(b)) + if err != nil { + return nil, err + } + return u, nil + case types.T_date: + return types.Date(int32(binary.LittleEndian.Uint32(b))), nil + case types.T_datetime: + return types.Datetime(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_time: + return types.Time(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_timestamp: + return types.Timestamp(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_decimal64: + return types.Decimal64(binary.LittleEndian.Uint64(b)), nil + case types.T_decimal128: + return types.Decimal128{ + B0_63: binary.LittleEndian.Uint64(b[0:8]), + B64_127: binary.LittleEndian.Uint64(b[8:16]), + }, nil + default: + return nil, moerr.NewInternalErrorNoCtxf("fulltext2: unsupported pk type %d", pkType) + } +} + +// encodePkLen writes a length-prefixed pk ([u32 len][content]) into the docmap buffer, +// deriving the content from encodePk (one codec switch, not a duplicate here). +func (w *leBuf) encodePkLen(pkType int32, v any) error { + pkb, err := encodePk(pkType, v) + if err != nil { + return err + } + w.u32(uint32(len(pkb))) + w.b.Write(pkb) + return nil +} + +// appendAnyPk re-encodes a boxed pk value into a ColumnBuffer's Data (used by the +// build-side / materialized-fallback streaming paths): fixed-width types contribute +// their width bytes, varlena types a [u32 len][content] entry. No NEW box. +func appendAnyPk(k *vectorindex.ColumnBuffer, fixedW int, v any) { + content, err := encodePk(int32(k.Type), v) + if err != nil { + return // unsupported pk type never reaches streaming (validated at build) + } + if fixedW > 0 { + k.Data = append(k.Data, content...) // exactly fixedW bytes + } else { + var lp [4]byte + binary.LittleEndian.PutUint32(lp[:], uint32(len(content))) + k.Data = append(k.Data, lp[:]...) + k.Data = append(k.Data, content...) + } + k.N++ +} + +func packUint64(v uint64) []byte { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v) + return b +} + +func packUint32(v uint32) []byte { + b := make([]byte, 4) + binary.LittleEndian.PutUint32(b, v) + return b +} + +func packUint16(v uint16) []byte { + b := make([]byte, 2) + binary.LittleEndian.PutUint16(b, v) + return b +} + +func asBytes(v any) []byte { + switch x := v.(type) { + case []byte: + return x + case string: + return []byte(x) + default: + return nil + } +} diff --git a/pkg/fulltext2/pkcodec_review_test.go b/pkg/fulltext2/pkcodec_review_test.go new file mode 100644 index 0000000000000..0db316c8cf74f --- /dev/null +++ b/pkg/fulltext2/pkcodec_review_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestBitPkRoundTrip pins the self-review blocker: a BIT primary key is accepted by +// build_ddl but the segment pk codec had no T_bit case, so encodePk returned +// "unsupported pk type" and stalled fulltext2 CDC on the first flush. BIT is backed by +// uint64, so it must round-trip through encode -> Serialize -> Deserialize -> pk(ord) +// -> query exactly like T_uint64. +func TestBitPkRoundTrip(t *testing.T) { + // encode/decode round-trip for the raw codec. + for _, v := range []uint64{0, 1, 255, 1 << 40, 1<<64 - 1} { + enc, err := encodePk(int32(types.T_bit), v) + require.NoError(t, err, "encodePk T_bit must not error (was the CDC-stall bug)") + dec, err := decodePk(int32(types.T_bit), enc) + require.NoError(t, err) + require.Equal(t, v, dec, "T_bit pk must round-trip") + } + + // full segment round-trip: build with BIT pks, serialize, reload, query back. + b := NewBuilder("bit", int32(types.T_bit)) + feed(t, b, uint64(10), "alpha", "beta") + feed(t, b, uint64(20), "alpha") + seg, err := b.Finish() + require.NoError(t, err) + + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("bit", bytes.NewReader(blob)) + require.NoError(t, err) + + idx := NewIndex([]*Segment{loaded}, nil) + require.Equal(t, int64(2), idx.globalN) + res := idx.SearchPhrase(phr("alpha"), BM25, 10, nil) + require.Len(t, res, 2, "both BIT-pk docs must be queryable after reload") + pks := map[uint64]bool{} + for _, r := range res { + pks[r.Pk.(uint64)] = true + } + require.True(t, pks[10] && pks[20], "both BIT pk values must survive the round-trip") +} + +// TestDecodeDocmapRejectsShortFixedPk pins the corrupt-blob guard: a fixed-width pk whose +// stored length prefix is smaller than the type width must be rejected at load (clean +// error) rather than passing decodeDocmap and later panicking in pk(ord) -> +// LittleEndian.Uint64 on a short slice. +func TestDecodeDocmapRejectsShortFixedPk(t *testing.T) { + // Hand-craft a minimal docmap: [pkType u32][n u64][ per pk: len u32 + bytes ][ docLen n*u32 ]. + // One doc, T_int64 pk, but the stored pk length is 4 (should be 8) -> corrupt. + var buf bytes.Buffer + le := func(v uint32) { _ = binary.Write(&buf, binary.LittleEndian, v) } + le64 := func(v uint64) { _ = binary.Write(&buf, binary.LittleEndian, v) } + le(uint32(types.T_int64)) // pkType + le64(1) // n = 1 doc + le(4) // pk length prefix = 4 (WRONG: T_int64 needs 8) + buf.Write([]byte{1, 2, 3, 4}) + le(7) // docLen[0] + + s := &Segment{} + err := s.decodeDocmap(buf.Bytes()) + require.Error(t, err, "a fixed-width pk with a short stored length must be rejected at load") + require.Contains(t, err.Error(), "width", "error should name the width mismatch") +} diff --git a/pkg/fulltext2/pkcodec_test.go b/pkg/fulltext2/pkcodec_test.go new file mode 100644 index 0000000000000..f05b05e59843e --- /dev/null +++ b/pkg/fulltext2/pkcodec_test.go @@ -0,0 +1,74 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestPkCodecAllTypesRoundTrip is the single-source-of-truth guardrail: EVERY supported +// pk type must survive encodePk → decodePk unchanged. A pk type added to one codec switch +// but forgotten in another (the class of bug that once stalled CDC on a BIT pk) fails here +// instead of in production. Add every new pk type to this table. +func TestPkCodecAllTypesRoundTrip(t *testing.T) { + u1, _ := types.ParseUuid("12345678-1234-1234-1234-1234567890ab") + cases := []struct { + typ types.T + val any + }{ + {types.T_int64, int64(-1 << 50)}, + {types.T_uint64, uint64(1<<64 - 1)}, + {types.T_bit, uint64(255)}, + {types.T_int32, int32(-123456)}, + {types.T_uint32, uint32(4000000000)}, + {types.T_int16, int16(-30000)}, + {types.T_uint16, uint16(60000)}, + {types.T_int8, int8(-128)}, + {types.T_uint8, uint8(200)}, + {types.T_date, types.Date(19000)}, + {types.T_datetime, types.Datetime(-42)}, + {types.T_time, types.Time(987654321)}, + {types.T_timestamp, types.Timestamp(1234567890)}, + {types.T_decimal64, types.Decimal64(999999)}, + {types.T_decimal128, types.Decimal128{B0_63: 7, B64_127: 11}}, + {types.T_varchar, []byte("hello pk")}, + {types.T_char, []byte("c")}, + {types.T_text, []byte("")}, + {types.T_blob, []byte{0, 1, 2, 255}}, + {types.T_json, []byte(`{"k":1}`)}, + {types.T_uuid, u1}, + } + for _, c := range cases { + b, err := encodePk(int32(c.typ), c.val) + require.NoErrorf(t, err, "encodePk %v", c.typ) + got, err := decodePk(int32(c.typ), b) + require.NoErrorf(t, err, "decodePk %v", c.typ) + // byte-slice pks: compare as strings so an empty pk's nil-vs-[]byte{} (which + // are semantically identical, and identical once appended to a vector) matches. + if wantB, ok := c.val.([]byte); ok { + require.Equalf(t, string(wantB), string(got.([]byte)), "round-trip %v", c.typ) + } else { + require.Equalf(t, c.val, got, "round-trip %v", c.typ) + } + + // fixedPkByteWidth must agree with the encoded length for fixed types. + if w, fixed := fixedPkByteWidth(int32(c.typ)); fixed { + require.Equalf(t, w, len(b), "fixed width mismatch for %v", c.typ) + } + } +} diff --git a/pkg/fulltext2/plugin/compile/compile.go b/pkg/fulltext2/plugin/compile/compile.go new file mode 100644 index 0000000000000..22544ca757912 --- /dev/null +++ b/pkg/fulltext2/plugin/compile/compile.go @@ -0,0 +1,479 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compile holds the fulltext2 index's compile-layer (DDL) hooks. +package compile + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/fulltext2" + fulltext2runtime "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/runtime" + compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" +) + +// parserFromParams extracts the "parser" field from an index's IndexAlgoParams +// JSON (empty → default parser). +func parserFromParams(params string) string { + if len(params) == 0 { + return "" + } + var p struct { + Parser string `json:"parser"` + } + if err := json.Unmarshal([]byte(params), &p); err != nil { + return "" + } + return p.Parser +} + +var _ compileplugin.Hooks = Hooks{} + +type Hooks struct{} + +// HandleCreateIndex creates the storage + metadata hidden tables and builds the +// tag=0 base segment from the current source rows (Go-side: SELECT pk,col → +// tokenize → Segment → persist). Step-4 first cut: whole-table read + one base +// segment; CDC-async incremental maintenance follows. +func (Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef) error { + // Gate CREATE FULLTEXT2 INDEX behind experimental_fulltext2_index. Frontend-only: + // re-checking here (in addition to the framework gate in pkg/sql/compile/util.go + // via ExperimentalFlag()) catches a flag toggled off since the original CREATE; + // the background CDC/reindex context may not surface the user's session value, so + // skip the check there. Mirrors bm25's HandleCreateIndex gate. + if ctx.IsFrontend() { + if ok, err := ctx.IsExperimentalEnabled(fulltext2runtime.Fulltext2IndexFlag); err != nil { + return err + } else if !ok { + return moerr.NewInternalErrorNoCtx("experimental_fulltext2_index is not enabled") + } + } + + storeDef, ok := indexDefs[catalog.FullText2Index_TblType_Storage] + if !ok { + return moerr.NewInternalErrorNoCtx("fulltext2 storage index definition not found") + } + metaDef, ok := indexDefs[catalog.FullText2Index_TblType_Metadata] + if !ok { + return moerr.NewInternalErrorNoCtx("fulltext2 metadata index definition not found") + } + + // 1. create the hidden tables. + if info := ctx.IndexInfo(); info != nil { + for _, table := range info.GetIndexTables() { + if err := ctx.BuildIndexTable(table); err != nil { + return err + } + } + } + + origTable := ctx.OriginalTableDef() + db := ctx.QryDatabase() + // CCPR: index data syncs via CCPR on a publication-subscribed table. + if ctx.IsCCPRTaskTransaction() && ctx.IsTableFromPublication(origTable) { + return nil + } + + // Fresh CREATE has no prior tail; build the base + register CDC. + return buildAndRegisterCDC(ctx, storeDef, metaDef, origTable, db, false) +} + +// buildAndRegisterCDC (re)builds the tag=0 base from source and registers the +// ISCP CDC task from now (startFromNow=true, no InitSQL): the inline build covers +// the current rows, so subsequent INSERT/UPDATE/DELETE flow into the tag=1 tail +// via RunFulltext2. The CDC task is dropped+recreated so a REINDEX re-entry does +// not replay history over the fresh base. On REBUILD (clearTail) the prior tag=1 +// tail is discarded first — the fresh base already reflects every committed row. +func buildAndRegisterCDC(ctx compileplugin.CompileContext, storeDef, metaDef *plan.IndexDef, origTable *plan.TableDef, db string, clearTail bool) error { + if clearTail { + cfg := fulltext2.TableConfig{DbName: db, IndexTable: storeDef.IndexTableName, MetadataTable: metaDef.IndexTableName} + // REBUILD: clear the tail AND the prior tag=0 base(s) here. The create TVF also + // clears the bases, but it SKIPS that when the rebuild sees zero source rows + // (empty/all-deleted table) — so without this a REBUILD over an emptied table + // would keep serving the stale old base. Evict the local cache too, since the + // create TVF's post-build eviction is likewise skipped on the zero-doc path. + clearSqls := append(fulltext2.DeleteTailSqls(cfg), fulltext2.DeleteAllBasesSqls(cfg)...) + for _, s := range clearSqls { + if err := ctx.RunSql(s); err != nil { + return err + } + } + fulltext2.NewFulltext2Search(cfg).OnCacheInvalidated(string(fulltext2.LoadMissRebuild)) + cache.Cache.Remove(storeDef.IndexTableName) + } + // buildFromSource clears the prior tag=0 bases (idempotent) and rebuilds them. + if err := buildFromSource(ctx, storeDef, metaDef, origTable, db); err != nil { + return err + } + sinkerType := ctx.SinkerTypeFromAlgo(catalog.MoIndexFullText2Algo.ToString()) + if err := ctx.DropIndexCdcTask(origTable, db, origTable.Name, storeDef.IndexName); err != nil { + return err + } + if err := ctx.CreateIndexCdcTask(db, origTable.Name, origTable.TblId, storeDef.IndexName, sinkerType, true, "", origTable); err != nil { + return err + } + return registerFulltext2Idxcron(ctx, storeDef, db, origTable) +} + +// DefaultMaxIndexCapacity caps each tag=0 sub-index's doc count when the +// WITH max_index_capacity option is omitted. +const DefaultMaxIndexCapacity = int64(1000000) + +// DefaultMaxPostingsCapacity caps each tag=0 sub-index's posting (term-occurrence) +// count when WITH max_postings_capacity is omitted (~512 MB peak build heap). Bounds +// build memory for long-document corpora that a doc-count cap alone would not. +const DefaultMaxPostingsCapacity = int64(8_000_000) + +// buildFromSource builds the index straight from the SOURCE table in one +// statement: SELECT f.* FROM src CROSS APPLY fulltext2_create(params, cfg, pk, +// cols…). The create TVF tokenizes each row in execution (datalink → plain text, +// json → values, ngram/gojieba parser), Add's the tokens to a builder, and at +// end() splits into capacity-bounded tag=0 bases and persists the chunk rows. No +// segment assembly happens in this (compile) layer. +func buildFromSource(ctx compileplugin.CompileContext, storeDef, metaDef *plan.IndexDef, origTable *plan.TableDef, db string) error { + if len(storeDef.Parts) == 0 { + return moerr.NewInternalErrorNoCtx("fulltext2 index has no indexed column") + } + capacity, err := resolveFulltext2Capacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + postingCap, err := resolveFulltext2PostingCapacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + sql, err := genFulltext2BuildFromSourceSQL(origTable, storeDef, metaDef, db, capacity, postingCap) + if err != nil { + return err + } + return ctx.RunSql(sql) +} + +// resolveFulltext2Capacity reads max_index_capacity from the index's algo_params, +// defaulting when the WITH option was omitted. +func resolveFulltext2Capacity(algoParams string) (int64, error) { + if algoParams == "" { + return DefaultMaxIndexCapacity, nil // no WITH options + } + flat, err := catalog.IndexParamsStringToMap(algoParams) + if err != nil { + return 0, err + } + if v, ok := flat[catalog.IndexAlgoParamMaxIndexCapacity]; ok && v != "" { + n, perr := strconv.ParseInt(v, 10, 64) + if perr != nil { + return 0, perr + } + if n > 0 { + return n, nil + } + } + return DefaultMaxIndexCapacity, nil +} + +// resolveFulltext2PostingCapacity reads max_postings_capacity from the index's +// algo_params, defaulting when the WITH option was omitted. Mirrors the doc-cap +// resolver; both caps are pinned at CREATE and read back on every build path. +func resolveFulltext2PostingCapacity(algoParams string) (int64, error) { + if algoParams == "" { + return DefaultMaxPostingsCapacity, nil + } + flat, err := catalog.IndexParamsStringToMap(algoParams) + if err != nil { + return 0, err + } + if v, ok := flat[catalog.IndexAlgoParamMaxPostingsCapacity]; ok && v != "" { + n, perr := strconv.ParseInt(v, 10, 64) + if perr != nil { + return 0, perr + } + if n > 0 { + return n, nil + } + } + return DefaultMaxPostingsCapacity, nil +} + +// resolveFulltext2PositionFree reads position_free from the index's algo_params +// (absence ⇒ false, i.e. a positional/phrase-capable index). +func resolveFulltext2PositionFree(algoParams string) (bool, error) { + if algoParams == "" { + return false, nil + } + flat, err := catalog.IndexParamsStringToMap(algoParams) + if err != nil { + return false, err + } + return flat[catalog.IndexAlgoParamPositionFree] == "true", nil +} + +// genFulltext2BuildFromSourceSQL emits the CROSS APPLY fulltext2_create build SQL. +func genFulltext2BuildFromSourceSQL(origTable *plan.TableDef, storeDef, metaDef *plan.IndexDef, db string, capacity, postingCap int64) (string, error) { + const srcAlias = "src" + positionFree, err := resolveFulltext2PositionFree(storeDef.IndexAlgoParams) + if err != nil { + return "", err + } + cfg := fulltext2.TableConfig{ + DbName: db, + SrcTable: origTable.Name, + IndexTable: storeDef.IndexTableName, + MetadataTable: metaDef.IndexTableName, + PKey: origTable.Pkey.PkeyColName, + Parser: parserFromParams(storeDef.IndexAlgoParams), + Capacity: capacity, + PostingCapacity: postingCap, + PositionFree: positionFree, + FromSource: true, + } + cols := make([]string, 0, len(storeDef.Parts)) + for _, p := range storeDef.Parts { + cols = append(cols, sqlquote.QualifiedIdent(srcAlias, p)) + } + // INCLUDE columns: their actual values are stored in the segment docmap (prefilter + + // coverage). They are passed as the TRAILING fulltext2_create args (after the text + // columns); cfg.IncludeTypes tells the TVF how many trailing args are INCLUDE values and + // their types. Resolve names from the first-class field, falling back to algo_params. + incNames := fulltext2IncludeColumns(storeDef) + if len(incNames) > 0 { + incTypes := make([]int32, len(incNames)) + for i, name := range incNames { + ci, ok := origTable.Name2ColIndex[name] + if !ok { + return "", moerr.NewInternalErrorNoCtx("fulltext2 build: INCLUDE column " + name + " not found on source table") + } + incTypes[i] = origTable.Cols[ci].Typ.Id + cols = append(cols, sqlquote.QualifiedIdent(srcAlias, name)) + } + cfg.IncludeTypes = incTypes + } + cfgbytes, err := json.Marshal(cfg) + if err != nil { + return "", err + } + sql := fmt.Sprintf("SELECT f.* FROM %s AS %s CROSS APPLY fulltext2_create(%s, %s, %s, %s) AS f", + sqlquote.QualifiedIdent(db, origTable.Name), + sqlquote.Ident(srcAlias), + sqlquote.String(storeDef.IndexAlgoParams), + sqlquote.String(string(cfgbytes)), + sqlquote.QualifiedIdent(srcAlias, origTable.Pkey.PkeyColName), + strings.Join(cols, ", ")) + return sql, nil +} + +// fulltext2IncludeColumns returns the index's INCLUDE column names, preferring the +// first-class IndexDef field and falling back to the persisted algo_param (reload-safe). +func fulltext2IncludeColumns(def *plan.IndexDef) []string { + if len(def.IncludedColumns) > 0 { + return def.IncludedColumns + } + m, err := catalog.IndexParamsStringToMap(def.IndexAlgoParams) + if err != nil { + return nil + } + raw := m[catalog.IncludedColumns] + if raw == "" { + return nil + } + names, err := catalog.ParseIncludeColumnsValue(raw) + if err != nil { + return nil + } + return names +} + +// HandleReindex runs ALTER … ALTER REINDEX: merge=true compacts the tag=1 tail +// into the tag=0 base (dropping dead docs) via fulltext2_compact; merge=false +// (REBUILD) discards the tail and rebuilds the base from source. Mirrors bm25. +func (Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, _, merge bool) error { + storeDef, ok := indexDefs[catalog.FullText2Index_TblType_Storage] + if !ok { + return moerr.NewInternalErrorNoCtx("fulltext2 storage index definition not found") + } + metaDef, ok := indexDefs[catalog.FullText2Index_TblType_Metadata] + if !ok { + return moerr.NewInternalErrorNoCtx("fulltext2 metadata index definition not found") + } + if merge { + return handleMergeCompact(ctx, storeDef, metaDef) + } + return buildAndRegisterCDC(ctx, storeDef, metaDef, ctx.OriginalTableDef(), ctx.QryDatabase(), true /* clear tail */) +} + +// handleMergeCompact folds the tag=1 CdcTail into the tag=0 base and drops dead +// docs, via the fulltext2_compact TVF (execution-side, one txn) — no re-tokenize. +// Capacity comes from the PERSISTED algo_params (pinned at CREATE) so a manual +// MERGE never depends on the triggering session. Mirrors bm25's handleMergeCompact. +func handleMergeCompact(ctx compileplugin.CompileContext, storeDef, metaDef *plan.IndexDef) error { + capacity, err := resolveFulltext2Capacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + postingCap, err := resolveFulltext2PostingCapacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + positionFree, err := resolveFulltext2PositionFree(storeDef.IndexAlgoParams) + if err != nil { + return err + } + sql := fmt.Sprintf("SELECT * FROM fulltext2_compact(%s, %s, %s, %s, %s, %s) AS f", + sqlquote.String(ctx.QryDatabase()), + sqlquote.String(storeDef.IndexTableName), + sqlquote.String(metaDef.IndexTableName), + sqlquote.String(strconv.FormatInt(capacity, 10)), + sqlquote.String(strconv.FormatBool(positionFree)), + sqlquote.String(strconv.FormatInt(postingCap, 10))) + return ctx.RunSql(sql) +} + +// RestoreInitSQL — the clone copies the storage+metadata rows; register CDC from +// the post-clone TS. +func (Hooks) RestoreInitSQL(_ compileplugin.CompileContext, _ map[string]*plan.IndexDef) (bool, string, error) { + return true, "SELECT 1", nil +} + +// ValidateReindexParams — fulltext2 honors position_free on a rebuild (its only +// reindex-time param): POSITION_FREE=TRUE sets it, =FALSE clears it (→ a positional +// rebuild that re-derives positions from source), absence keeps the current setting. +// Other reindex options are ignored. The REBUILD (buildFromSource) then reads the +// merged algo_params. NOTE: changing position_free needs REBUILD (no MERGE) — a MERGE +// cannot re-derive positions when turning position_free OFF. +func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { + pf, hasPF := alter.Params[catalog.IndexAlgoParamPositionFree] + pc, hasPC := alter.Params[catalog.IndexAlgoParamMaxPostingsCapacity] + if !hasPF && !hasPC { + return old, nil // nothing fulltext2 honors on a reindex + } + merged := make(map[string]string, len(old)+1) + for k, v := range old { + merged[k] = v + } + if hasPF { + // A MERGE folds the tag=1 tail into the existing base IN PLACE — it re-serializes the + // current segments and cannot change their positional format: turning POSITION_FREE OFF has + // no positions to re-derive (the base dropped them), and CompactSegments deletes the old + // rows before rebuilding, so a `MERGE POSITION_FREE=FALSE` would commit an EMPTY base even + // though the source rows are unchanged — data loss. Changing POSITION_FREE requires a full + // REBUILD (re-derives from the source table). Reject the combination BEFORE the params are + // persisted, so a MERGE can never mint a format-mismatched base. + curPF := old[catalog.IndexAlgoParamPositionFree] == "true" + newPF := pf == "true" + if alter.Merge && newPF != curPF { + return nil, moerr.NewInvalidInputNoCtx( + "fulltext2: changing POSITION_FREE requires a REBUILD (ALTER … REINDEX without MERGE); " + + "a MERGE compacts the tail into the existing base and cannot re-derive positions") + } + if pf == "true" { + // Only gojieba is meaningful position-free (ngram/json trigrams are noise as + // bag-of-words). Mirrors the CREATE-time check. + if old["parser"] != "gojieba" { + return nil, moerr.NewInvalidInputNoCtxf( + "fulltext2 POSITION_FREE requires the gojieba parser (this index uses %q); ngram/json need positions", old["parser"]) + } + merged[catalog.IndexAlgoParamPositionFree] = "true" + } else { + delete(merged, catalog.IndexAlgoParamPositionFree) // false ⇒ positional (param absent) + } + } + // max_postings_capacity override: the grammar already rejected <=0, so a present + // value is valid; the subsequent REBUILD/MERGE reads it back from the merged + // algo_params and seals segments at the new posting bound. (max_index_capacity is + // NOT re-honored on a reindex — a pre-existing fulltext2 limitation, unchanged.) + if hasPC { + merged[catalog.IndexAlgoParamMaxPostingsCapacity] = pc + } + return merged, nil +} + +// HandleDropIndex clears any pooled immutable generations as well as the +// generic cache entry. The cache entry may already be absent after TTL +// eviction, while its base pool is intentionally retained for a later warm +// load; DROP must not leave that storage mapping behind. +func (Hooks) HandleDropIndex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef) error { + if ctx == nil { + return nil + } + storeDef, ok := indexDefs[catalog.FullText2Index_TblType_Storage] + if !ok || storeDef == nil || storeDef.IndexTableName == "" { + return nil + } + cfg := fulltext2.TableConfig{DbName: ctx.QryDatabase(), IndexTable: storeDef.IndexTableName} + fulltext2.NewFulltext2Search(cfg).OnCacheInvalidated("") + cache.Cache.Remove(storeDef.IndexTableName) + return nil +} + +// HiddenTableDropPriority: fulltext2's storage and metadata hidden tables have no +// inter-dependency that constrains drop order, so all types drop at the same +// priority (mirrors classic fulltext). +func (Hooks) HiddenTableDropPriority(_ string) int { + return 0 +} + +// actionFulltext2Reindex is the idxcron action key for fulltext2's scheduled +// compaction; must match the runtime plugin's SyncDescriptor.IdxcronAction. +const actionFulltext2Reindex = "fulltext2_reindex" + +// IdxcronMetadata — the scheduled compaction reads its cadence knobs +// (max_index_capacity, tail thresholds) from the PERSISTED algo_params, so the +// only thing worth pinning in the blob is lower_case_table_names: its VALUE +// matters because the background reindex must fold identifiers in the rebuild +// SQL exactly as the user built under (a mismatch resolves the wrong +// table/column). This mirrors the vector-index plugins' capture list. sql_mode +// is deliberately NOT captured — a background reindex is a system op that runs +// permissive, defaulted at the reindex hook via +// sqlexec.Metadata.ResolveVariableWithSessionDefaults (capturing the user's +// possibly-strict sql_mode would be wrong). +// +// A background re-entry (not frontend) returns nil so the existing task row +// persists. If lower_case_table_names can't be resolved in this context we still +// register with an empty blob (fulltext2 needs only a non-nil blob to register; +// it reads everything else from algo_params). Mirrors bm25. +func (Hooks) IdxcronMetadata(ctx compileplugin.CompileContext) ([]byte, error) { + if !ctx.IsFrontend() { + return nil, nil + } + blob, err := compileplugin.BuildIdxcronMetadata(ctx, compileplugin.IdxcronVarSpec{ + Capture: []string{"lower_case_table_names"}, + }) + if err != nil { + return nil, err + } + if len(blob) == 0 { + return []byte("{}"), nil + } + return blob, nil +} + +// registerFulltext2Idxcron registers the scheduled-compaction task (skipped on a +// background re-entry so the existing task row persists). Mirrors bm25. +func registerFulltext2Idxcron(ctx compileplugin.CompileContext, storeDef *plan.IndexDef, db string, origTable *plan.TableDef) error { + metadata, err := Hooks{}.IdxcronMetadata(ctx) + if err != nil { + return err + } + if len(metadata) == 0 { + return nil + } + return ctx.RegisterIdxcronUpdate(origTable.TblId, db, origTable.Name, storeDef.IndexName, actionFulltext2Reindex, metadata) +} diff --git a/pkg/fulltext2/plugin/compile/compile_test.go b/pkg/fulltext2/plugin/compile/compile_test.go new file mode 100644 index 0000000000000..5907d3ce775bf --- /dev/null +++ b/pkg/fulltext2/plugin/compile/compile_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compile + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" + "github.com/stretchr/testify/require" +) + +// TestValidateReindexParams covers the fulltext2 ALTER REINDEX param merge — in +// particular that max_postings_capacity is APPLIED (not the silent no-op it was +// before), so `ALTER … REINDEX … max_postings_capacity=N` actually re-bounds the +// rebuild. +func TestValidateReindexParams(t *testing.T) { + up := func(m map[string]string) compileplugin.ReindexParamUpdate { + return compileplugin.ReindexParamUpdate{Params: m} + } + + // No honored key → old returned unchanged (a bare reindex / MERGE). + old := map[string]string{"parser": "ngram", catalog.IndexAlgoParamMaxIndexCapacity: "1000000"} + got, err := Hooks{}.ValidateReindexParams(old, up(nil)) + require.NoError(t, err) + require.Equal(t, old, got) + + // max_postings_capacity present → merged into the rebuild's algo_params. + got, err = Hooks{}.ValidateReindexParams(old, up(map[string]string{ + catalog.IndexAlgoParamMaxPostingsCapacity: "4000000", + })) + require.NoError(t, err) + require.Equal(t, "4000000", got[catalog.IndexAlgoParamMaxPostingsCapacity]) + require.Equal(t, "ngram", got["parser"]) // untouched + require.Equal(t, "1000000", got[catalog.IndexAlgoParamMaxIndexCapacity]) // untouched + require.NotContains(t, old, catalog.IndexAlgoParamMaxPostingsCapacity) // old not mutated + + // position_free + max_postings_capacity together (gojieba): both applied. + oldG := map[string]string{"parser": "gojieba"} + got, err = Hooks{}.ValidateReindexParams(oldG, up(map[string]string{ + catalog.IndexAlgoParamPositionFree: "true", + catalog.IndexAlgoParamMaxPostingsCapacity: "2000000", + })) + require.NoError(t, err) + require.Equal(t, "true", got[catalog.IndexAlgoParamPositionFree]) + require.Equal(t, "2000000", got[catalog.IndexAlgoParamMaxPostingsCapacity]) + + // position_free=true on a non-gojieba parser is still rejected, even when a + // posting-capacity override rides along. + _, err = Hooks{}.ValidateReindexParams(old, up(map[string]string{ + catalog.IndexAlgoParamPositionFree: "true", + catalog.IndexAlgoParamMaxPostingsCapacity: "2000000", + })) + require.Error(t, err) + + // max_index_capacity alone is NOT honored on a reindex (pre-existing fulltext2 + // limitation): old returned unchanged. + got, err = Hooks{}.ValidateReindexParams(old, up(map[string]string{ + catalog.IndexAlgoParamMaxIndexCapacity: "5", + })) + require.NoError(t, err) + require.Equal(t, old, got) +} + +// TestValidateReindexParamsMergePositionFree covers the data-loss guard: a MERGE compacts the tail +// into the EXISTING base in place and cannot change the positional format (turning POSITION_FREE +// OFF has no positions to re-derive; CompactSegments would commit an empty base). So a +// `MERGE POSITION_FREE=…` that flips the format is rejected BEFORE the params are persisted, while +// a REBUILD (Merge=false) with the same change is allowed and a format-preserving MERGE passes. +func TestValidateReindexParamsMergePositionFree(t *testing.T) { + upM := func(m map[string]string, merge bool) compileplugin.ReindexParamUpdate { + return compileplugin.ReindexParamUpdate{Params: m, Merge: merge} + } + posFree := map[string]string{"parser": "gojieba", catalog.IndexAlgoParamPositionFree: "true"} + positional := map[string]string{"parser": "gojieba"} // position_free absent ⇒ positional + + // MERGE turning position_free OFF (the reported data-loss case) → rejected. + _, err := Hooks{}.ValidateReindexParams(posFree, upM(map[string]string{ + catalog.IndexAlgoParamPositionFree: "false"}, true)) + require.ErrorContains(t, err, "REBUILD") + + // MERGE turning position_free ON (also a format change) → rejected. + _, err = Hooks{}.ValidateReindexParams(positional, upM(map[string]string{ + catalog.IndexAlgoParamPositionFree: "true"}, true)) + require.ErrorContains(t, err, "REBUILD") + + // REBUILD (Merge=false) with the same OFF change → allowed (re-derives from source). + got, err := Hooks{}.ValidateReindexParams(posFree, upM(map[string]string{ + catalog.IndexAlgoParamPositionFree: "false"}, false)) + require.NoError(t, err) + require.NotContains(t, got, catalog.IndexAlgoParamPositionFree) // false ⇒ positional (absent) + + // MERGE that does NOT change the format (re-asserts the current true) → allowed. + got, err = Hooks{}.ValidateReindexParams(posFree, upM(map[string]string{ + catalog.IndexAlgoParamPositionFree: "true"}, true)) + require.NoError(t, err) + require.Equal(t, "true", got[catalog.IndexAlgoParamPositionFree]) + + // bare MERGE (no position_free param) → allowed. + got, err = Hooks{}.ValidateReindexParams(posFree, upM(nil, true)) + require.NoError(t, err) + require.Equal(t, posFree, got) +} diff --git a/pkg/fulltext2/plugin/compile/parser_coverage_test.go b/pkg/fulltext2/plugin/compile/parser_coverage_test.go new file mode 100644 index 0000000000000..f361f8c8e1b56 --- /dev/null +++ b/pkg/fulltext2/plugin/compile/parser_coverage_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compile + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParserFromParams(t *testing.T) { + require.Empty(t, parserFromParams("")) + require.Empty(t, parserFromParams("{")) + require.Equal(t, "gojieba", parserFromParams(`{"parser":"gojieba"}`)) +} diff --git a/pkg/fulltext2/plugin/idxcron/idxcron.go b/pkg/fulltext2/plugin/idxcron/idxcron.go new file mode 100644 index 0000000000000..31d4b985e07a7 --- /dev/null +++ b/pkg/fulltext2/plugin/idxcron/idxcron.go @@ -0,0 +1,164 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package idxcron is the fulltext2 index's scheduled-compaction hook (mirrors +// bm25's). fulltext2 participates in scheduled reindex +// (SyncDescriptor.IdxcronAction = "fulltext2_reindex"); the cron executor runs +// `ALTER … REINDEX … FULLTEXT2 [MERGE] FORCE_SYNC` when Updatable fires — MERGE +// folds the tag=1 CdcTail into the tag=0 base, or a full REBUILD once the dead-doc +// fraction is high (ReindexOption). +package idxcron + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fulltext2" + idxcronplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/idxcron" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// TailChunkThreshold gates the scheduled compaction on tag=1 CdcTail growth: fire +// only once the tail has at least this many chunk rows since the last reindex. A +// dev/test deploy can lower it via MO_IDXCRON_FULLTEXT2_TAIL_THRESHOLD (e.g. 1). +// Mirrors bm25's TailChunkThreshold. +var TailChunkThreshold = func() int64 { + if v := os.Getenv("MO_IDXCRON_FULLTEXT2_TAIL_THRESHOLD"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return 1024 +}() + +type Hooks struct{} + +var _ idxcronplugin.Hooks = Hooks{} +var _ idxcronplugin.ReindexOptioner = Hooks{} + +// Updatable fires the scheduled compaction once the tag=1 CdcTail has grown past +// TailChunkThreshold (the framework already applied the auto_update/hour/cadence +// gates). Mirrors bm25's Updatable. +func (Hooks) Updatable(in idxcronplugin.UpdatableInput) (bool, string, error) { + if in.LastUpdateAt != nil { + last := time.Unix(in.LastUpdateAt.Unix(), 0) + if last.Add(in.Interval).After(time.Now()) { + return false, fmt.Sprintf("within reindex interval (last %s + %v)", + last.Format("2006-01-02 15:04:05"), in.Interval), nil + } + } + + var storageTbl string + for _, idx := range in.TableDef.Indexes { + if idx.IndexName == in.IndexName && idx.IndexAlgoTableType == catalog.FullText2Index_TblType_Storage { + storageTbl = idx.IndexTableName + break + } + } + if storageTbl == "" { + return false, "not a fulltext2 index (no storage table)", nil + } + + cfg := fulltext2.TableConfig{DbName: in.TableDef.DbName, IndexTable: storageTbl} + count, err := fulltext2.CountTailChunks(in.Sqlproc, cfg) + if err != nil { + return false, "", err + } + logutil.Infof("[idxcron][ftv2] Updatable index=%s: tag=1 tail chunks=%d threshold=%d", + in.IndexName, count, TailChunkThreshold) + if count < TailChunkThreshold { + return false, fmt.Sprintf("tag=1 tail chunks %d < threshold %d", count, TailChunkThreshold), nil + } + return true, "", nil +} + +// RebuildDeadPct: once the dead-doc percentage in the base exceeds this, the +// scheduled compaction fires a full REBUILD instead of an incremental MERGE. +const RebuildDeadPct = 30 + +// ReindexOption picks the scheduled reindex mode: "MERGE" normally, "" (full +// REBUILD) once the dead-doc fraction (1 - liveSourceRows/baseDocs) exceeds +// RebuildDeadPct. Mirrors bm25's ReindexOption. +func (Hooks) ReindexOption(in idxcronplugin.UpdatableInput) (string, error) { + var metaTbl string + for _, idx := range in.TableDef.Indexes { + if idx.IndexName == in.IndexName && idx.IndexAlgoTableType == catalog.FullText2Index_TblType_Metadata { + metaTbl = idx.IndexTableName + break + } + } + if metaTbl == "" { + return "MERGE", nil + } + cfg := fulltext2.TableConfig{DbName: in.TableDef.DbName, MetadataTable: metaTbl} + baseDocs, err := fulltext2.SumBaseNrow(in.Sqlproc, cfg) + if err != nil { + return "", err + } + if baseDocs == 0 { + return "MERGE", nil + } + liveRows, err := countSourceRows(in.Sqlproc, in.TableDef.DbName, in.TableDef.Name) + if err != nil { + return "", err + } + opt := reindexOptionForCounts(liveRows, baseDocs) + deadPct := int64(0) + if baseDocs > 0 { + deadPct = 100 - liveRows*100/baseDocs + } + logutil.Infof("[idxcron][ftv2] ReindexOption index=%s: liveRows=%d baseDocs=%d dead=%d%% threshold=%d%% -> %s", + in.IndexName, liveRows, baseDocs, deadPct, RebuildDeadPct, optLabel(opt)) + return opt, nil +} + +// reindexOptionForCounts is the pure decision: "" (REBUILD) once dead% > +// RebuildDeadPct, else "MERGE". Integer arithmetic for an exact boundary. +func reindexOptionForCounts(liveRows, baseDocs int64) string { + if baseDocs <= 0 { + return "MERGE" + } + if liveRows*100 < baseDocs*(100-RebuildDeadPct) { + return "" // dead % > RebuildDeadPct → full REBUILD + } + return "MERGE" +} + +func optLabel(opt string) string { + if opt == "" { + return "REBUILD" + } + return opt +} + +// countSourceRows returns the source table's live row count (= live docs). +func countSourceRows(sqlproc *sqlexec.SqlProcess, db, table string) (int64, error) { + res, err := sqlexec.RunSql(sqlproc, fmt.Sprintf("SELECT COUNT(*) FROM %s", sqlquote.QualifiedIdent(db, table))) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat != nil && bat.RowCount() > 0 { + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + } + return 0, nil +} diff --git a/pkg/fulltext2/plugin/idxcron/idxcron_cover_test.go b/pkg/fulltext2/plugin/idxcron/idxcron_cover_test.go new file mode 100644 index 0000000000000..7696bb198cafc --- /dev/null +++ b/pkg/fulltext2/plugin/idxcron/idxcron_cover_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package idxcron + +import ( + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + idxcronplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/idxcron" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/stretchr/testify/require" +) + +// TestReindexOptionForCounts exercises the pure MERGE-vs-REBUILD decision on both +// sides of the RebuildDeadPct (30%) boundary. dead% = 100 - liveRows*100/baseDocs. +func TestReindexOptionForCounts(t *testing.T) { + const merge = "MERGE" + const rebuild = "" // full REBUILD + + cases := []struct { + name string + liveRows int64 + baseDocs int64 + want string + }{ + // baseDocs guard branches. + {"zero base", 0, 0, merge}, + {"negative base", 100, -1, merge}, + + // dead% low → MERGE. All rows live = 0% dead. + {"no dead docs", 100, 100, merge}, + // 20% dead (< 30) → MERGE. + {"20pct dead", 80, 100, merge}, + // Exactly at boundary: dead = 30% → liveRows*100 == baseDocs*70, NOT strictly + // less-than, so still MERGE (boundary is inclusive of MERGE). + {"exactly 30pct dead", 70, 100, merge}, + + // dead% high → REBUILD. 31% dead → 69*100 < 100*70 → REBUILD. + {"31pct dead", 69, 100, rebuild}, + {"all dead", 0, 100, rebuild}, + {"half dead", 50, 100, rebuild}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := reindexOptionForCounts(tc.liveRows, tc.baseDocs) + require.Equal(t, tc.want, got, "liveRows=%d baseDocs=%d", tc.liveRows, tc.baseDocs) + }) + } +} + +// TestOptLabel covers the human-readable label mapping used in logging. +func TestOptLabel(t *testing.T) { + require.Equal(t, "REBUILD", optLabel("")) + require.Equal(t, "MERGE", optLabel("MERGE")) +} + +// TestUpdatableWithinInterval covers the cadence short-circuit in Updatable: when +// LastUpdateAt + Interval is still in the future, Updatable returns (false, reason) +// BEFORE touching the SqlProcess, so it is reachable with plain inputs. +func TestUpdatableWithinInterval(t *testing.T) { + // LastUpdateAt = now, Interval = 1h → last+interval is in the future → not + // updatable, no engine needed. + now := types.UnixToTimestamp(time.Now().Unix()) + in := idxcronplugin.UpdatableInput{ + IndexName: "idx1", + TableDef: &plan.TableDef{DbName: "db", Name: "t"}, + LastUpdateAt: &now, + Interval: time.Hour, + } + ok, reason, err := Hooks{}.Updatable(in) + require.NoError(t, err) + require.False(t, ok) + require.Contains(t, reason, "within reindex interval") +} + +// TestUpdatableNoStorageTable covers the branch where the interval gate has passed +// (LastUpdateAt in the distant past) but the TableDef has no matching fulltext2 +// storage index, so Updatable bails out before any SQL against the SqlProcess. +func TestUpdatableNoStorageTable(t *testing.T) { + // LastUpdateAt far in the past so last+interval is already elapsed. + past := types.UnixToTimestamp(time.Now().Add(-100 * time.Hour).Unix()) + in := idxcronplugin.UpdatableInput{ + IndexName: "idx_missing", + TableDef: &plan.TableDef{ + DbName: "db", + Name: "t", + Indexes: []*plan.IndexDef{}, // no storage index for idx_missing + }, + LastUpdateAt: &past, + Interval: time.Hour, + } + ok, reason, err := Hooks{}.Updatable(in) + require.NoError(t, err) + require.False(t, ok) + require.Contains(t, reason, "not a fulltext2 index") +} + +// TestReindexOptionNoMetaTable covers the ReindexOption fast-path: when the +// TableDef has no matching metadata index, it returns "MERGE" without consulting +// the SqlProcess. +func TestReindexOptionNoMetaTable(t *testing.T) { + in := idxcronplugin.UpdatableInput{ + IndexName: "idx_missing", + TableDef: &plan.TableDef{ + DbName: "db", + Name: "t", + Indexes: []*plan.IndexDef{}, // no metadata index + }, + } + opt, err := Hooks{}.ReindexOption(in) + require.NoError(t, err) + require.Equal(t, "MERGE", opt) +} + +// TestTailChunkThreshold sanity-checks the default threshold constant is positive +// (the env override path is exercised at package-init time). +func TestTailChunkThreshold(t *testing.T) { + require.Greater(t, TailChunkThreshold, int64(0)) + require.Equal(t, 30, RebuildDeadPct) +} diff --git a/pkg/fulltext2/plugin/iscp/iscp.go b/pkg/fulltext2/plugin/iscp/iscp.go new file mode 100644 index 0000000000000..6d02fc718dcba --- /dev/null +++ b/pkg/fulltext2/plugin/iscp/iscp.go @@ -0,0 +1,46 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package iscp is fulltext2's ISCP hook layer. A fulltext2 index is +// model-building: its CDC sinker builds tag=1 CdcTail delta frames (the writer / +// consumer loop in pkg/iscp), NOT SQL text. Mirrors bm25's iscp hooks. +// +// Registered from pkg/indexplugin/iscp/import.go via iscp.Register. +package iscp + +import ( + "context" + + "github.com/matrixorigin/matrixone/pkg/catalog" + iscppkg "github.com/matrixorigin/matrixone/pkg/iscp" + "github.com/matrixorigin/matrixone/pkg/pb/plan" +) + +func init() { + iscppkg.Register(catalog.MoIndexFullText2Algo.ToString(), Hooks{}) +} + +// Hooks implements iscp.Hooks for fulltext2. +type Hooks struct{} + +var _ iscppkg.Hooks = Hooks{} + +func (Hooks) NewSqlWriter(jobID iscppkg.JobID, info *iscppkg.ConsumerInfo, + tabledef *plan.TableDef, indexdefs []*plan.IndexDef) (iscppkg.IndexSqlWriter, error) { + return iscppkg.NewFulltext2SqlWriter(catalog.MoIndexFullText2Algo.ToString(), jobID, info, tabledef, indexdefs) +} + +func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error, r iscppkg.DataRetriever) { + iscppkg.RunFulltext2(c, ctx, errch, r) +} diff --git a/pkg/fulltext2/plugin/iscp/iscp_cover_test.go b/pkg/fulltext2/plugin/iscp/iscp_cover_test.go new file mode 100644 index 0000000000000..60daac99480c8 --- /dev/null +++ b/pkg/fulltext2/plugin/iscp/iscp_cover_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + iscppkg "github.com/matrixorigin/matrixone/pkg/iscp" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/stretchr/testify/require" +) + +// TestHooksRegistered confirms init() registered fulltext2's ISCP Hooks with +// the iscp framework and that the value satisfies the iscp.Hooks interface +// (mirrors the `var _ iscppkg.Hooks = Hooks{}` assertion in iscp.go). +func TestHooksRegistered(t *testing.T) { + var _ iscppkg.Hooks = Hooks{} + + h, ok := iscppkg.GetHooks(catalog.MoIndexFullText2Algo.ToString()) + require.True(t, ok) + require.NotNil(t, h) + require.True(t, iscppkg.HasHooks(catalog.MoIndexFullText2Algo.ToString())) +} + +// TestNewSqlWriterErrors drives Hooks.NewSqlWriter through the two error guards +// reachable with plain inputs (no live txn/engine required): +// - the fulltext2 storage/metadata hidden tables are missing, and +// - the index has no source column (empty Parts). +// +// The success path and the Run hook need a live CDC consumer pipeline / typed +// tabledef pk resolution, so they are exercised by the fulltext2 BVT cases, not +// here. +func TestNewSqlWriterErrors(t *testing.T) { + // Missing storage/metadata hidden tables → guarded error before any tabledef + // dereference, so a nil tabledef is safe here. + _, err := Hooks{}.NewSqlWriter( + iscppkg.JobID{}, + &iscppkg.ConsumerInfo{DBName: "db"}, + nil, + []*plan.IndexDef{{ + IndexTableName: "not_a_hidden_table", + IndexAlgoTableType: "other", + Parts: []string{"body"}, + }}, + ) + require.Error(t, err) + + // Storage + metadata present but the index has no source column (empty Parts) + // → distinct guarded error, again before any tabledef dereference. + _, err = Hooks{}.NewSqlWriter( + iscppkg.JobID{}, + &iscppkg.ConsumerInfo{DBName: "db"}, + nil, + []*plan.IndexDef{ + { + IndexTableName: "storage_tbl", + IndexAlgoTableType: catalog.FullText2Index_TblType_Storage, + Parts: nil, + }, + { + IndexTableName: "meta_tbl", + IndexAlgoTableType: catalog.FullText2Index_TblType_Metadata, + }, + }, + ) + require.Error(t, err) +} diff --git a/pkg/fulltext2/plugin/plan/plan.go b/pkg/fulltext2/plugin/plan/plan.go new file mode 100644 index 0000000000000..0ad9bde9c6e21 --- /dev/null +++ b/pkg/fulltext2/plugin/plan/plan.go @@ -0,0 +1,50 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package plan holds the fulltext2 index's plan-layer hooks: hidden-table schema +// construction (BuildFullTextIndexDefs) and the inert vector-sort hooks (fulltext2 +// is queried via MATCH, not the ORDER BY rewrite). +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +var _ planplugin.Hooks = Hooks{} + +type Hooks struct{} + +// BuildSecondaryIndexDefs — fulltext2 is created via CREATE FULLTEXT2 INDEX +// (*tree.FullTextIndex → BuildFullTextIndexDefs), not the vector `USING` path. +func (Hooks) BuildSecondaryIndexDefs( + _ planplugin.CompilerContext, _ *tree.Index, _ map[string]*plan.ColDef, _ []*plan.IndexDef, _ string, +) ([]*plan.IndexDef, []*plan.TableDef, error) { + return nil, nil, moerr.NewNotSupportedNoCtx("fulltext2 index is created via CREATE FULLTEXT2 INDEX") +} + +// CanApply — fulltext2 is queried via MATCH (the apply_indices_fulltext path), +// never the vector ORDER BY LIMIT rewrite. +func (Hooks) CanApply(planplugin.PlanBuilder, *planplugin.VectorSortContext, *planplugin.MultiTableIndexRef) (bool, error) { + return false, nil +} + +// ApplyForSort — inert (returns the node id unchanged). +func (Hooks) ApplyForSort( + _ planplugin.PlanBuilder, _ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, nodeID int32, _ planplugin.ApplyForSortOpts, +) (int32, bool, error) { + return nodeID, false, nil +} diff --git a/pkg/fulltext2/plugin/plan/plan_cover_test.go b/pkg/fulltext2/plugin/plan/plan_cover_test.go new file mode 100644 index 0000000000000..3b394adda1630 --- /dev/null +++ b/pkg/fulltext2/plugin/plan/plan_cover_test.go @@ -0,0 +1,548 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CPU-only unit coverage for the fulltext2 plan-layer hooks. plan.go's inert +// vector-sort redirects (CanApply / ApplyForSort), schema.go's hidden-table +// builder (BuildFullTextIndexDefs) + buildFullText2Params, and tablefunc.go's +// pure TVF builders (buildFullText2Create / buildFullText2Search / +// getFullText2Params) are all reachable without a live cluster. The +// happy-path builders call two planplugin helper vars (production wires them +// in pkg/sql/plan's init) which the init() below stubs to avoid an import +// cycle. +package plan + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +func init() { + if planplugin.MakeHiddenColDefByName == nil { + planplugin.MakeHiddenColDefByName = func(name string) *plan.ColDef { + return &plan.ColDef{Name: name, Typ: plan.Type{Id: int32(types.T_varchar)}} + } + } + if planplugin.DeepCopyColDefList == nil { + planplugin.DeepCopyColDefList = func(in []*plan.ColDef) []*plan.ColDef { return in } + } + if planplugin.ValidateIncludeColumns == nil { + // Minimal faithful stub of the production validator (pkg/sql/plan build_ddl.go): + // reject an INCLUDE column whose type is not in supportedTypes. Enough to exercise + // BuildFullTextIndexDefs' include wiring without the pkg/sql/plan import cycle. + planplugin.ValidateIncludeColumns = func(ctx planplugin.CompilerContext, includeCols []*tree.UnresolvedName, + colMap map[string]*plan.ColDef, _, _ string, supportedTypes []types.T) error { + for _, uc := range includeCols { + col, ok := colMap[uc.ColName()] + if !ok { + return moerr.NewInvalidInputNoCtxf("INCLUDE column '%s' is not exist", uc.ColName()) + } + ok = false + for _, st := range supportedTypes { + if types.T(col.Typ.Id) == st { + ok = true + break + } + } + if !ok { + return moerr.NewNotSupportedNoCtxf("INCLUDE column '%s' has unsupported type", uc.ColName()) + } + } + return nil + } + } +} + +// ftIndexInclude builds a *tree.FullTextIndex over one indexed col plus an INCLUDE list. +func ftIndexInclude(name, indexedCol string, includeCols ...string) *tree.FullTextIndex { + idx := ftIndex(name, indexedCol) + inc := make([]*tree.UnresolvedName, 0, len(includeCols)) + for _, c := range includeCols { + inc = append(inc, tree.NewUnresolvedName(tree.NewCStr(c, 0))) + } + idx.IndexOption = &tree.IndexOption{IncludeColumns: inc} + return idx +} + +// --- stubs ----------------------------------------------------------------- + +// stubCompilerContext is the minimal planplugin.CompilerContext (GetContext + +// ResolveVariable). +type stubCompilerContext struct{ ctx context.Context } + +func (c stubCompilerContext) GetContext() context.Context { return c.ctx } +func (c stubCompilerContext) ResolveVariable(string, bool, bool) (interface{}, error) { + return nil, nil +} + +var _ planplugin.CompilerContext = stubCompilerContext{} + +func newStubCompilerContext() stubCompilerContext { + return stubCompilerContext{ctx: context.Background()} +} + +// stubPlanBuilder is the minimal planplugin.PlanBuilder. The fulltext2 +// tablefunc builders only use GenNewBindTag / AppendNode / GetContext; the +// vector-sort redirect methods are never reached from these tests, so they +// panic. +type stubPlanBuilder struct { + ctx context.Context + nodes []*plan.Node + nextTag int32 +} + +func newStubPlanBuilder() *stubPlanBuilder { + return &stubPlanBuilder{ctx: context.Background()} +} + +func (b *stubPlanBuilder) GetContext() context.Context { return b.ctx } + +func (b *stubPlanBuilder) GenNewBindTag() int32 { + b.nextTag++ + return b.nextTag +} + +func (b *stubPlanBuilder) AppendNode(node *plan.Node, _ planplugin.BindContext) int32 { + id := int32(len(b.nodes)) + b.nodes = append(b.nodes, node) + return id +} + +func (b *stubPlanBuilder) ApplyIndicesForSortUsingHnsw(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, _ int32, _ planplugin.ApplyForSortOpts) (int32, bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) ApplyIndicesForSortUsingCagra(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, _ int32, _ planplugin.ApplyForSortOpts) (int32, bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) ApplyIndicesForSortUsingIvfpq(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, _ int32, _ planplugin.ApplyForSortOpts) (int32, bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) ApplyIndicesForSortUsingIvfflat(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, _ int32, _ planplugin.ApplyForSortOpts) (int32, bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) CanApplyHnsw(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef) (bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) CanApplyCagra(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef) (bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) CanApplyIvfpq(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef) (bool, error) { + panic("not used") +} +func (b *stubPlanBuilder) CanApplyIvfflat(_ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef) (bool, error) { + panic("not used") +} + +var _ planplugin.PlanBuilder = (*stubPlanBuilder)(nil) + +// --- builders for AST inputs ---------------------------------------------- + +// ftIndex builds a *tree.FullTextIndex over the given column names (single or +// multi-column) with no IndexOption. +func ftIndex(name string, cols ...string) *tree.FullTextIndex { + kp := make([]*tree.KeyPart, 0, len(cols)) + for _, c := range cols { + kp = append(kp, &tree.KeyPart{ColName: tree.NewUnresolvedName(tree.NewCStr(c, 0))}) + } + return &tree.FullTextIndex{Name: name, KeyParts: kp} +} + +// textColMap returns a colMap with a varchar pk and a text-typed indexed col. +func textColMap(pkName string, cols ...string) map[string]*plan.ColDef { + m := map[string]*plan.ColDef{ + pkName: {Name: pkName, Typ: plan.Type{Id: int32(types.T_varchar)}}, + } + for _, c := range cols { + m[c] = &plan.ColDef{Name: c, Typ: plan.Type{Id: int32(types.T_text)}} + } + return m +} + +func numValTblFunc(s string) *tree.TableFunction { + nv := tree.NewNumVal[string](s, s, false, tree.P_char) + return &tree.TableFunction{Func: &tree.FuncExpr{Exprs: tree.Exprs{nv}}} +} + +func nonNumValTblFunc() *tree.TableFunction { + un := tree.NewUnresolvedName(tree.NewCStr("col", 0)) + return &tree.TableFunction{Func: &tree.FuncExpr{Exprs: tree.Exprs{un}}} +} + +// makeArgs builds n plan.Expr args: [param(varchar), ints...]. +func makeArgs(n int) []*plan.Expr { + out := make([]*plan.Expr, 0, n) + out = append(out, &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_varchar)}, + Expr: &plan.Expr_Lit{Lit: &plan.Literal{Value: &plan.Literal_Sval{Sval: "{}"}}}, + }) + for i := 1; i < n; i++ { + out = append(out, &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_int64)}, + Expr: &plan.Expr_Lit{Lit: &plan.Literal{Value: &plan.Literal_I64Val{I64Val: int64(i)}}}, + }) + } + return out +} + +// --- plan.go: inert vector-sort hooks -------------------------------------- + +func TestCanApply_AlwaysFalse(t *testing.T) { + ok, err := Hooks{}.CanApply(newStubPlanBuilder(), &planplugin.VectorSortContext{}, &planplugin.MultiTableIndexRef{}) + require.NoError(t, err) + require.False(t, ok) +} + +func TestApplyForSort_Inert(t *testing.T) { + id, applied, err := Hooks{}.ApplyForSort(newStubPlanBuilder(), &planplugin.VectorSortContext{}, &planplugin.MultiTableIndexRef{}, 42, planplugin.ApplyForSortOpts{}) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, int32(42), id) +} + +// --- plan.go: BuildSecondaryIndexDefs (always errors) ---------------------- + +func TestBuildSecondaryIndexDefs_NotSupported(t *testing.T) { + idxDefs, tblDefs, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), nil, nil, nil, "id") + require.Error(t, err) + require.Nil(t, idxDefs) + require.Nil(t, tblDefs) +} + +// --- schema.go: BuildFullTextIndexDefs error paths ------------------------- + +func TestBuildFullTextIndexDefs_EmptyPkey(t *testing.T) { + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), textColMap("id", "body"), nil, "") + require.Error(t, err) +} + +func TestBuildFullTextIndexDefs_FakePkey(t *testing.T) { + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), textColMap("id", "body"), nil, catalog.FakePrimaryKeyColName) + require.Error(t, err) +} + +func TestBuildFullTextIndexDefs_DuplicateColumn(t *testing.T) { + existed := []*plan.IndexDef{{ + IndexAlgo: tree.INDEX_TYPE_FULLTEXT2.ToString(), + Parts: []string{"body"}, + }} + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), textColMap("id", "body"), existed, "id") + require.Error(t, err) +} + +func TestBuildFullTextIndexDefs_ColNotExist(t *testing.T) { + // indexed column missing from colMap. + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "nope"), textColMap("id", "body"), nil, "id") + require.Error(t, err) +} + +func TestBuildFullTextIndexDefs_UnsupportedColType(t *testing.T) { + m := textColMap("id", "body") + m["body"].Typ.Id = int32(types.T_int64) // fulltext2 only supports char/varchar/text/json/datalink + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), m, nil, "id") + require.Error(t, err) +} + +// TestBuildFullTextIndexDefs_UnsupportedPkType: a FLOAT/DOUBLE primary key must be +// rejected at CREATE (the segment pk codec can't encode it) rather than accepted and +// failing later at build/CDC — the SupportedPrimaryKeyTypes/encodePk parity guard. +func TestBuildFullTextIndexDefs_UnsupportedPkType(t *testing.T) { + m := textColMap("id", "body") + m["id"].Typ.Id = int32(types.T_float64) // codec has no float case + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), m, nil, "id") + require.ErrorContains(t, err, "primary key type") + + // a codec-supported pk (int64) clears the pk-type gate. + ok := textColMap("id", "body") + ok["id"].Typ.Id = int32(types.T_int64) + _, _, err = Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), ok, nil, "id") + require.NoError(t, err) +} + +func TestBuildFullTextIndexDefs_BadParser(t *testing.T) { + idx := ftIndex("ft", "body") + idx.IndexOption = &tree.IndexOption{ParserName: "bogus"} + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), idx, textColMap("id", "body"), nil, "id") + require.Error(t, err) +} + +func TestBuildFullTextIndexDefs_PositionFreeNonGojieba(t *testing.T) { + idx := ftIndex("ft", "body") + idx.IndexOption = &tree.IndexOption{ParserName: "ngram", PositionFree: true} + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), idx, textColMap("id", "body"), nil, "id") + require.Error(t, err) +} + +// --- schema.go: BuildFullTextIndexDefs happy path -------------------------- + +func TestBuildFullTextIndexDefs_OK(t *testing.T) { + idxDefs, tblDefs, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "body"), textColMap("id", "body"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) + require.Len(t, tblDefs, 2) + + // index defs: both stamped algo=fulltext2, told apart by IndexAlgoTableType. + algo := tree.INDEX_TYPE_FULLTEXT2.ToString() + require.Equal(t, algo, idxDefs[0].IndexAlgo) + require.Equal(t, algo, idxDefs[1].IndexAlgo) + require.Equal(t, catalog.FullText2Index_TblType_Storage, idxDefs[0].IndexAlgoTableType) + require.Equal(t, catalog.FullText2Index_TblType_Metadata, idxDefs[1].IndexAlgoTableType) + require.Equal(t, []string{"body"}, idxDefs[0].Parts) + require.NotEmpty(t, idxDefs[0].IndexAlgoParams) + + // storage table: 4 declared cols + hidden composite pk; metadata: 6 cols. + require.Equal(t, catalog.FullText2Index_TblType_Storage, tblDefs[0].TableType) + require.Equal(t, catalog.FullText2Index_TblType_Metadata, tblDefs[1].TableType) + require.Len(t, tblDefs[0].Cols, 5) + require.Len(t, tblDefs[1].Cols, 6) + require.NotNil(t, tblDefs[0].Pkey) + require.NotNil(t, tblDefs[1].Pkey) +} + +func TestBuildFullTextIndexDefs_MultiColumnOK(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), ftIndex("ft", "title", "body"), textColMap("id", "title", "body"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) + require.ElementsMatch(t, []string{"title", "body"}, idxDefs[0].Parts) +} + +func TestBuildFullTextIndexDefs_GojiebaPositionFreeOK(t *testing.T) { + idx := ftIndex("ft", "body") + idx.IndexOption = &tree.IndexOption{ParserName: "gojieba", PositionFree: true} + idxDefs, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), idx, textColMap("id", "body"), nil, "id") + require.NoError(t, err) + params, perr := catalog.IndexParamsStringToMap(idxDefs[0].IndexAlgoParams) + require.NoError(t, perr) + require.Equal(t, "gojieba", params["parser"]) + require.Equal(t, "true", params[catalog.IndexAlgoParamPositionFree]) +} + +// --- schema.go: INCLUDE columns -------------------------------------------- + +// TestBuildFullTextIndexDefs_IncludeOK: INCLUDE(status, prio) with supported types +// populates IndexDef.IncludedColumns on BOTH hidden-table defs and persists the names in +// algo_params (for reload). The two hidden tables are unchanged — include values live in +// the segment docmap, not a SQL column. +func TestBuildFullTextIndexDefs_IncludeOK(t *testing.T) { + m := textColMap("id", "body") + m["status"] = &plan.ColDef{Name: "status", Typ: plan.Type{Id: int32(types.T_varchar)}} + m["prio"] = &plan.ColDef{Name: "prio", Typ: plan.Type{Id: int32(types.T_int64)}} + + idxDefs, tblDefs, err := Hooks{}.BuildFullTextIndexDefs( + newStubCompilerContext(), ftIndexInclude("ft", "body", "status", "prio"), m, nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) + + // IncludedColumns set on both defs (storage + metadata). + require.Equal(t, []string{"status", "prio"}, idxDefs[0].IncludedColumns) + require.Equal(t, []string{"status", "prio"}, idxDefs[1].IncludedColumns) + + // Persisted in algo_params under the shared catalog key, and round-trips. + params, perr := catalog.IndexParamsStringToMap(idxDefs[0].IndexAlgoParams) + require.NoError(t, perr) + names, cerr := catalog.ParseIncludeColumnsValue(params[catalog.IncludedColumns]) + require.NoError(t, cerr) + require.Equal(t, []string{"status", "prio"}, names) + + // Hidden tables are unchanged (no per-include SQL column). + require.Len(t, tblDefs[0].Cols, 5) + require.Len(t, tblDefs[1].Cols, 6) +} + +// TestBuildFullTextIndexDefs_IncludeUnsupportedType: an INCLUDE column of a type outside +// the plugin's SupportedIncludeColumnTypes() set is rejected at CREATE. +func TestBuildFullTextIndexDefs_IncludeUnsupportedType(t *testing.T) { + m := textColMap("id", "body") + m["amount"] = &plan.ColDef{Name: "amount", Typ: plan.Type{Id: int32(types.T_float64)}} // float not supported + _, _, err := Hooks{}.BuildFullTextIndexDefs( + newStubCompilerContext(), ftIndexInclude("ft", "body", "amount"), m, nil, "id") + require.Error(t, err) +} + +// TestBuildFullTextIndexDefs_IncludeIndexedColumn: the indexed text column itself cannot be +// an INCLUDE column (it is tokenized into the postings; storing the raw text again is +// redundant). Rejected before the type gate, so the reason is "indexed text column" rather +// than a type error. +func TestBuildFullTextIndexDefs_IncludeIndexedColumn(t *testing.T) { + _, _, err := Hooks{}.BuildFullTextIndexDefs( + newStubCompilerContext(), ftIndexInclude("ft", "body", "body"), textColMap("id", "body"), nil, "id") + require.Error(t, err) + require.Contains(t, err.Error(), "indexed text column") +} + +// TestBuildFullTextIndexDefs_IncludeIndexedColumnMulti: in a multi-column MATCH(body, title) +// index, INCLUDE-ing ANY of the indexed columns is rejected (all KeyParts are excluded). +func TestBuildFullTextIndexDefs_IncludeIndexedColumnMulti(t *testing.T) { + idx := ftIndex("ft", "body", "title") + idx.IndexOption = &tree.IndexOption{IncludeColumns: []*tree.UnresolvedName{ + tree.NewUnresolvedName(tree.NewCStr("title", 0)), + }} + _, _, err := Hooks{}.BuildFullTextIndexDefs( + newStubCompilerContext(), idx, textColMap("id", "body", "title"), nil, "id") + require.Error(t, err) + require.Contains(t, err.Error(), "indexed text column") +} + +// TestBuildFullTextIndexDefs_NoInclude: without INCLUDE, IncludedColumns is empty and no +// included_columns param is written. +func TestBuildFullTextIndexDefs_NoInclude(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildFullTextIndexDefs( + newStubCompilerContext(), ftIndex("ft", "body"), textColMap("id", "body"), nil, "id") + require.NoError(t, err) + require.Empty(t, idxDefs[0].IncludedColumns) + params, perr := catalog.IndexParamsStringToMap(idxDefs[0].IndexAlgoParams) + require.NoError(t, perr) + _, ok := params[catalog.IncludedColumns] + require.False(t, ok) +} + +// --- schema.go: buildFullText2Params directly ------------------------------ + +func TestBuildFullText2Params_DefaultParser(t *testing.T) { + s, err := buildFullText2Params(ftIndex("ft", "body")) + require.NoError(t, err) + m, err := catalog.IndexParamsStringToMap(s) + require.NoError(t, err) + require.Equal(t, "ngram", m["parser"]) // default parser always recorded +} + +func TestBuildFullText2Params_AllOptions(t *testing.T) { + idx := ftIndex("ft", "body") + idx.IndexOption = &tree.IndexOption{ + ParserName: "gojieba", + MaxIndexCapacity: 1000000, + MaxPostingsCapacity: 4000000, + PositionFree: true, + AutoUpdate: true, + Day: 1, + Hour: 2, + Second: 3, + } + s, err := buildFullText2Params(idx) + require.NoError(t, err) + m, err := catalog.IndexParamsStringToMap(s) + require.NoError(t, err) + require.Equal(t, "gojieba", m["parser"]) + require.Equal(t, "1000000", m[catalog.IndexAlgoParamMaxIndexCapacity]) + require.Equal(t, "4000000", m[catalog.IndexAlgoParamMaxPostingsCapacity]) + require.Equal(t, "true", m[catalog.IndexAlgoParamPositionFree]) + require.Equal(t, "true", m[catalog.AutoUpdate]) + require.Equal(t, "1", m[catalog.Day]) + require.Equal(t, "2", m[catalog.Hour]) + require.Equal(t, "3", m[catalog.Second]) +} + +func TestBuildFullText2Params_PositionFreeNonGojiebaError(t *testing.T) { + idx := ftIndex("ft", "body") + idx.IndexOption = &tree.IndexOption{ParserName: "ngram", PositionFree: true} + _, err := buildFullText2Params(idx) + require.Error(t, err) +} + +// --- tablefunc.go: getFullText2Params -------------------------------------- + +func TestGetFullText2Params_OK(t *testing.T) { + b := newStubPlanBuilder() + nv := tree.NewNumVal[string](`{"parser":"ngram"}`, `{"parser":"ngram"}`, false, tree.P_char) + out, err := getFullText2Params(b, &tree.FuncExpr{Exprs: tree.Exprs{nv}}) + require.NoError(t, err) + require.Equal(t, `{"parser":"ngram"}`, out) +} + +func TestGetFullText2Params_Error(t *testing.T) { + b := newStubPlanBuilder() + un := tree.NewUnresolvedName(tree.NewCStr("col", 0)) + _, err := getFullText2Params(b, &tree.FuncExpr{Exprs: tree.Exprs{un}}) + require.Error(t, err) +} + +// --- tablefunc.go: buildFullText2Create ------------------------------------ + +func TestBuildFullText2Create_TooFewArgs(t *testing.T) { + b := newStubPlanBuilder() + _, err := buildFullText2Create(b, numValTblFunc(`{}`), nil, makeArgs(3), nil) + require.Error(t, err) +} + +func TestBuildFullText2Create_BadParams(t *testing.T) { + b := newStubPlanBuilder() + _, err := buildFullText2Create(b, nonNumValTblFunc(), nil, makeArgs(4), nil) + require.Error(t, err) +} + +func TestBuildFullText2Create_OK(t *testing.T) { + b := newStubPlanBuilder() + id, err := buildFullText2Create(b, numValTblFunc(`{"parser":"ngram"}`), nil, makeArgs(4), nil) + require.NoError(t, err) + node := b.nodes[id] + require.Equal(t, plan.Node_FUNCTION_SCAN, node.NodeType) + require.Equal(t, FullText2CreateFuncName, node.TableDef.TblFunc.Name) + require.True(t, node.TableDef.TblFunc.IsSingle) + require.Len(t, node.TblFuncExprList, 3) // param stripped from the 4 args +} + +// --- tablefunc.go: buildFullText2Search ------------------------------------ + +func TestBuildFullText2Search_BadArgCount(t *testing.T) { + b := newStubPlanBuilder() + _, err := buildFullText2Search(b, numValTblFunc(`{}`), nil, makeArgs(3), nil) + require.Error(t, err) + _, err = buildFullText2Search(b, numValTblFunc(`{}`), nil, makeArgs(5), nil) + require.Error(t, err) +} + +func TestBuildFullText2Search_BadParams(t *testing.T) { + b := newStubPlanBuilder() + _, err := buildFullText2Search(b, nonNumValTblFunc(), nil, makeArgs(4), nil) + require.Error(t, err) +} + +func TestBuildFullText2Search_OK(t *testing.T) { + b := newStubPlanBuilder() + id, err := buildFullText2Search(b, numValTblFunc(`{"parser":"ngram"}`), nil, makeArgs(4), nil) + require.NoError(t, err) + node := b.nodes[id] + require.Equal(t, plan.Node_FUNCTION_SCAN, node.NodeType) + require.Equal(t, FullText2SearchFuncName, node.TableDef.TblFunc.Name) + require.False(t, node.TableDef.TblFunc.IsSingle) + require.Len(t, node.TblFuncExprList, 3) // param stripped from the 4 args +} + +// --- tablefunc.go: buildFullText2Compact ----------------------------------- + +func TestBuildFullText2Compact_OK(t *testing.T) { + b := newStubPlanBuilder() + id, err := buildFullText2Compact(b, numValTblFunc(`{}`), nil, makeArgs(4), nil) + require.NoError(t, err) + node := b.nodes[id] + require.Equal(t, plan.Node_FUNCTION_SCAN, node.NodeType) + require.Equal(t, FullText2CompactFuncName, node.TableDef.TblFunc.Name) + require.Len(t, node.TblFuncExprList, 4) // args passed as-is (no param strip) + require.Len(t, node.TableDef.Cols, 1) + require.Equal(t, "live_docs", node.TableDef.Cols[0].Name) +} + +// the compact builder is wired into the plan-side TVF registry, so the plan +// dispatch routes it through indexplugin instead of a hardcoded switch case. +func TestFullText2Compact_Registered(t *testing.T) { + _, ok := planplugin.TableFunc(FullText2CompactFuncName) + require.True(t, ok) +} diff --git a/pkg/fulltext2/plugin/plan/schema.go b/pkg/fulltext2/plugin/plan/schema.go new file mode 100644 index 0000000000000..f4dbdb5c45cc4 --- /dev/null +++ b/pkg/fulltext2/plugin/plan/schema.go @@ -0,0 +1,299 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "fmt" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + ftv2rt "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/runtime" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/sql/util" +) + +// fulltext2CatalogHooks is the runtime catalog contract used here for pk-type +// validation, so DDL rejects a primary key the segment codec can't encode. +var fulltext2CatalogHooks = ftv2rt.CatalogHooks{} + +// BuildFullTextIndexDefs constructs a fulltext2 index's two hidden tables — a +// chunked segment store + a metadata table (bm25 layout) — from CREATE FULLTEXT2 +// INDEX. Both defs are stamped algo="fulltext2" (told apart by IndexAlgoTableType) +// so every downstream hook dispatches to this plugin. +func (Hooks) BuildFullTextIndexDefs( + ctx planplugin.CompilerContext, + indexInfo *tree.FullTextIndex, + colMap map[string]*plan.ColDef, + existedIndexes []*plan.IndexDef, + pkeyName string, +) ([]*plan.IndexDef, []*plan.TableDef, error) { + if pkeyName == "" || pkeyName == catalog.FakePrimaryKeyColName { + return nil, nil, moerr.NewInternalErrorNoCtx("primary key cannot be empty for fulltext2 index") + } + // Reject a primary key the segment pk codec can't encode (e.g. FLOAT/DOUBLE) HERE, + // at CREATE, instead of letting an accepted DDL fail later at build/CDC. A composite + // PK is a synthesized CP key absent from colMap (ok=false) and is encoded as bytes, so + // it is intentionally not gated. + if col, ok := colMap[pkeyName]; ok && !catalogplugin.SupportsPrimaryKeyType(fulltext2CatalogHooks, types.T(col.Typ.Id)) { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), + fmt.Sprintf("fulltext2 index does not support primary key type '%s'", types.T(col.Typ.Id).String())) + } + + // Reject a duplicate fulltext2 index on the same column set. + for _, existed := range existedIndexes { + if existed.IndexAlgo != tree.INDEX_TYPE_FULLTEXT2.ToString() || len(indexInfo.KeyParts) != len(existed.Parts) { + continue + } + n := 0 + for _, keyPart := range indexInfo.KeyParts { + for _, ePart := range existed.Parts { + if ePart == keyPart.ColName.ColName() { + n++ + break + } + } + } + if n == len(indexInfo.KeyParts) { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "fulltext2 index are not allowed to use the same column") + } + } + + // Validate column types (char/varchar/text/json/datalink). + for _, keyPart := range indexInfo.KeyParts { + name := keyPart.ColName.ColName() + col, ok := colMap[name] + if !ok { + return nil, nil, moerr.NewInvalidInput(ctx.GetContext(), fmt.Sprintf("column '%s' does not exist", keyPart.ColName.ColNameOrigin())) + } + typid := col.Typ.Id + if !(typid == int32(types.T_text) || typid == int32(types.T_char) || + typid == int32(types.T_varchar) || typid == int32(types.T_json) || typid == int32(types.T_datalink)) { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "fulltext2 index only support char, varchar, text, datalink and json") + } + } + // Validate parser. + if indexInfo.IndexOption != nil && indexInfo.IndexOption.ParserName != "" { + p := strings.ToLower(indexInfo.IndexOption.ParserName) + if p != "ngram" && p != "default" && p != "json" && p != "json_value" && p != "gojieba" { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), fmt.Sprintf("fulltext2 parser %s not supported", p)) + } + } + + // Validate INCLUDE columns (stored as actual values in the segment docmap for + // prefilter + coverage). vecColName is "" — fulltext2 has no single indexed vector + // column; the plugin's SupportedIncludeColumnTypes() gates the accepted types. + var includeCols []string + if indexInfo.IndexOption != nil && len(indexInfo.IndexOption.IncludeColumns) > 0 { + // The indexed text column(s) themselves cannot be INCLUDE columns: they are + // tokenized into the postings, and storing the raw text again in the docmap + // would bloat the index for no prefilter/coverage benefit. fulltext2 may index + // multiple columns (MATCH(body, title)), so exclude every KeyPart. Checked + // before the shared validator so an indexed column reports THIS reason rather + // than a type error (an indexed varchar column would otherwise pass the type + // gate, and an indexed text column would report "unsupported type text"). + indexed := make(map[string]struct{}, len(indexInfo.KeyParts)) + for _, kp := range indexInfo.KeyParts { + indexed[kp.ColName.ColName()] = struct{}{} + } + for _, c := range indexInfo.IndexOption.IncludeColumns { + if _, ok := indexed[c.ColName()]; ok { + return nil, nil, moerr.NewInvalidInput(ctx.GetContext(), + fmt.Sprintf("INCLUDE column '%s' cannot be an indexed text column", c.ColNameOrigin())) + } + } + if err := planplugin.ValidateIncludeColumns(ctx, indexInfo.IndexOption.IncludeColumns, colMap, "", pkeyName, + fulltext2CatalogHooks.SupportedIncludeColumnTypes()); err != nil { + return nil, nil, err + } + includeCols = make([]string, 0, len(indexInfo.IndexOption.IncludeColumns)) + for _, c := range indexInfo.IndexOption.IncludeColumns { + includeCols = append(includeCols, c.ColName()) + } + } + + params, err := buildFullText2Params(indexInfo) + if err != nil { + return nil, nil, err + } + indexParts := make([]string, 0, len(indexInfo.KeyParts)) + for _, keyPart := range indexInfo.KeyParts { + indexParts = append(indexParts, keyPart.ColName.ColName()) + } + var option *plan.IndexOption + var comment string + if indexInfo.IndexOption != nil { + if indexInfo.IndexOption.ParserName != "" { + option = &plan.IndexOption{ParserName: indexInfo.IndexOption.ParserName, NgramTokenSize: int32(3)} + } + comment = indexInfo.IndexOption.Comment + } + algo := tree.INDEX_TYPE_FULLTEXT2.ToString() + mkIdx := func(tblName, tblType string) *plan.IndexDef { + return &plan.IndexDef{ + Unique: false, + IndexName: indexInfo.Name, + IndexTableName: tblName, + IndexAlgo: algo, + IndexAlgoTableType: tblType, + IndexAlgoParams: params, + Parts: indexParts, + IncludedColumns: includeCols, // stored in the segment docmap, not a hidden table + TableExist: true, + Option: option, + Comment: comment, + } + } + + // storage (chunk) table. + storeName, err := util.BuildIndexTableName(ctx.GetContext(), false) + if err != nil { + return nil, nil, err + } + storeTbl := &plan.TableDef{ + Name: storeName, + TableType: catalog.FullText2Index_TblType_Storage, + Cols: []*plan.ColDef{ + {Name: catalog.FullText2Index_TblCol_Storage_Index_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Storage_Chunk_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Storage_Data, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_blob), Width: 65536}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Storage_Tag, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + }, + } + storePk := planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) + storePk.Alg = plan.CompressType_Lz4 + storePk.Primary = true + storeTbl.Cols = append(storeTbl.Cols, storePk) + storeTbl.Pkey = &plan.PrimaryKeyDef{ + Names: []string{catalog.FullText2Index_TblCol_Storage_Index_Id, catalog.FullText2Index_TblCol_Storage_Chunk_Id}, + PkeyColName: catalog.CPrimaryKeyColName, + CompPkeyCol: storeTbl.Cols[3], + } + storeTbl.Defs = append(storeTbl.Defs, &plan.TableDef_DefType{ + Def: &plan.TableDef_DefType_Properties{Properties: &plan.PropertiesDef{Properties: []*plan.Property{ + {Key: catalog.SystemRelAttr_Kind, Value: catalog.FullText2Index_TblType_Storage}, + }}}, + }) + + // metadata table. + metaName, err := util.BuildIndexTableName(ctx.GetContext(), false) + if err != nil { + return nil, nil, err + } + metaTbl := &plan.TableDef{ + Name: metaName, + TableType: catalog.FullText2Index_TblType_Metadata, + Cols: []*plan.ColDef{ + {Name: catalog.FullText2Index_TblCol_Metadata_Index_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}, Primary: true, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Metadata_Timestamp, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Metadata_Checksum, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: types.MaxVarcharLen}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Metadata_Filesize, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Metadata_Recency, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.FullText2Index_TblCol_Metadata_Nrow, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + }, + } + metaTbl.Pkey = &plan.PrimaryKeyDef{ + Names: []string{catalog.FullText2Index_TblCol_Metadata_Index_Id}, + PkeyColName: catalog.FullText2Index_TblCol_Metadata_Index_Id, + } + metaTbl.Defs = append(metaTbl.Defs, &plan.TableDef_DefType{ + Def: &plan.TableDef_DefType_Properties{Properties: &plan.PropertiesDef{Properties: []*plan.Property{ + {Key: catalog.SystemRelAttr_Kind, Value: catalog.FullText2Index_TblType_Metadata}, + }}}, + }) + + return []*plan.IndexDef{ + mkIdx(storeName, catalog.FullText2Index_TblType_Storage), + mkIdx(metaName, catalog.FullText2Index_TblType_Metadata), + }, []*plan.TableDef{storeTbl, metaTbl}, nil +} + +// buildFullText2Params records the parser name and max_index_capacity in +// algo_params (fulltext2 is always-async by algo, so no async/version params). +func buildFullText2Params(idx *tree.FullTextIndex) (string, error) { + res := make(map[string]string) + // Always record the parser (default "ngram" when unspecified), so algo_params + // is never an empty string — every consumer IndexParamsStringToMaps it (incl. + // the ALTER REINDEX flow in ddl.go, which errors on ""), and it self-documents + // the parser. Mirrors classic fulltext storing "parser" when set. + parser := "ngram" + if idx.IndexOption != nil && idx.IndexOption.ParserName != "" { + parser = strings.ToLower(idx.IndexOption.ParserName) + } + res["parser"] = parser + // Persist INCLUDE column names in algo_params so they survive catalog reload (the + // first-class IndexDef.IncludedColumns is repopulated from here via + // indexDefIncludedColumns / parseIncludedColumnsFromParams). Mirrors ivfpq's + // ParamsFromTree. Not rendered by IndexParamsToStringList (SHOW CREATE uses the field). + if idx.IndexOption != nil && len(idx.IndexOption.IncludeColumns) > 0 { + names := make([]string, 0, len(idx.IndexOption.IncludeColumns)) + for _, c := range idx.IndexOption.IncludeColumns { + if n := c.ColName(); n != "" { + names = append(names, n) + } + } + if len(names) > 0 { + marshaled, err := catalog.MarshalIncludeColumnsValue(names) + if err != nil { + return "", err + } + res[catalog.IncludedColumns] = marshaled + } + } + if idx.IndexOption != nil { + if idx.IndexOption.MaxIndexCapacity > 0 { + res[catalog.IndexAlgoParamMaxIndexCapacity] = strconv.FormatInt(idx.IndexOption.MaxIndexCapacity, 10) + } + // max_postings_capacity: postings (term occurrences) per built segment — the + // memory-correlated seal (a long-doc corpus seals on postings, not docs). Absence + // ⇒ DefaultPostingCapacity in the build paths. + if idx.IndexOption.MaxPostingsCapacity > 0 { + res[catalog.IndexAlgoParamMaxPostingsCapacity] = strconv.FormatInt(idx.IndexOption.MaxPostingsCapacity, 10) + } + // position_free ⇒ build without the positional payload (bag-of-words retrieval + // only). Recorded only when TRUE; absence ⇒ positional (phrase-capable). Only + // gojieba (word tokens) is meaningful position-free: ngram/json emit overlapping + // trigrams/values whose bag-of-words (IN BM25 MODE) result is noise — those need + // positions to reconstruct the term, so POSITION_FREE is rejected for them. + if idx.IndexOption.PositionFree { + if parser != "gojieba" { + return "", moerr.NewInvalidInputNoCtxf( + "fulltext2 POSITION_FREE requires WITH PARSER gojieba (got %q): a position-free index answers only IN BM25 MODE bag-of-words, and ngram/json tokens are meaningless without positions", parser) + } + res[catalog.IndexAlgoParamPositionFree] = "true" + } + // auto_update + day/hour/second drive the idxcron scheduled compaction + // cadence (read from algo_params by the idxcron executor). Mirrors bm25's + // ParamsFromTree. + if idx.IndexOption.AutoUpdate { + res[catalog.AutoUpdate] = "true" + } + if idx.IndexOption.Day > 0 { + res[catalog.Day] = strconv.FormatInt(idx.IndexOption.Day, 10) + } + if idx.IndexOption.Hour > 0 { + res[catalog.Hour] = strconv.FormatInt(idx.IndexOption.Hour, 10) + } + if idx.IndexOption.Second > 0 { + res[catalog.Second] = strconv.FormatInt(idx.IndexOption.Second, 10) + } + } + return catalog.IndexParamsMapToJsonString(res) +} diff --git a/pkg/fulltext2/plugin/plan/tablefunc.go b/pkg/fulltext2/plugin/plan/tablefunc.go new file mode 100644 index 0000000000000..e8c26c02b635b --- /dev/null +++ b/pkg/fulltext2/plugin/plan/tablefunc.go @@ -0,0 +1,159 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// FullText2SearchFuncName is the search TVF: MATCH over a fulltext2 index → +// ranked (doc_id, score). Registered into the plan-side TVF dispatch like the +// vector plugins' *_search. Args: [param, TableConfig(JSON), pattern]. +const FullText2SearchFuncName = "fulltext2_search" + +// Must match the executor (fulltext2_search.go): doc_id via AppendAny (the pk's own +// type — int/varchar/uuid/…, so T_any, NOT T_int64), score via AppendFixed[float32] +// (T_float32/Width 4, NOT T_float64 — an 8-byte score column read 4-byte writes as +// garbage). Same shape as the MATCH-rewrite path's ftIndexColdefs; a direct +// `FROM fulltext2_search(...)` call uses THESE defs, so they must be correct too. +var fulltext2SearchColDefs = []*plan.ColDef{ + {Name: catalog.FullText2Search_OutCol_DocId, Typ: plan.Type{Id: int32(types.T_any)}}, + {Name: catalog.FullText2Search_OutCol_Score, Typ: plan.Type{Id: int32(types.T_float32), Width: 4}}, +} + +// FullText2CreateFuncName is the build TVF: CROSS APPLY'd over the source table +// at CREATE INDEX / REINDEX time, it tokenizes each row (datalink/json/parser +// resolved in execution), builds capacity-bounded tag=0 base segments, and +// persists them. Args: [param, TableConfig(JSON), pk, cols...]. Output: a single +// discarded status row. +const FullText2CreateFuncName = "fulltext2_create" + +var fulltext2CreateColDefs = []*plan.ColDef{ + {Name: "status", Typ: plan.Type{Id: int32(types.T_int32), Width: 4}}, +} + +// FullText2CompactFuncName is the standalone MERGE-compaction TVF: +// `fulltext2_compact(db, store, meta, capacity[, position_free[, posting_capacity]])`. +// No driving table; args are passed as-is (no leading param strip, unlike +// search/create). Output: a single `live_docs` count row. +const FullText2CompactFuncName = "fulltext2_compact" + +var fulltext2CompactColDefs = []*plan.ColDef{ + {Name: "live_docs", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}}, +} + +func init() { + planplugin.RegisterTableFunc(FullText2SearchFuncName, buildFullText2Search) + planplugin.RegisterTableFunc(FullText2CreateFuncName, buildFullText2Create) + planplugin.RegisterTableFunc(FullText2CompactFuncName, buildFullText2Compact) +} + +// buildFullText2Create — arg list: [param, TableConfig(JSON), pk, cols...]. +func buildFullText2Create(pb planplugin.PlanBuilder, tbl *tree.TableFunction, ctx planplugin.BindContext, exprs []*plan.Expr, children []int32) (int32, error) { + if len(exprs) < 4 { + return 0, moerr.NewInvalidInput(pb.GetContext(), "fulltext2_create: invalid number of arguments (NARGS < 4)") + } + colDefs := planplugin.DeepCopyColDefList(fulltext2CreateColDefs) + params, err := getFullText2Params(pb, tbl.Func) + if err != nil { + return 0, err + } + exprs = exprs[1:] + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: FullText2CreateFuncName, + Param: []byte(params), + IsSingle: true, + }, + Cols: colDefs, + }, + BindingTags: []int32{pb.GenNewBindTag()}, + TblFuncExprList: exprs, + Children: children, + } + return pb.AppendNode(node, ctx), nil +} + +// buildFullText2Compact — the standalone `fulltext2_compact(db, store, meta, +// capacity, ...)` MERGE table function. No driving table; the varchar args are +// passed through as-is (no leading param strip). Mirrors the search/create +// registration so the plan-side dispatch routes through the plugin registry +// (query_builder.go) instead of a hardcoded switch case. +func buildFullText2Compact(pb planplugin.PlanBuilder, tbl *tree.TableFunction, ctx planplugin.BindContext, exprs []*plan.Expr, children []int32) (int32, error) { + colDefs := planplugin.DeepCopyColDefList(fulltext2CompactColDefs) + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: FullText2CompactFuncName, + }, + Cols: colDefs, + }, + BindingTags: []int32{pb.GenNewBindTag()}, + Children: children, + TblFuncExprList: exprs, + } + return pb.AppendNode(node, ctx), nil +} + +func getFullText2Params(pb planplugin.PlanBuilder, fn *tree.FuncExpr) (string, error) { + if _, ok := fn.Exprs[0].(*tree.NumVal); ok { + return fn.Exprs[0].String(), nil + } + return "", moerr.NewNoConfig(pb.GetContext(), "first parameter must be string") +} + +// buildFullText2Search — arg list: [param, TableConfig(JSON), pattern, mode]. +// param is stripped; the exec side (fulltext2_search.go) reads [cfg, pattern, mode]. +func buildFullText2Search(pb planplugin.PlanBuilder, tbl *tree.TableFunction, ctx planplugin.BindContext, exprs []*plan.Expr, children []int32) (int32, error) { + if len(exprs) != 4 { + return 0, moerr.NewInvalidInput(pb.GetContext(), "fulltext2_search: invalid number of arguments (NARGS != 4)") + } + colDefs := planplugin.DeepCopyColDefList(fulltext2SearchColDefs) + params, err := getFullText2Params(pb, tbl.Func) + if err != nil { + return 0, err + } + exprs = exprs[1:] + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: FullText2SearchFuncName, + Param: []byte(params), + }, + Cols: colDefs, + }, + BindingTags: []int32{pb.GenNewBindTag()}, + TblFuncExprList: exprs, + Children: children, + } + return pb.AppendNode(node, ctx), nil +} diff --git a/pkg/fulltext2/plugin/plugin.go b/pkg/fulltext2/plugin/plugin.go new file mode 100644 index 0000000000000..7df5102073c4a --- /dev/null +++ b/pkg/fulltext2/plugin/plugin.go @@ -0,0 +1,66 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package plugin is the fulltext2 index plugin registration point. +// +// fulltext2 is the WAND positional fulltext engine (NL phrase + boolean, TF-IDF +// and BM25), a DISTINCT algo from classic fulltext so its plugin hooks stay +// static (bm25-shaped: storage+metadata hidden tables, always-async CDC). It is +// created via `CREATE FULLTEXT2 INDEX name ON t(col) [WITH PARSER ]` (parsed +// to *tree.FullTextIndex with IsV2, dispatched through BuildFullTextIndexDefs) +// and queried via `MATCH(col) AGAINST('query')`. The engine lives in pkg/fulltext2. +package plugin + +import ( + "github.com/matrixorigin/matrixone/pkg/catalog" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" + idxcronplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/idxcron" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + + ft2compile "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/compile" + ft2idxcron "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/idxcron" + ft2plan "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/plan" + ft2runtime "github.com/matrixorigin/matrixone/pkg/fulltext2/plugin/runtime" +) + +// Plugin is the fulltext2 AlgoPlugin. +type Plugin struct { + catalogHooks catalogplugin.Hooks + compileHooks compileplugin.Hooks + planHooks planplugin.Hooks + idxcronHooks idxcronplugin.Hooks +} + +func New() *Plugin { + return &Plugin{ + catalogHooks: ft2runtime.CatalogHooks{}, + compileHooks: ft2compile.Hooks{}, + planHooks: ft2plan.Hooks{}, + idxcronHooks: ft2idxcron.Hooks{}, + } +} + +func (*Plugin) Algo() string { return catalog.MoIndexFullText2Algo.ToString() } +func (p *Plugin) Catalog() catalogplugin.Hooks { return p.catalogHooks } +func (p *Plugin) Compile() compileplugin.Hooks { return p.compileHooks } +func (p *Plugin) Plan() planplugin.Hooks { return p.planHooks } +func (p *Plugin) Idxcron() idxcronplugin.Hooks { return p.idxcronHooks } + +var _ indexplugin.AlgoPlugin = (*Plugin)(nil) + +// init registers fulltext2. CREATE FULLTEXT2 INDEX now creates the hidden tables; +// build-from-source / MATCH search / CDC / reindex land in the following steps. +func init() { indexplugin.Register(New()) } diff --git a/pkg/fulltext2/plugin/plugin_cover_test.go b/pkg/fulltext2/plugin/plugin_cover_test.go new file mode 100644 index 0000000000000..9ea3945c68b44 --- /dev/null +++ b/pkg/fulltext2/plugin/plugin_cover_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugin + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" + "github.com/stretchr/testify/require" +) + +// TestPluginHooks exercises New(), Algo() and every hook accessor on the +// fulltext2 Plugin, and confirms it satisfies the indexplugin.AlgoPlugin +// interface (the `var _ ...AlgoPlugin` assertion in plugin.go). +func TestPluginHooks(t *testing.T) { + p := New() + require.NotNil(t, p) + + // Algo string is the fulltext2 algo name. + require.Equal(t, catalog.MoIndexFullText2Algo.ToString(), p.Algo()) + require.Equal(t, "fulltext2", p.Algo()) + + // Every hook accessor returns a usable (non-nil interface) hook struct. + require.NotNil(t, p.Catalog()) + require.NotNil(t, p.Compile()) + require.NotNil(t, p.Plan()) + require.NotNil(t, p.Idxcron()) + + // The Plugin satisfies the AlgoPlugin interface (mirrors the compile-time + // `var _ indexplugin.AlgoPlugin = (*Plugin)(nil)` assertion). + var ap indexplugin.AlgoPlugin = p + require.Equal(t, p.Algo(), ap.Algo()) +} + +// TestPluginRegistered confirms init() registered the fulltext2 plugin so the +// framework can look it up by algo name. +func TestPluginRegistered(t *testing.T) { + got, ok := indexplugin.Get(catalog.MoIndexFullText2Algo.ToString()) + require.True(t, ok) + require.NotNil(t, got) + require.Equal(t, "fulltext2", got.Algo()) +} diff --git a/pkg/fulltext2/plugin/runtime/runtime.go b/pkg/fulltext2/plugin/runtime/runtime.go new file mode 100644 index 0000000000000..3001c072ad9c7 --- /dev/null +++ b/pkg/fulltext2/plugin/runtime/runtime.go @@ -0,0 +1,145 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runtime holds the fulltext2 index's catalog-side metadata. fulltext2 +// is a DISTINCT algo (CREATE FULLTEXT2 INDEX, the WAND positional engine), so — +// unlike a version-router — every hook here is a clean STATIC answer: the +// storage+metadata hidden-table layout (bm25-shaped), always-async CDC +// maintenance, and index_id-keyed clone/restore. +package runtime + +import ( + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +var _ catalogplugin.Hooks = CatalogHooks{} + +// CatalogHooks implements plugin/catalog.Hooks for fulltext2. +type CatalogHooks struct{} + +// HiddenTableTypes — a chunked segment store + a metadata table (no postings +// table; segments are built from source and CDC-maintained). +func (CatalogHooks) HiddenTableTypes() []string { + return []string{ + catalog.FullText2Index_TblType_Storage, + catalog.FullText2Index_TblType_Metadata, + } +} + +// ShouldTruncateHiddenTable — both hidden tables are derived from source rows and +// reset on TRUNCATE. +func (CatalogHooks) ShouldTruncateHiddenTable(_ string) bool { return true } + +// AlterTableCloneBehavior — both hidden tables are empty at CREATE-INDEX time +// (segments land via the build/CDC), so the whole index is skipped when cloning +// (rebuilt from source + CDC on the copy). +func (CatalogHooks) AlterTableCloneBehavior() catalogplugin.AlterTableCloneBehavior { + return catalogplugin.AlterTableCloneBehavior{SkipWholeIndex: true} +} + +// RestoreBehavior — the storage+metadata rows are index_id-keyed, so a clone +// overwrites rather than appends: no delete-before-clone. The compile hook's +// RestoreInitSQL registers CDC from the post-clone TS. +func (CatalogHooks) RestoreBehavior() catalogplugin.RestoreBehavior { + return catalogplugin.RestoreBehavior{} +} + +// BuildSessionVars — fulltext2 captures no session vars into algo_params. +// BuildSessionVars — fulltext2 pins lower_case_table_names into +// algo_params.session_vars so the background reindex folds identifiers in the +// rebuild SQL exactly as the user built under (its VALUE matters). sql_mode is +// intentionally omitted: a background reindex runs permissive and gets sql_mode +// defaulted at the reindex hook (sqlexec.Metadata.ResolveVariableWithSessionDefaults), +// not captured. Mirrors the vector-index plugins' capture list, and matches this +// plugin's IdxcronMetadata capture. +func (CatalogHooks) BuildSessionVars() []string { + return []string{"lower_case_table_names"} +} + +// DefaultOptions — no statement-level option JSON required. +func (CatalogHooks) DefaultOptions() map[string]string { return nil } + +// Fulltext2IndexFlag gates CREATE FULLTEXT2 INDEX behind the +// experimental_fulltext2_index session var (mirrors bm25's Bm25IndexFlag / +// HNSW's HnswIndexFlag). Both the framework gate (pkg/sql/compile/util.go, via +// ExperimentalFlag()) and the per-plugin HandleCreateIndex gate reference it. +const Fulltext2IndexFlag = "experimental_fulltext2_index" + +// ExperimentalFlag — CREATE FULLTEXT2 INDEX is gated by experimental_fulltext2_index. +func (CatalogHooks) ExperimentalFlag() string { return Fulltext2IndexFlag } + +// SupportedVectorTypes / SupportedOpTypes — fulltext2 has no vector/op-type concept. +func (CatalogHooks) SupportedVectorTypes() []types.T { return nil } +func (CatalogHooks) IsVectorIndex() bool { return false } // fulltext-family, not ANN +func (CatalogHooks) SupportedOpTypes() map[string]string { return nil } + +// SupportedIncludeColumnTypes lists the scalar column types accepted as INCLUDE columns — +// stored as their ACTUAL value in the segment docmap (not a hash), so both prefiltering and +// covering projection work. The set is EXACTLY what the segment pk codec +// (pkcodec.go encodePk/decodePk) already round-trips, so include values reuse that codec +// verbatim: the integer family (dense fixed-width) and varchar/char (varlena [len][value]). +// float32/64, date/decimal/text are intentionally deferred — pkcodec has no float case and +// widening the set is a codec extension, not a plan change. +func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { + return []types.T{ + types.T_int8, types.T_int16, types.T_int32, types.T_int64, + types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64, + types.T_varchar, types.T_char, + } +} + +// SupportedPrimaryKeyTypes lists EXACTLY the pk types the segment codec (pkcodec.go +// encodePk/decodePk) can round-trip. It is deliberately NOT nil ("any type"): a nil let +// a FLOAT / DOUBLE (or any other unsupported) primary key pass CREATE INDEX validation +// and then fail at build / CDC with "unsupported pk type" — an accepted DDL that dies at +// runtime. Keep this in lockstep with encodePk's switch so DDL validation and the codec +// agree. +func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { + return []types.T{ + types.T_int8, types.T_int16, types.T_int32, types.T_int64, + types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64, + types.T_bit, types.T_decimal64, types.T_decimal128, + types.T_date, types.T_datetime, types.T_time, types.T_timestamp, + types.T_char, types.T_varchar, types.T_text, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json, + types.T_datalink, types.T_uuid, + } +} + +// ValidQuantization — no quantization. +func (CatalogHooks) ValidQuantization(_, _ string) error { return nil } + +// ParamsFromTree — fulltext2 parses to *tree.FullTextIndex (handled by the plan +// hook's BuildFullTextIndexDefs), not *tree.Index, so this is never reached. +func (CatalogHooks) ParamsFromTree(_ *tree.Index) (map[string]string, error) { + return nil, moerr.NewNotSupportedNoCtx("fulltext2 index parses to *tree.FullTextIndex, not *tree.Index") +} + +// SyncDescriptor — fulltext2 is ALWAYS async (CDC-maintained). UsesCDC drives the +// ISCP consumer (RunFulltext2); the Idxcron fields drive the scheduled MERGE +// compaction (ALTER … REINDEX … FULLTEXT2 MERGE FORCE_SYNC). Mirrors bm25. +func (CatalogHooks) SyncDescriptor() catalogplugin.SyncDescriptor { + return catalogplugin.SyncDescriptor{ + UsesCDC: true, + SinkerType: catalogplugin.SinkerType_IndexSync, + AlwaysAsync: true, + IdxcronAction: "fulltext2_reindex", + IdxcronAlgoToken: "FULLTEXT2", + IdxcronReindexOption: "MERGE", + } +} diff --git a/pkg/fulltext2/plugin/runtime/runtime_test.go b/pkg/fulltext2/plugin/runtime/runtime_test.go new file mode 100644 index 0000000000000..836feabe9d56f --- /dev/null +++ b/pkg/fulltext2/plugin/runtime/runtime_test.go @@ -0,0 +1,105 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + "github.com/stretchr/testify/require" +) + +// TestCatalogHooks exercises every static accessor on CatalogHooks so the +// fulltext2 runtime metadata contract is pinned (storage+metadata hidden-table +// layout, always-async CDC maintenance, experimental gating, no vector/op/PK +// constraints, and the *tree.Index-parse NotSupported guard). +func TestCatalogHooks(t *testing.T) { + // Interface compliance is asserted in the source; re-assign here too. + var _ catalogplugin.Hooks = CatalogHooks{} + h := CatalogHooks{} + + // HiddenTableTypes — [Storage, Metadata]. + require.Equal(t, []string{ + catalog.FullText2Index_TblType_Storage, + catalog.FullText2Index_TblType_Metadata, + }, h.HiddenTableTypes()) + + // ShouldTruncateHiddenTable — always true for any input. + require.True(t, h.ShouldTruncateHiddenTable("")) + require.True(t, h.ShouldTruncateHiddenTable(catalog.FullText2Index_TblType_Storage)) + require.True(t, h.ShouldTruncateHiddenTable("anything")) + + // AlterTableCloneBehavior — whole index skipped on clone. + require.True(t, h.AlterTableCloneBehavior().SkipWholeIndex) + + // RestoreBehavior — zero value (no delete-before-clone). + require.Equal(t, catalogplugin.RestoreBehavior{}, h.RestoreBehavior()) + + // BuildSessionVars pins lower_case_table_names so the background reindex / + // ISCP build fold identifiers the way the index was created (mirrors the + // vector-index plugins); sql_mode is intentionally NOT captured (defaulted at + // the reindex hook). + require.Equal(t, []string{"lower_case_table_names"}, h.BuildSessionVars()) + + // Nil / empty accessors. + require.Nil(t, h.DefaultOptions()) + require.Nil(t, h.SupportedVectorTypes()) + require.Nil(t, h.SupportedOpTypes()) + // INCLUDE columns are supported (actual-value storage): the integer family + varchar/char, + // exactly the set the pk codec round-trips. Must EXCLUDE float (codec has no float case). + incTypes := h.SupportedIncludeColumnTypes() + require.NotEmpty(t, incTypes) + require.Contains(t, incTypes, types.T_int64) + require.Contains(t, incTypes, types.T_varchar) + require.NotContains(t, incTypes, types.T_float32) + // SupportedPrimaryKeyTypes mirrors the pk codec's domain (NOT nil "any type"): it + // includes int/varchar/uuid/temporal/decimal but must EXCLUDE FLOAT/DOUBLE, which the + // codec cannot encode — else a float pk would pass CREATE and fail at build. + pkTypes := h.SupportedPrimaryKeyTypes() + require.NotEmpty(t, pkTypes) + require.Contains(t, pkTypes, types.T_int64) + require.Contains(t, pkTypes, types.T_varchar) + require.NotContains(t, pkTypes, types.T_float32) + require.NotContains(t, pkTypes, types.T_float64) + + // IsVectorIndex — fulltext2 is a fulltext-family engine, NOT an ANN vector index; + // this is the capability behind indexplugin.IsVectorIndexAlgo, so a misreport here + // would let fulltext2 wrongly enter vector-index collection / ANN dispatch. + require.False(t, h.IsVectorIndex()) + + // ExperimentalFlag — the experimental_fulltext2_index gate. + require.Equal(t, Fulltext2IndexFlag, h.ExperimentalFlag()) + require.Equal(t, "experimental_fulltext2_index", h.ExperimentalFlag()) + + // ValidQuantization — no quantization, never errors. + require.NoError(t, h.ValidQuantization("", "")) + require.NoError(t, h.ValidQuantization("foo", "bar")) + + // ParamsFromTree — always NotSupported (fulltext2 parses *tree.FullTextIndex). + params, err := h.ParamsFromTree(nil) + require.Nil(t, params) + require.Error(t, err) + + // SyncDescriptor — always-async CDC + idxcron MERGE. + sd := h.SyncDescriptor() + require.True(t, sd.UsesCDC) + require.True(t, sd.AlwaysAsync) + require.Equal(t, catalogplugin.SinkerType_IndexSync, sd.SinkerType) + require.Equal(t, "fulltext2_reindex", sd.IdxcronAction) + require.Equal(t, "FULLTEXT2", sd.IdxcronAlgoToken) + require.Equal(t, "MERGE", sd.IdxcronReindexOption) +} diff --git a/pkg/fulltext2/pool.go b/pkg/fulltext2/pool.go new file mode 100644 index 0000000000000..dea1197fe3073 --- /dev/null +++ b/pkg/fulltext2/pool.go @@ -0,0 +1,117 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "sync" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// Per-query cursor block buffers are BlockSize-sized and short-lived: buildWandIters +// makes one pair per term and newPhraseCursor one pair per slot, then they die when +// the search returns. Under concurrent query load that churn outruns the GC, so the +// buffers are recycled through sync.Pool instead of freshly allocated per query. Each +// buffer is Get by exactly one (single-goroutine) search and Put back on return; the +// returned Result/docTf values copy out pk+score and never alias the buffers, so there +// is no use-after-Put. Reused buffers may hold stale entries past the current block's +// blen, but cursors only ever read within [0,blen), so staleness is invisible. + +// wandBuf is a wandIter's decoded-block buffer pair (docIDs + capped tfs). +type wandBuf struct { + docs []int64 // cap BlockSize + tfs []uint8 // cap BlockSize +} + +var wandBufPool = sync.Pool{ + New: func() any { + return &wandBuf{docs: make([]int64, BlockSize), tfs: make([]uint8, BlockSize)} + }, +} + +func getWandBuf() *wandBuf { return wandBufPool.Get().(*wandBuf) } + +// releaseWandIters returns every cursor's block buffer to the pool. Safe to defer right +// after buildWandIters (a nil/empty slice is a no-op) and idempotent per cursor (buf is +// cleared so a double release cannot double-Put the same buffer). +func releaseWandIters(iters []*wandIter) { + for _, it := range iters { + if it == nil || it.buf == nil { + continue + } + wandBufPool.Put(it.buf) + it.buf = nil + it.bDocs = nil + it.bTfs = nil + } +} + +// phraseBuf is a phraseCursor's decoded-block buffers: docIDs + per-doc positions. +type phraseBuf struct { + docs []int64 + pos [][]int32 +} + +var phraseBufPool = sync.Pool{ + New: func() any { + return &phraseBuf{ + docs: make([]int64, BlockSize), + pos: make([][]int32, BlockSize), + } + }, +} + +func getPhraseBuf() *phraseBuf { return phraseBufPool.Get().(*phraseBuf) } + +// releasePhraseCursors returns every cursor's block buffers to the pool; skips nil +// entries so it is safe to defer over a partially-built cursors slice (matchPhraseCursor +// bails out mid-build when a slot's term is absent). +func releasePhraseCursors(cursors []*phraseCursor) { + for _, c := range cursors { + if c == nil || c.buf == nil { + continue + } + phraseBufPool.Put(c.buf) + c.buf = nil + c.bDocs = nil + c.bPos = nil + } +} + +// The no-LIMIT streaming path emits one vectorindex.ColumnBuffer per streamBatch rows. +// Its Data backing is recycled here so a million-row result does not allocate a fresh +// buffer per batch. GC-managed: a buffer never Put back (an aborted/drained query) is +// just reclaimed — a missed reuse, not a leak. The struct lives in vectorindex (the Emit +// contract references it); the POOLING POLICY lives here with its only user, fulltext2. +var columnBufferPool = sync.Pool{New: func() any { return &vectorindex.ColumnBuffer{} }} + +// GetColumnBuffer returns a reset ColumnBuffer of pk type t. The streaming producer Gets +// one per batch; the consumer PutColumnBuffers it back once it has copied Data out. +func GetColumnBuffer(t types.T) *vectorindex.ColumnBuffer { + k := columnBufferPool.Get().(*vectorindex.ColumnBuffer) + k.Type = t + k.Reset() + return k +} + +// PutColumnBuffer returns a ColumnBuffer to the pool. Call only after the batch is fully +// consumed (its Data copied out); the buffer must not be referenced afterward. Exported +// because the consumer (pkg/sql/colexec/table_function) returns the batch it received. +func PutColumnBuffer(k *vectorindex.ColumnBuffer) { + if k != nil { + columnBufferPool.Put(k) + } +} diff --git a/pkg/fulltext2/pool_bench_test.go b/pkg/fulltext2/pool_bench_test.go new file mode 100644 index 0000000000000..d31cedfe34d00 --- /dev/null +++ b/pkg/fulltext2/pool_bench_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// benchWandSegment builds a loaded segment with `nterms` distinct terms Zipf-ish +// spread across `ndocs` docs (so several terms span many blocks), then returns the +// deserialized (mmap-shaped) segment the query path actually runs against. +func benchWandSegment(b *testing.B, ndocs, nterms int) (*Segment, []clause) { + b.Helper() + terms := make([]string, nterms) + for i := range terms { + terms[i] = "term" + string(rune('a'+i%26)) + string(rune('0'+i/26)) + } + rng := rand.New(rand.NewSource(11)) + bld := NewBuilder("bench", int32(types.T_int64)) + for d := 0; d < ndocs; d++ { + pos := int32(0) + // each doc gets a handful of terms; earlier terms are more common (block spread). + for w := 0; w < 6; w++ { + ti := int(rng.ExpFloat64()*float64(nterms)/4) % nterms + if ti < 0 { + ti = -ti + } + if e := bld.Add(terms[ti], pos, int64(d)); e != nil { + b.Fatal(e) + } + pos += int32(len(terms[ti])) + 1 + } + } + seg, err := bld.Finish() + if err != nil { + b.Fatal(err) + } + blob, err := seg.Serialize() + if err != nil { + b.Fatal(err) + } + loaded, err := Deserialize("bench", bytes.NewReader(blob)) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = loaded.dict.Close() }) + + // a 4-term disjunction (4 wandIters => 4 pooled block-buffer pairs per query). + cs := []clause{ + {kind: clauseTerm, terms: []string{terms[0]}, weight: 1}, + {kind: clauseTerm, terms: []string{terms[1]}, weight: 1}, + {kind: clauseTerm, terms: []string{terms[2]}, weight: 1}, + {kind: clauseTerm, terms: []string{terms[3]}, weight: 1}, + } + return loaded, cs +} + +// BenchmarkSearchWANDAlloc reports allocs/op for the full searchWAND path. With the +// sync.Pool for cursor block buffers, the per-query BlockSize []int64/[]uint8 pairs are +// recycled, so they do NOT show up as steady-state allocations. +func BenchmarkSearchWANDAlloc(b *testing.B) { + seg, cs := benchWandSegment(b, 20000, 40) + gs := (&Index{segments: []*Segment{seg}, globalN: seg.N}).newGlobalStats() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = seg.searchWAND(cs, BM25, 10, nil, gs) + } +} + +// BenchmarkSearchWANDParallel exercises the pool under concurrency — the case where +// per-query buffer churn would otherwise outrun the GC. -race over this proves the +// Get/Put lifecycle has no cross-goroutine buffer sharing. +func BenchmarkSearchWANDParallel(b *testing.B) { + seg, cs := benchWandSegment(b, 20000, 40) + gs := (&Index{segments: []*Segment{seg}, globalN: seg.N}).newGlobalStats() + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _ = seg.searchWAND(cs, BM25, 10, nil, gs) + } + }) +} + +// BenchmarkStreamAlloc reports allocs/op for the no-LIMIT streaming path. With typed +// ColumnBuffer the per-pk boxing is gone: a batch allocates only its Data/scores buffers +// (a few allocs per 8192 rows) instead of one boxed interface per doc. +func BenchmarkStreamAlloc(b *testing.B) { + seg, cs := benchWandSegment(b, 20000, 40) + idx := NewIndex([]*Segment{seg}, nil) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + sink := newStreamSink(idx, false, nil, func(o *vectorindex.SearchOutput) error { PutColumnBuffer(o.Keys); return nil }) + _ = idx.streamDisjunction(cs, BM25, nil, sink) + } +} diff --git a/pkg/fulltext2/positionfree_reconstruct_test.go b/pkg/fulltext2/positionfree_reconstruct_test.go new file mode 100644 index 0000000000000..7da66ca9782e5 --- /dev/null +++ b/pkg/fulltext2/positionfree_reconstruct_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestReconstructLiveDocsPositionFree: a position-free segment has no positions, so +// ReconstructLiveDocs(true) must rebuild each doc's terms from tf (each term tf times), +// preserving the term SET + tf so a position-free MERGE keeps bag-of-words results. +func TestReconstructLiveDocsPositionFree(t *testing.T) { + b := NewBuilder("pf", int32(types.T_int64), WithPositionFree()) + feed(t, b, int64(0), "apple", "pie", "apple") // apple x2, pie x1 + feed(t, b, int64(1), "banana", "bread") + seg := loadedSeg(t, b) + idx := NewIndex([]*Segment{seg}, nil) + + docs, err := collectLiveDocs(idx, true) + require.NoError(t, err) + + got := map[any]map[string]int{} + for _, d := range docs { + m := map[string]int{} + for _, w := range d.Terms { + m[w]++ + } + got[normalizeKey(d.Pk)] = m + } + require.Equal(t, map[string]int{"apple": 2, "pie": 1}, got[normalizeKey(int64(0))]) + require.Equal(t, map[string]int{"banana": 1, "bread": 1}, got[normalizeKey(int64(1))]) + + // Rebuild position-free from the reconstruction → bag-of-words queries still work. + nb := NewBuilder("merged", int32(types.T_int64), WithPositionFree()) + for _, d := range docs { + for i, w := range d.Terms { + require.NoError(t, nb.Add(w, d.Positions[i], d.Pk)) + } + } + merged, err := nb.Finish() + require.NoError(t, err) + midx := NewIndex([]*Segment{merged}, nil) + res, err := midx.SearchBagOfWords([]byte("apple"), ParserDefault, BM25, 10, nil) + require.NoError(t, err) + require.Equal(t, []any{int64(0)}, resultIDs(res)) +} diff --git a/pkg/fulltext2/positionfree_test.go b/pkg/fulltext2/positionfree_test.go new file mode 100644 index 0000000000000..9d177c02d3f48 --- /dev/null +++ b/pkg/fulltext2/positionfree_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "fmt" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// vocabWord is an all-alphabetic term (so the query tokenizer maps it 1:1). +func vocabWord(i int) string { return fmt.Sprintf("word%c%c", 'a'+byte(i/26), 'a'+byte(i%26)) } + +// TestPositionFreeFootprint: WithPositionFree() drops the positional payload while +// keeping the FST + docID/tf postings, so the segment serializes smaller and still +// answers bag-of-words (WAND) queries with the same doc set + tf-based ranking. +func TestPositionFreeFootprint(t *testing.T) { + const ( + ndoc = 300 + ntok = 60 + nterm = 40 // words 0..19 appear twice/doc, 20..39 once — every word in every doc + ) + build := func(opts ...BuildOpt) *Segment { + b := NewBuilder("idx", int32(types.T_int64), opts...) + for d := 0; d < ndoc; d++ { + for i := 0; i < ntok; i++ { + require.NoError(t, b.Add(vocabWord(i%nterm), int32(i), int64(d))) + } + } + seg, err := b.Finish() + require.NoError(t, err) + return seg + } + + full := build() + pf := build(WithPositionFree()) + + fullBytes, err := full.Serialize() + require.NoError(t, err) + pfBytes, err := pf.Serialize() + require.NoError(t, err) + + saved := 100 * float64(len(fullBytes)-len(pfBytes)) / float64(len(fullBytes)) + t.Logf("serialized: full=%d bytes position-free=%d bytes saved=%.1f%%", + len(fullBytes), len(pfBytes), saved) + require.Less(t, len(pfBytes), len(fullBytes), "position-free segment must be smaller") + + // The position-free segment round-trips and answers a bag-of-words WAND query + // (boolean bare term → searchWAND, which never touches positions) with the full + // doc set — proving docID/tf postings and the FST survive. + loaded, err := Deserialize("idx", bytes.NewReader(pfBytes)) + require.NoError(t, err) + idx := NewIndex([]*Segment{loaded}, nil) + + res, err := idx.SearchQuery([]byte(vocabWord(1)), true, ParserDefault, BM25, ndoc+10, nil) + require.NoError(t, err) + require.Len(t, res, ndoc, "bag-of-words query must match every doc on the position-free index") + + // Parity: the full (positional) segment returns the identical doc set for the + // same bag-of-words query. + fidx := NewIndex([]*Segment{full}, nil) + fres, err := fidx.SearchQuery([]byte(vocabWord(1)), true, ParserDefault, BM25, ndoc+10, nil) + require.NoError(t, err) + require.ElementsMatch(t, resultIDs(fres), resultIDs(res)) +} + +// TestSearchBagOfWordsCJK: the IN BM25 MODE engine path (SearchBagOfWords) turns a CJK +// query into a disjunction of its gojieba tokens, so it returns the BAG-OF-WORDS +// superset — every doc containing any of 我家/有/三个/人 — whereas NL/phrase mode +// returns only the exact contiguous phrase. (Mirrors the SQL pre-flight.) +func TestSearchBagOfWordsCJK(t *testing.T) { + docs := []Doc{ + {Pk: int64(1), Text: []byte("我家有三个人")}, + {Pk: int64(2), Text: []byte("三个人都住在我家")}, + {Pk: int64(3), Text: []byte("我家的花园很漂亮")}, + {Pk: int64(4), Text: []byte("教室里有三个人")}, + {Pk: int64(5), Text: []byte("昨天我家有三个人来吃饭")}, + } + seg, err := BuildSegmentFromDocsParser("zh", int32(types.T_int64), docs, ParserGojieba) + require.NoError(t, err) + idx := NewIndex([]*Segment{seg}, nil) + + phrase, err := idx.SearchQuery([]byte("我家有三个人"), false, ParserGojieba, BM25, 10, nil) + require.NoError(t, err) + phraseIDs := resultIDs(phrase) + + bow, err := idx.SearchBagOfWords([]byte("我家有三个人"), ParserGojieba, BM25, 10, nil) + require.NoError(t, err) + bowIDs := resultIDs(bow) + + t.Logf("phrase(exact)=%v bag-of-words=%v", phraseIDs, bowIDs) + require.Contains(t, phraseIDs, int64(1), "doc 1 is the exact phrase") + require.Subset(t, bowIDs, phraseIDs, "bag-of-words must be a superset of the exact phrase") + require.Greater(t, len(bowIDs), len(phraseIDs), "bag-of-words must match strictly more docs than the phrase") +} diff --git a/pkg/fulltext2/query.go b/pkg/fulltext2/query.go new file mode 100644 index 0000000000000..2e0925183797c --- /dev/null +++ b/pkg/fulltext2/query.go @@ -0,0 +1,678 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/bytejson" + "github.com/matrixorigin/matrixone/pkg/fulltext" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// BooleanTreeString parses a boolean-mode query and returns its parse tree as an +// S-expression (e.g. `+Matrix -Origin` → `(+ (text 0 matrix)) (- (text 1 origin))`). +// A debug aid: it reuses classic fulltext's PatternToString so the printed shape +// is identical to pkg/fulltext (fulltext2's boolean structure is parity with it). +func BooleanTreeString(query string) (string, error) { + return fulltext.PatternToString(query, int64(tree.FULLTEXT_BOOLEAN)) +} + +// BooleanTreeStringWithPosition is BooleanTreeString with each leaf's byte +// position included: `(text )` — mirrors classic +// fulltext's PatternToStringWithPosition (used for phrase-position debugging). +func BooleanTreeStringWithPosition(query string) (string, error) { + return fulltext.PatternToStringWithPosition(query, int64(tree.FULLTEXT_BOOLEAN)) +} + +// NLTreeString parses a natural-language query and returns its parse tree (the +// ngram/word patterns AGAINST(...) expands to), reusing classic fulltext's +// PatternToString. Debug aid for the NL side. +func NLTreeString(query string) (string, error) { + return fulltext.PatternToString(query, int64(tree.FULLTEXT_NL)) +} + +// NLTreeStringWithPosition is NLTreeString with each leaf's byte position: +// `(text )` — mirrors PatternToStringWithPosition. +func NLTreeStringWithPosition(query string) (string, error) { + return fulltext.PatternToStringWithPosition(query, int64(tree.FULLTEXT_NL)) +} + +// Parser selects how documents and queries are tokenized. fulltext2 mirrors +// classic fulltext's parser set: +// - default / ngram : SimpleTokenizer — whole words for Latin, sliding +// 3-grams for CJK (the classic build-side ngram tokenizer). +// - gojieba : dictionary word segmentation (CJK + Latin) via jieba. +// - json / json_value: json values are flattened to text, then tokenized as +// ngram; the query pattern is plain text (also ngram). +const ( + ParserDefault = "default" + ParserNgram = "ngram" + ParserGojieba = "gojieba" + ParserJSON = "json" + ParserJSONValue = "json_value" + + ngramSize = 3 +) + +// normalizeParser lowercases/trims and maps "" to the default parser. +func normalizeParser(p string) string { + p = strings.ToLower(strings.TrimSpace(p)) + if p == "" { + return ParserDefault + } + return p +} + +// hasCJK reports whether s contains a CJK-class rune — a non-Latin, non-breaker +// rune, matching SimpleTokenizer's own CJK classification (isLatin: rune<0x7FF). +// A bare CJK boolean operand is routed to the positional (byte-offset) phrase path +// so a short compound (e.g. 中文, a 2-char bigram absent as a whole token from an +// ngram-3 index) matches via a word* prefix rather than an empty exact-term lookup. +func hasCJK(s string) bool { + for _, r := range s { + if r > 0x7FF && !unicode.IsPunct(r) && !unicode.IsSpace(r) { + return true + } + } + return false +} + +// DocTokenizer returns the tokenizer that indexes DOCUMENTS under parser. +// json/json_value documents must be flattened first (FlattenJSON); their +// flattened text is tokenized as ngram. +func DocTokenizer(parser string) (tokenizer.Tokenizer, error) { + switch normalizeParser(parser) { + case ParserDefault, ParserNgram, ParserJSON, ParserJSONValue: + return tokenizer.NewSimpleTokenizer(), nil + case ParserGojieba: + return tokenizer.SharedJiebaTokenizer(false) + default: + return nil, moerr.NewInternalErrorNoCtx("fulltext2: invalid parser " + parser) + } +} + +// queryTokenizer returns the tokenizer for a QUERY pattern under parser. It is +// SimpleTokenizer for every parser except gojieba — a json index is queried with +// plain text, so json uses the ngram tokenizer at query time. +func queryTokenizer(parser string) (tokenizer.Tokenizer, error) { + if normalizeParser(parser) == ParserGojieba { + return tokenizer.SharedJiebaTokenizer(false) + } + return tokenizer.NewSimpleTokenizer(), nil +} + +// flattenJSONValues parses raw as a TEXT JSON document and returns its leaf +// values joined by spaces. Convenience for text/varchar json columns and tests. +func flattenJSONValues(raw []byte) ([]byte, error) { + return FlattenJSON(raw, false) +} + +// FlattenJSON returns a JSON document's leaf values joined by spaces, so an ngram +// tokenizer can index them as ordinary text. binary selects the encoding: true +// for a T_json column's stored bytes (bytejson.Unmarshal), false for text/varchar +// json (bytejson.ParseFromString). +func FlattenJSON(raw []byte, binary bool) ([]byte, error) { + var bj bytejson.ByteJson + var err error + if binary { + err = bj.Unmarshal(raw) + } else { + bj, err = bytejson.ParseFromString(string(raw)) + } + if err != nil { + return nil, err + } + return flattenBJ(bj), nil +} + +// flattenBJ joins a parsed json document's leaf values with spaces (shared by +// FlattenJSON and FlattenJSONColumn). +func flattenBJ(bj bytejson.ByteJson) []byte { + var b strings.Builder + for tk := range bj.TokenizeValue(false) { + n := int(tk.TokenBytes[0]) + if b.Len() > 0 { + b.WriteByte(' ') + } + b.Write(tk.TokenBytes[1 : 1+n]) + } + return []byte(b.String()) +} + +// FlattenJSONColumn flattens ONE indexed column's value to its space-joined leaf +// values for a json parser — the CDC-write-side analogue of the create-TVF's +// per-column flatten (rowTerms). A T_json column arrives as an already-parsed +// bytejson.ByteJson (types.DecodeJson, repr-independent); a text/varchar json column +// arrives as a string/[]byte of json TEXT. Doing this per column (vs joining raw +// columns then flattening the blob once) is what makes CDC json tokens match CREATE: +// the old joined-then-flattened path yielded ZERO tokens for T_json and for +// multi-column json, silently making CDC-inserted rows unsearchable. +func FlattenJSONColumn(v any) ([]byte, error) { + switch t := v.(type) { + case bytejson.ByteJson: + return flattenBJ(t), nil + case []byte: + return FlattenJSON(t, false) + case string: + return FlattenJSON([]byte(t), false) + default: + return nil, nil + } +} + +// IsJSONParser reports whether parser is one of the json variants. +func IsJSONParser(parser string) bool { + p := normalizeParser(parser) + return p == ParserJSON || p == ParserJSONValue +} + +// IsJSONValueParser reports whether parser is the json_value variant. json_value +// indexes each json leaf value as ONE whole atomic token and matches it EXACTLY +// (classic fulltext: tokenize.go stores the whole value, sql.go:173 queries +// `word = value`) — distinct from json, which ngrams over the flattened values. +func IsJSONValueParser(parser string) bool { + return normalizeParser(parser) == ParserJSONValue +} + +// flattenBJLines joins a parsed json document's leaf values with NEWLINES (not +// spaces): the json_value parser keeps each whole value as a separable atomic token, +// so a '\n' separator lets JSONValueTokenize recover the values (a space join would +// fuse values and lose the boundaries — and values may themselves contain spaces). +func flattenBJLines(bj bytejson.ByteJson) []byte { + var b strings.Builder + for tk := range bj.TokenizeValue(false) { + n := int(tk.TokenBytes[0]) + if b.Len() > 0 { + b.WriteByte('\n') + } + b.Write(tk.TokenBytes[1 : 1+n]) + } + return []byte(b.String()) +} + +// FlattenJSONLines is FlattenJSON for the json_value parser: leaf values joined by +// '\n' so each stays a separable whole token. binary selects the encoding as in +// FlattenJSON. +func FlattenJSONLines(raw []byte, binary bool) ([]byte, error) { + var bj bytejson.ByteJson + var err error + if binary { + err = bj.Unmarshal(raw) + } else { + bj, err = bytejson.ParseFromString(string(raw)) + } + if err != nil { + return nil, err + } + return flattenBJLines(bj), nil +} + +// FlattenJSONValueColumn is FlattenJSONColumn for the json_value parser: ONE indexed +// column's leaf values joined by '\n' (the CDC-write-side analogue of the create-TVF's +// per-column flatten). Mirrors FlattenJSONColumn's input-type handling. +func FlattenJSONValueColumn(v any) ([]byte, error) { + switch t := v.(type) { + case bytejson.ByteJson: + return flattenBJLines(t), nil + case []byte: + return FlattenJSONLines(t, false) + case string: + return FlattenJSONLines([]byte(t), false) + default: + return nil, nil + } +} + +// JSONValueTokenize splits '\n'-joined json values into whole (word, byte-position) +// tokens — the json_value parser's atomic-value tokenization: no ngram, no +// case-folding, punctuation kept, so `red..blue--brown` and `中文學習教材` each index +// as ONE token that a query matches exactly (classic fulltext json_value parity). +func JSONValueTokenize(text []byte) []WordPos { + var words []WordPos + for i := 0; i < len(text); { + j := i + for j < len(text) && text[j] != '\n' { + j++ + } + if j > i { + words = append(words, WordPos{Word: string(text[i:j]), Pos: int32(i)}) + } + i = j + 1 + } + return words +} + +// CdcTokenizer returns the parser-aware tokenize closure the CDC consumer feeds to +// TailBuilder — ordered words per parser. For a json parser the CDC WRITER +// (Fulltext2SqlWriter.rowText) already flattens each json column to its values, +// binary-aware and PER COLUMN, exactly as the create-TVF's rowTerms does — so the CDC +// text is plain flattened text here and is tokenized as ngram, NOT flattened again. +// (The old form flattened the whole joined multi-column blob once with binary=false, +// which produced zero tokens for a T_json column and mis-parsed multi-column json — so +// CDC-inserted json rows were unsearchable.) +func CdcTokenizer(parser string) (func(string) []WordPos, error) { + if IsJSONValueParser(parser) { + // json_value: the CDC writer (rowText) already flattened each json column to its + // '\n'-joined whole values, so here we just recover them as atomic tokens — NOT + // ngram — matching the create/base build. + return func(text string) []WordPos { return JSONValueTokenize([]byte(text)) }, nil + } + tok, err := DocTokenizer(parser) + if err != nil { + return nil, err + } + return func(text string) []WordPos { return tokenizeWords(tok, []byte(text)) }, nil +} + +// WordPos is a tokenized word with its BYTE position in the source text — carried +// through the CDC/merge build so positional phrase queries match by byte offset. +type WordPos struct { + Word string + Pos int32 +} + +// tokenizeWords flattens a tokenizer stream into ordered (word, byte-position) pairs. +func tokenizeWords(tok tokenizer.Tokenizer, text []byte) []WordPos { + // token count is unknown up front; estimate from input length to avoid regrowth. + words := make([]WordPos, 0, len(text)/4) + for tk, err := range tok.Tokenize(text) { + if err != nil { + break + } + words = append(words, WordPos{tokenWord(tk), tk.BytePos}) + } + return words +} + +// BuildSegmentFromDocsParser builds a segment from docs under parser, selecting +// the right document tokenizer and flattening json docs first. It is the +// parser-aware build entry (compile-side build-from-source uses it). +func BuildSegmentFromDocsParser(id string, pkType int32, docs []Doc, parser string) (*Segment, error) { + p := normalizeParser(parser) + if p == ParserJSONValue { + // json_value: each leaf value is one whole atomic token (no ngram). Build the + // tokenized docs directly from the '\n'-joined values so build matches query. + tds := make([]TokenizedDoc, 0, len(docs)) + for _, d := range docs { + lines, ferr := FlattenJSONLines(d.Text, false) + if ferr != nil { + return nil, ferr + } + td := TokenizedDoc{Pk: d.Pk} + for _, wp := range JSONValueTokenize(lines) { + td.Terms = append(td.Terms, wp.Word) + td.Positions = append(td.Positions, wp.Pos) + } + tds = append(tds, td) + } + return BuildSegmentFromTokenized(id, pkType, tds) + } + tok, err := DocTokenizer(parser) + if err != nil { + return nil, err + } + if p == ParserJSON { + fdocs := make([]Doc, 0, len(docs)) + for _, d := range docs { + ft, ferr := flattenJSONValues(d.Text) + if ferr != nil { + return nil, ferr + } + fdocs = append(fdocs, Doc{Pk: d.Pk, Text: ft}) + } + docs = fdocs + } + return BuildSegmentFromDocs(id, pkType, docs, tok) +} + +// SearchQuery runs a MATCH query over the index under parser and mode. It is the +// single query entry the fulltext2 search TVF calls. +// +// NL mode (boolean=false): an EXACT ORDERED PHRASE for EVERY parser — the terms +// must occur contiguously and in order (classic fulltext NL semantics). There is +// NO bag-of-words path: a CJK query is matched positionally by BYTE offset over +// classic fulltext's redundant-ngram-removed slots (full trigrams as TEXT, short +// tails as word* prefixes), so the phrase is robust despite overlapping/tail 3-grams. +// +// Boolean mode (boolean=true): standard +/-/>/= 3 chars), so matching is exact-term lookups + byte-position adjacency instead of the +// old ParsePatternInNLMode prefix decomposition, whose word* expansion + full-position +// materialization OOM-killed the CN at scale. Offsets are relative to the first slot. +func phraseSlots(pattern, parser string) ([]phraseSlot, error) { + if IsJSONValueParser(parser) { + // json_value: the whole pattern is ONE atomic value matched exactly (no ngram) — + // a single exact-term slot, mirroring classic `word = value`. + if pattern == "" { + return nil, nil + } + return []phraseSlot{{term: pattern, off: 0}}, nil + } + if normalizeParser(parser) == ParserGojieba { + tok, err := queryTokenizer(ParserGojieba) + if err != nil { + return nil, err + } + return tokenizePhraseSlots(tok, []byte(pattern)), nil + } + return ngramPhraseSlots(pattern), nil +} + +// isBreakerRune / isLatinRune mirror SimpleTokenizer's run classification (simple.go) so +// the query decomposition segments runs exactly as the document tokenizer does. +func isBreakerRune(r rune) bool { + if r < 128 { + return !((r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z')) + } + return unicode.IsPunct(r) || unicode.IsSpace(r) +} + +func isLatinRune(r rune) bool { return r < 0x7FF } + +// ngramPhraseSlots decomposes an ngram-parser query into positional phrase slots that +// match the SimpleTokenizer document index WITHOUT word* prefixes for CJK runs >= 3 +// chars. A CJK run is covered by EXACT full trigrams at char step ngramSize PLUS a final +// trigram that ends at the run's end (the last trigram is shifted back to cover the tail), +// so byte-position adjacency alone pins the whole run — the overlapping/adjacent trigrams +// uniquely determine the substring. Latin runs become whole (lowercased) word slots. Only +// a CJK run SHORTER than ngramSize (1-2 chars, e.g. a bare 2-char term) has no trigram and +// falls back to a single prefix (star) slot, handled presence-only at match time. +func ngramPhraseSlots(pattern string) []phraseSlot { + runes := []rune(pattern) + n := len(runes) + if n == 0 { + return nil + } + bpos := make([]int32, n+1) // byte offset of each rune index + b := int32(0) + for i, r := range runes { + bpos[i] = b + b += int32(utf8.RuneLen(r)) + } + bpos[n] = b + + var slots []phraseSlot + i := 0 + for i < n { + if isBreakerRune(runes[i]) { + i++ + continue + } + if isLatinRune(runes[i]) { + j := i + for j < n && !isBreakerRune(runes[j]) && isLatinRune(runes[j]) { + j++ + } + slots = append(slots, phraseSlot{term: strings.ToLower(string(runes[i:j])), off: bpos[i]}) + i = j + continue + } + // CJK run [i, j) + j := i + for j < n && !isBreakerRune(runes[j]) && !isLatinRune(runes[j]) { + j++ + } + if j-i < ngramSize { + // 1-2 char CJK run: no trigram exists → prefix (rare; presence-only in matchPhrase) + slots = append(slots, phraseSlot{term: string(runes[i:j]), star: true, off: bpos[i]}) + i = j + continue + } + last := j - ngramSize // start index of the trigram ending at the run's end + for c := i; c <= last; c += ngramSize { + slots = append(slots, phraseSlot{term: string(runes[c : c+ngramSize]), off: bpos[c]}) + } + if (last-i)%ngramSize != 0 { // tail not covered by the step-ngramSize loop → add the final trigram + slots = append(slots, phraseSlot{term: string(runes[last : last+ngramSize]), off: bpos[last]}) + } + i = j + } + if len(slots) == 0 { + return nil + } + base := slots[0].off + for k := range slots { + slots[k].off -= base + } + return slots +} + +// slotsKey is a deterministic string key for a phrase (term+star+offset per slot), used +// to memoize phrase hits/df per query. +func slotsKey(slots []phraseSlot) string { + var b strings.Builder + for _, s := range slots { + b.WriteString(s.term) + if s.star { + b.WriteByte('*') + } + b.WriteByte(0) + b.WriteString(strconv.Itoa(int(s.off))) + b.WriteByte(0) + } + return b.String() +} + +// tokenizePhraseSlots tokenizes text into (word, byte-position) TEXT slots — for gojieba +// NL and boolean-mode explicit "phrases". +func tokenizePhraseSlots(tok tokenizer.Tokenizer, text []byte) []phraseSlot { + wps := tokenizeWords(tok, text) + if len(wps) == 0 { + return nil + } + base := wps[0].Pos + slots := make([]phraseSlot, len(wps)) + for i, wp := range wps { + slots[i] = phraseSlot{term: wp.Word, off: wp.Pos - base} + } + return slots +} + +// buildBooleanQuery parses a boolean-mode query parser-aware: gojieba/Latin +// operands tokenize into word phrases/terms; a bare CJK operand under ngram/json +// is an exact positional (byte-offset) phrase, matching it contiguously — NOT an +// OR-group of its ngrams (which would over-match any doc sharing a single ngram). +func buildBooleanQuery(query, parser string) (BoolQuery, error) { + var q BoolQuery + for _, rc := range scanBoolean(query) { + c, ok, err := rawToClauseParser(rc, parser) + if err != nil { + return q, err + } + if !ok { + continue + } + switch bucketOf(rc.prefix) { + case bMust: + q.must = append(q.must, c) + case bMustNot: + q.mustNot = append(q.mustNot, c) + case bAdjust: + q.adjust = append(q.adjust, c) + default: + q.should = append(q.should, c) + } + } + return q, nil +} + +// rawToClauseParser is the parser-aware form of rawToClause (boolean.go): a bare +// CJK operand under ngram/json becomes an exact positional phrase (byte-offset +// slots); quoted phrases, prefixes, groups and gojieba/Latin operands behave as +// in rawToClause. +func rawToClauseParser(rc rawClause, parser string) (clause, bool, error) { + w := weightOf(rc.prefix) + if rc.group { + var children []clause + for _, ch := range rc.children { + cc, ok, err := rawToClauseParser(ch, parser) + if err != nil { + return clause{}, false, err + } + if ok { + children = append(children, cc) + } + } + if len(children) == 0 { + return clause{}, false, nil + } + return clause{kind: clauseGroup, children: children, weight: w}, true, nil + } + + // json_value: the whole operand is ONE atomic value matched exactly (no ngram, no + // tokenization) — classic fulltext queries `word = operand` (sql.go:173). A quoted + // operand ("update_json") is the same whole-value match. A trailing * is a value + // prefix. + if IsJSONValueParser(parser) { + if rc.text == "" { + return clause{}, false, nil + } + if rc.star { + return clause{kind: clausePrefix, terms: []string{rc.text}, weight: w}, true, nil + } + return clause{kind: clauseTerm, terms: []string{rc.text}, weight: w}, true, nil + } + + // A bare (non-quoted, non-prefix) CJK operand under ngram/json is an EXACT positional + // phrase (byte-offset slots), the same as NL and as classic fulltext — which matches + // the operand contiguously. (It previously expanded to an OR-group of its n-grams, + // which over-matched any doc sharing a single n-gram, e.g. `+中文學習` matched 中文學校. + // That was a workaround for the old token-index phrase matcher; byte positions make + // the phrase robust, so it is no longer needed.) + if parser != ParserGojieba && !rc.quoted && !rc.star && hasCJK(rc.text) { + slots, err := phraseSlots(rc.text, parser) + if err != nil { + return clause{}, false, err + } + if len(slots) == 0 { + return clause{}, false, nil + } + return clause{kind: clausePhrase, phrase: slots, weight: w}, true, nil + } + + tok, err := queryTokenizer(parser) + if err != nil { + return clause{}, false, err + } + terms, err := tokenizeToTerms([]byte(rc.text), tok) + if err != nil { + return clause{}, false, err + } + if len(terms) == 0 { + return clause{}, false, nil + } + switch { + case rc.quoted: + // Explicit "phrase": positional (byte-offset) slots — deduped/prefixed the same + // way NL is, so a CJK phrase matches by byte offset, not fragile token adjacency. + slots, serr := phraseSlots(rc.text, parser) + if serr != nil { + return clause{}, false, serr + } + if len(slots) == 0 { + return clause{}, false, nil + } + return clause{kind: clausePhrase, phrase: slots, weight: w}, true, nil + case rc.star: + return clause{kind: clausePrefix, terms: terms[:1], weight: w}, true, nil + case len(terms) == 1: + return clause{kind: clauseTerm, terms: terms, weight: w}, true, nil + default: + slots, serr := phraseSlots(rc.text, parser) + if serr != nil { + return clause{}, false, serr + } + if len(slots) == 0 { + return clause{}, false, nil + } + return clause{kind: clausePhrase, phrase: slots, weight: w}, true, nil + } +} diff --git a/pkg/fulltext2/reuse_pool_coverage_test.go b/pkg/fulltext2/reuse_pool_coverage_test.go new file mode 100644 index 0000000000000..ff8ae3658f902 --- /dev/null +++ b/pkg/fulltext2/reuse_pool_coverage_test.go @@ -0,0 +1,515 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/stretchr/testify/require" +) + +func TestReusableBasePoolBoundaryStates(t *testing.T) { + p := &immutableBasePool{entries: make(map[baseKey]*baseEntry)} + maxEntries, maxBytes, idleTTL := p.limits() + require.Equal(t, basePoolMaxEntries, maxEntries) + require.Equal(t, basePoolMaxBytes, maxBytes) + require.Equal(t, basePoolIdleTTL, idleTTL) + require.Zero(t, baseEntryBytes(baseKey{})) + require.Equal(t, int64(7), baseEntryBytes(baseKey{filesize: 7})) + + zeroLease := &segmentLease{} + zeroLease.release() + zeroLease.retire() + lease := newSegmentLease(NewSegment("lease", 0)) + view := lease.acquire(0) + require.NotNil(t, view) + view.Free() + lease.retire() + lease.release() + lease.retire() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + key := baseKey{index: "db.store", id: "cancelled", filesize: 1} + _, err := p.acquire(ctx, key, func() (*Segment, error) { + return nil, errors.New("cancelled context became the loader") + }, 0) + require.ErrorIs(t, err, context.Canceled) + + _, err = p.acquire(nil, key, func() (*Segment, error) { + return nil, nil + }, 0) + require.ErrorIs(t, err, errBaseLeaseGone) + require.Empty(t, p.entries) + + _, err = p.acquire(nil, key, func() (*Segment, error) { + panic("synthetic loader panic") + }, 0) + require.ErrorContains(t, err, "base load panic") + require.Empty(t, p.entries) + + p.entries[baseKey{index: "loading"}] = &baseEntry{loading: true} + p.entries[baseKey{index: "empty"}] = &baseEntry{} + p.evict(time.Now()) + loading := p.entries[baseKey{index: "loading"}] + require.NotNil(t, loading) + p.commit("loading", nil) + require.True(t, loading.retired) + loading.loading = false + p.clearAll() + require.Empty(t, p.entries) + + key = baseKey{index: "db.store", id: "kept", filesize: 1} + view, err = p.acquire(context.Background(), key, func() (*Segment, error) { + return NewSegment(key.id, 0), nil + }, 0) + require.NoError(t, err) + view.Free() + p.commit(key.index, map[baseKey]struct{}{key: {}}) + require.Contains(t, p.entries, key) + p.clearIndex(key.index) + require.Empty(t, p.entries) +} + +func TestLoadReasonRegistryBoundariesAndInvalidation(t *testing.T) { + restore := setLoadObserver(func(LoadEvent) {}) + defer restore() + + pendingLoadReasons.Lock() + old := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = old + pendingLoadReasons.Unlock() + }) + + require.Equal(t, "store", loadReasonKey("", "store")) + rememberLoadReason("", LoadMissCDCFlush) + rememberLoadReason("store", "") + reason, generation := peekLoadReason("store") + require.Empty(t, reason) + require.Zero(t, generation) + + pendingLoadReasons.Lock() + pendingLoadReasons.m["expired"] = pendingLoadReason{ + reason: LoadMissCDCFlush, + at: time.Now().Add(-loadReasonTTL), + } + pendingLoadReasons.Unlock() + rememberLoadReason("fresh", LoadMissMerge) + reason, generation = peekLoadReason("expired") + require.Empty(t, reason) + require.Zero(t, generation) + reason, generation = peekLoadReason("fresh") + require.Equal(t, LoadMissMerge, reason) + consumeLoadReason("fresh", generation) + + for i := 0; i < loadReasonSize+1; i++ { + rememberLoadReason(fmt.Sprintf("db.%d", i), LoadMissCDCFlush) + } + pendingLoadReasons.Lock() + require.LessOrEqual(t, len(pendingLoadReasons.m), loadReasonSize) + pendingLoadReasons.Unlock() + + cfg := TableConfig{DbName: "db", IndexTable: "store"} + invalidateLoadGeneration(cfg, LoadMissCDCFlush) + reason, generation = peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Equal(t, LoadMissCDCFlush, reason) + consumeLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable), generation) + invalidateLoadGeneration(cfg, LoadMissTTLExpired) + invalidateLoadGeneration(cfg, LoadMissGenerationChange) + invalidateLoadGeneration(cfg, LoadMissMerge) + invalidateLoadGeneration(cfg, LoadMissRebuild) + invalidateLoadGeneration(cfg, LoadMissReason("process_shutdown")) +} + +func TestInvalidateLoadGenerationPreservesReplacementReasonAndGeneration(t *testing.T) { + restore := setLoadObserver(func(LoadEvent) {}) + defer restore() + + pendingLoadReasons.Lock() + oldReasons := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + loadGenerations.Lock() + oldGenerations := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = oldReasons + pendingLoadReasons.Unlock() + loadGenerations.Lock() + loadGenerations.m = oldGenerations + loadGenerations.Unlock() + }) + + cfg := TableConfig{DbName: "db", IndexTable: "store"} + for generation, reason := range []LoadMissReason{LoadMissMerge, LoadMissRebuild} { + invalidateLoadGeneration(cfg, reason) + gotReason, gotGeneration := peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Equal(t, reason, gotReason) + require.Equal(t, uint64(generation+1), gotGeneration) + consumeLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable), gotGeneration) + } + NewFulltext2Search(cfg).OnCacheInvalidated("") + gotReason, gotGeneration := peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Empty(t, gotReason) + require.Zero(t, gotGeneration) +} + +func TestLoadReasonRegistryDoesNotConsumeNewerInvalidation(t *testing.T) { + restore := setLoadObserver(func(LoadEvent) {}) + defer restore() + + pendingLoadReasons.Lock() + old := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = old + pendingLoadReasons.Unlock() + }) + + key := loadReasonKey("db", "store") + rememberLoadReason(key, LoadMissCDCFlush) + firstReason, firstGeneration := peekLoadReason(key) + require.Equal(t, LoadMissCDCFlush, firstReason) + rememberLoadReason(key, LoadMissRebuild) + consumeLoadReason(key, firstGeneration) + secondReason, secondGeneration := peekLoadReason(key) + require.Equal(t, LoadMissRebuild, secondReason) + consumeLoadReason(key, secondGeneration) +} + +func TestLoadGenerationRegistryBoundsInactiveKeys(t *testing.T) { + restoreObserver := setLoadObserver(nil) + defer restoreObserver() + + loadGenerations.Lock() + previous := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + defer func() { + loadGenerations.Lock() + loadGenerations.m = previous + loadGenerations.Unlock() + }() + + active := beginLoadGeneration("db.active") + for i := 0; i < loadGenerationSize*2; i++ { + generation := beginLoadGeneration(fmt.Sprintf("db.%d", i)) + endLoadGeneration(generation) + } + require.True(t, loadGenerationCurrent(active)) + + loadGenerations.Lock() + require.LessOrEqual(t, len(loadGenerations.m), loadGenerationSize) + require.Equal(t, uint32(1), loadGenerations.m[active.index].active) + loadGenerations.Unlock() + + endLoadGeneration(active) + loadGenerations.Lock() + require.LessOrEqual(t, len(loadGenerations.m), loadGenerationSize) + loadGenerations.Unlock() +} + +func TestReusableLoadLifecycleHookRunsHousekeepingAndShutdown(t *testing.T) { + ensureReusableLoadLifecycle() + c := veccache.NewVectorIndexCache() + c.HouseKeeping() + c.Destroy() +} + +func TestReasonlessDropInvalidatesGenerationBeforeClearingReusablePools(t *testing.T) { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + + cfg := TableConfig{DbName: "db", IndexTable: "drop-race"} + index := loadReasonKey(cfg.DbName, cfg.IndexTable) + generation := beginLoadGeneration(index) + t.Cleanup(func() { endLoadGeneration(generation) }) + t.Cleanup(func() { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + }) + + oldBaseKey := baseKey{index: index, id: "old-base", checksum: "sum", filesize: 1} + oldBaseLease := newSegmentLease(NewSegment(oldBaseKey.id, 0)) + loadedBasePool.mu.Lock() + loadedBasePool.entries[oldBaseKey] = &baseEntry{ + lease: oldBaseLease, + published: true, + lastUsed: time.Now(), + } + loadedBasePool.totalBytes += baseEntryBytes(oldBaseKey) + loadedBasePool.mu.Unlock() + oldBaseLease.mu.Lock() + + oldTail := newTailState(1, 1, []*Segment{NewSegment("old-tail", 0)}, nil) + oldTailLease := oldTail.segments[0] + loadedTailPool.mu.Lock() + loadedTailPool.states[index] = oldTail + loadedTailPool.totalBytes += oldTail.bytes + loadedTailPool.mu.Unlock() + oldTailLease.mu.Lock() + + var releaseBase, releaseTail sync.Once + unlockBase := func() { releaseBase.Do(func() { oldBaseLease.mu.Unlock() }) } + unlockTail := func() { releaseTail.Do(func() { oldTailLease.mu.Unlock() }) } + t.Cleanup(unlockBase) + t.Cleanup(unlockTail) + + done := make(chan struct{}) + go func() { + NewFulltext2Search(cfg).OnCacheInvalidated("") + close(done) + }() + + require.Eventually(t, func() bool { + loadedBasePool.mu.Lock() + _, ok := loadedBasePool.entries[oldBaseKey] + loadedBasePool.mu.Unlock() + return !ok + }, time.Second, time.Millisecond) + + newBaseKey := baseKey{index: index, id: "new-base", checksum: "sum", filesize: 1} + baseView, err := loadedBasePool.acquireOwned(context.Background(), newBaseKey, func() (*Segment, error) { + return NewSegment(newBaseKey.id, 0), nil + }, 0, generation.owner) + require.NoError(t, err) + baseView.Free() + basePublished := loadedBasePool.commitOwnedIfCurrent( + index, + map[baseKey]struct{}{newBaseKey: {}}, + generation.owner, + func() bool { return loadGenerationCurrent(generation) }, + ) + if !basePublished { + loadedBasePool.rollback(generation.owner) + } + require.False(t, basePublished, "DROP must invalidate an active base load before pool clearing") + + unlockBase() + require.Eventually(t, func() bool { + loadedTailPool.mu.Lock() + _, ok := loadedTailPool.states[index] + loadedTailPool.mu.Unlock() + return !ok + }, time.Second, time.Millisecond) + + newTail := newTailState(1, 2, []*Segment{NewSegment("new-tail", 0)}, nil) + views, _, tailPublished := loadedTailPool.installAndAcquireIfCurrent( + index, + newTail, + func() bool { return loadGenerationCurrent(generation) }, + ) + if tailPublished { + freeSegs(views) + } else { + retireTailStates([]*tailState{newTail}) + } + require.False(t, tailPublished, "DROP must invalidate an active tail load before pool clearing") + + unlockTail() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("reasonless DROP did not finish") + } + + loadedBasePool.mu.Lock() + baseEntries := len(loadedBasePool.entries) + loadedBasePool.mu.Unlock() + loadedTailPool.mu.Lock() + tailStates := len(loadedTailPool.states) + loadedTailPool.mu.Unlock() + require.Zero(t, baseEntries) + require.Zero(t, tailStates) +} + +func TestLoadTailWithReuseReusesUnchangedState(t *testing.T) { + index := "db.store" + loadedTailPool.clear(index) + t.Cleanup(func() { loadedTailPool.clear(index) }) + state := newTailState(7, 11, []*Segment{NewSegment("tail", 11)}, nil) + views, _ := loadedTailPool.installAndAcquire(index, state) + freeSegs(views) + + segs, deletes, maxChunk, err := loadTailWithReuse(nil, TableConfig{DbName: "db", IndexTable: "store"}, 7, 11, nil) + require.NoError(t, err) + require.Len(t, segs, 1) + require.Nil(t, deletes) + require.Equal(t, int64(11), maxChunk) + freeSegs(segs) +} + +func TestTailPoolBoundaryAccountingAndReplacement(t *testing.T) { + require.Zero(t, estimateSegmentBytes(nil)) + seg := &Segment{ + pks: []any{int64(1)}, + pkOffsets: []int32{0}, + pkRaw: []byte("pk"), + docLen: []int32{2}, + includeTypes: []int32{int32(types.T_int64)}, + includeVals: [][]any{{int64(7)}}, + includeRaw: []byte("include"), + includeVarOffsets: []int32{0}, + terms: map[string]*termPostings{ + "term": {docIDs: []int64{1}, tfs: []uint8{1}, positions: [][]int32{{2, 3}}}, + }, + } + require.Positive(t, estimateSegmentBytes(seg)) + owned := *seg + owned.ownedBytes = 64 + require.Greater(t, estimateSegmentBytes(&owned), int64(64)) + require.Equal(t, int64(35), estimateDeleteKeyBytes("abc")) + require.Equal(t, int64(35), estimateDeleteKeyBytes([]byte("abc"))) + require.Equal(t, int64(32), estimateDeleteKeyBytes(7)) + require.Nil(t, cloneDeletes(nil)) + require.Equal(t, map[any]int64{"old": 2, "new": 1}, mergeTailDeletes( + map[any]int64{"old": 1}, map[any]int64{"old": 2, "new": 1})) + views, deletes := acquireTailViews(nil) + require.Nil(t, views) + require.Nil(t, deletes) + + p := &tailPool{states: make(map[string]*tailState)} + expected := newTailState(1, 10, []*Segment{NewSegment("old", 0)}, nil) + _, _, ok := p.acquireViews("missing", expected) + require.False(t, ok) + require.False(t, p.current("missing", expected)) + state := newTailState(1, 11, []*Segment{NewSegment("delta", 0)}, nil) + _, _, ok = p.installDeltaAndAcquire("missing", expected, state) + require.False(t, ok) + retireTailStates([]*tailState{state}) + + views, _ = p.installAndAcquire("db.store", expected) + for _, view := range views { + view.Free() + } + state = p.state("db.store") + require.Same(t, expected, state) + views, deletes, ok = p.acquireViews("db.store", expected) + require.True(t, ok) + require.Nil(t, deletes) + for _, view := range views { + view.Free() + } + require.True(t, p.current("db.store", expected)) + views, _ = p.installAndAcquire("db.store", expected) + for _, view := range views { + view.Free() + } + + replacement := newTailState(1, 12, []*Segment{NewSegment("replacement", 0)}, nil) + views, _ = p.installAndAcquire("db.store", replacement) + for _, view := range views { + view.Free() + } + p.clear("missing") + p.clear("db.store") + p.clearAll() +} + +func TestSearchLifecycleInvalidationAndPrefilterErrors(t *testing.T) { + s := NewFulltext2Search(TableConfig{DbName: "db", IndexTable: "store"}) + s.OnCacheInvalidated("") + s.OnCacheInvalidated(string(LoadMissCDCFlush)) + s.OnCacheInvalidated(string(LoadMissTTLExpired)) + s.OnCacheInvalidated(string(LoadMissGenerationChange)) + s.FinishLoadObservation() + s.SetLoadWaiters(3) + + proc := newSearchProc(t) + loaded := loadedSearch(t) + defer loaded.Destroy() + _, _, err := loaded.Search(proc, Fulltext2Query{ + Pattern: []byte("fox"), + IncludePredsJSON: []byte("{"), + }, vectorindex.RuntimeConfig{Limit: 10}) + require.Error(t, err) + keys, _, err := loaded.Search(proc, Fulltext2Query{ + Pattern: []byte("fox"), + Algo: BM25, + ScoreRange: &ScoreRange{HasMin: true, Min: -1}, + }, vectorindex.RuntimeConfig{Limit: 10}) + require.NoError(t, err) + require.NotEmpty(t, keys) +} + +func TestLoadTraceConcurrentPhaseUpdates(t *testing.T) { + events := make(chan LoadEvent, 1) + restore := setLoadObserver(func(event LoadEvent) { events <- event }) + defer restore() + + trace := newLoadTrace("db.store", LoadMissProcessStart) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + trace.addInternalSQL(time.Microsecond) + trace.addTempWrite(time.Microsecond) + trace.addMmap(time.Microsecond) + trace.addChecksum(time.Microsecond) + trace.addBaseBytes(1) + trace.addTailBytes(1) + }() + } + wg.Wait() + trace.finish(nil, false, 2) + + event := <-events + require.Equal(t, int64(4), event.BaseBytes) + require.Equal(t, int64(4), event.TailBytes) + require.Equal(t, int64(2), event.SingleflightWaiters) + require.NotZero(t, event.InternalSQLTimeMicros) + require.NotZero(t, event.TempFileWriteMicros) + require.NotZero(t, event.MmapMicros) + require.NotZero(t, event.ChecksumMicros) +} + +func TestStorageBudgetAfterAndEmptyFilesize(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + trace := &loadTrace{} + var seenSQL string + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + seenSQL = sql + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + }) + require.NoError(t, checkTailLoadBudgetAfter(sp, cfg, 7, trace)) + require.Contains(t, seenSQL, "chunk_id > 7") + + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{metaBatch(mp, "sum", 0, 1)}}, nil + }) + _, err := LoadFromStorage(sp, cfg, "empty") + require.ErrorContains(t, err, "empty filesize") +} diff --git a/pkg/fulltext2/search.go b/pkg/fulltext2/search.go new file mode 100644 index 0000000000000..228e877434bbd --- /dev/null +++ b/pkg/fulltext2/search.go @@ -0,0 +1,559 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math" + "math/bits" + "slices" + "sort" +) + +// ScoreAlgo selects the relevance formula (mirrors ft_relevancy_algorithm). Both +// use MatrixOne's squared-idf convention. +type ScoreAlgo int + +const ( + TfIdf ScoreAlgo = iota // w·tf·idf² + BM25 // w·idf²·bm25Factor(tf, docLen, avgDocLen) +) + +// fulltext2 has its OWN relevance session variable, distinct from classic +// fulltext's ft_relevancy_algorithm, and defaults to BM25 (the better ranked- +// retrieval default) rather than TF-IDF. Users opt into TF-IDF explicitly with +// SET ft2_relevancy_algorithm='TF-IDF'. +const ( + Fulltext2RelevancyAlgo = "ft2_relevancy_algorithm" + Fulltext2RelevancyAlgo_bm25 = "BM25" + Fulltext2RelevancyAlgo_tfidf = "TF-IDF" +) + +const ( + bm25K1 = 1.5 + bm25B = 0.75 +) + +// Result is one ranked hit: the source pk and its relevance score. Score is float32 +// (matches classic fulltext, whose score/score-map are float32 — float32 relevance is +// enough for BM25/TF-IDF); only the idf/avgDocLen intermediates are computed in float64 +// and narrowed, as classic does. +type Result struct { + Pk any + Score float32 + // Include carries the doc's INCLUDE column values (all of the index's INCLUDE columns, + // in index order; a nil element = SQL NULL) when the index has INCLUDE columns — so a + // COVERING query is served from the index without a base-table JOIN. nil otherwise. + Include []any +} + +// lookup resolves a term to its posting list on either a build-side segment (the +// terms map) or a loaded one (the FST ordinal). This lets the evaluator run +// against a freshly built segment or a deserialized one uniformly. +func (s *Segment) lookup(term string) (*termPostings, bool) { + if s.dict != nil { + return s.LookupLoaded(term) + } + p, ok := s.terms[term] + return p, ok +} + +// SearchPhrase runs an NL exact-phrase query: the terms (the tokenized query, in +// order) must occur as a CONTIGUOUS phrase in a document (MatrixOne NL semantics, +// §6). A single term degenerates to a plain term query. Returns up to k hits +// ordered by score desc (ties by ascending doc ord, for determinism). +// +// This is the correctness-first evaluator: it intersects the terms' posting +// lists and verifies positional adjacency exactly, without WAND block-max early +// termination (that optimization layers on later without changing results). +func (s *Segment) SearchPhrase(slots []phraseSlot, algo ScoreAlgo, k int) []Result { + if k <= 0 || s.N == 0 { + return nil + } + hits := s.matchPhrase(slots) + if len(hits) == 0 { + return nil + } + + // idf uses the PHRASE document frequency (number of docs the phrase occurs + // in), squared per MatrixOne. df <= N, so idf >= 0. + idf2 := idfSquared(s.N, len(hits)) + avgDocLen := s.avgDocLenOrMean() + + results := make([]Result, len(hits)) + for i, h := range hits { + results[i] = Result{ + Pk: s.pk(h.ord), + Score: s.scoreTerm(algo, float64(h.tf), idf2, h.ord, avgDocLen), + Include: s.decodeInclude(h.ord), + } + } + + // Order by score desc, ties by ascending ord (hits are already in ord order, + // so a stable sort on score desc keeps the ord tiebreak). + sort.SliceStable(results, func(a, b int) bool { return results[a].Score > results[b].Score }) + if len(results) > k { + results = results[:k] + } + return results +} + +// docTf is a document ord and the term/phrase occurrence count in it. +type docTf struct { + ord int64 + tf int +} + +// matchPhrase returns the documents that contain terms as a CONTIGUOUS phrase, +// each with the phrase occurrence count, in ascending doc ord. Returns nil if +// terms is empty or any term is absent from the segment. Shared by SearchPhrase +// and the boolean phrase clause. +// phraseSlot is one position of a positional phrase: a term (star=false) or a word* +// prefix (star=true) that must occur at byte offset `off` from the phrase's first slot. +// Byte offsets (not token indices) match classic fulltext's ngram phrase JOIN, so a +// redundant-ngram-removed CJK phrase (with gaps between kept trigrams) and a prefix tail +// both verify correctly against the doc's byte-position postings. +type phraseSlot struct { + term string + star bool + off int32 +} + +// matchPhrase verifies a positional phrase against the segment, returning per-doc phrase +// occurrence counts. The common shape — a multi-slot phrase of EXACT trigrams (every CJK +// query >= 3 chars decomposes to these) — runs the BLOCK-CURSOR conjunctive intersection +// (matchPhraseCursor): it advances a Block-Max cursor per slot in doc order, decoding one +// docID block + one position block at a time, so peak memory is O(nslots × BlockSize) +// regardless of corpus size. A single-slot phrase or one containing a star (prefix) slot +// — only 1-2 char CJK runs — takes the materialize fallback (which handles the presence- +// only prefix union). +func (s *Segment) matchPhrase(slots []phraseSlot) []docTf { + if len(slots) < 2 { + return s.matchPhraseFallback(slots) + } + for _, sl := range slots { + if sl.star { + return s.matchPhraseFallback(slots) + } + } + return s.matchPhraseCursor(slots) +} + +// matchPhraseFallback is the materialize-then-probe evaluator: it resolves each slot to +// doc-sorted postings, anchors on the rarest, and probes the others by binary search. It +// backs the single-slot and star-slot cases (a lone star slot is presence-only via a +// doc-ord bitset). No per-slot map, no re-sort (positions are stored ascending). +func (s *Segment) matchPhraseFallback(slots []phraseSlot) []docTf { + if len(slots) == 0 { + return nil + } + // Resolve each slot to its doc-sorted postings (docs[i] parallel to pos[i]). A lone + // star slot skips positions (presence-only); a resolved slot no doc satisfies ⇒ nil. + single := len(slots) == 1 + docsOf := make([][]int64, len(slots)) + posOf := make([][][]int32, len(slots)) + rare := 0 + for i, sl := range slots { + d, p, ok := s.slotPostings(sl, single) + if !ok { + return nil + } + docsOf[i], posOf[i] = d, p + if len(d) < len(docsOf[rare]) { + rare = i + } + } + + // Single slot: every doc containing it is a hit; tf = its occurrence count (1 for a + // presence-only star slot). + if single { + hits := make([]docTf, 0, len(docsOf[0])) + for di, ord := range docsOf[0] { + tf := 1 + if posOf[0] != nil { + tf = len(posOf[0][di]) + } + hits = append(hits, docTf{ord, tf}) + } + return hits + } + + // Multi-slot: iterate the rarest slot's docs; for each, look up this doc's positions in + // every slot (binary search), then anchor on the rare slot's positions and verify the + // others at B+off by binary search. Common slots are probed, never materialized whole. + var hits []docTf + posForDoc := make([][]int32, len(slots)) // scratch, reused across docs + for ri, ord := range docsOf[rare] { + present := true + for j := range slots { + if j == rare { + posForDoc[j] = posOf[rare][ri] + continue + } + k := sortedIndexInt64(docsOf[j], ord) + if k < 0 { + present = false + break + } + posForDoc[j] = posOf[j][k] + } + if !present { + continue + } + cnt := 0 + for _, pr := range posForDoc[rare] { + start := pr - slots[rare].off // candidate phrase-start byte position + matched := true + for j := range slots { + if j == rare { + continue + } + if !sortedContainsInt32(posForDoc[j], start+slots[j].off) { + matched = false + break + } + } + if matched { + cnt++ + } + } + if cnt > 0 { + hits = append(hits, docTf{ord, cnt}) + } + } + return hits +} + +// slotPostings resolves one phrase slot to doc-sorted postings (docs parallel to pos). +// An exact slot reuses the term's stored, already-ascending positions directly (no copy, +// no re-sort). A star slot expands its prefix: a LONE star slot (single-slot phrase) is +// presence-only — it unions just the doc IDs (pos==nil) so a hot 2-char prefix cannot +// materialize millions of positions; an embedded star slot (rare) merges positions. +func (s *Segment) slotPostings(sl phraseSlot, single bool) ([]int64, [][]int32, bool) { + if !sl.star { + pl, ok := s.lookup(sl.term) + if !ok { + return nil, nil, false + } + return pl.materializeDocIDs(), pl.materializePositions(), true + } + terms, err := s.prefixTerms(sl.term) + if err != nil || len(terms) == 0 { + return nil, nil, false + } + if single { + docs := s.unionDocIDs(terms) + if len(docs) == 0 { + return nil, nil, false + } + return docs, nil, true + } + return s.mergeStarPostings(terms) +} + +// unionDocIDs returns the ascending union of the doc IDs of the given terms — no +// positions (the presence-only path for a lone prefix slot). It unions into a doc-ord +// BITSET rather than a map: a hot 2-char prefix (e.g. 中文*) expands to many trigrams +// covering most of the corpus, and a growing map[int64]struct{} rehashed itself to death +// (the dominant cost once matchPhrase's own map was removed). A bitset is O(N/8) with no +// per-doc allocation or rehash, and the bit walk yields docs already ascending. +func (s *Segment) unionDocIDs(terms []string) []int64 { + if s.N == 0 { + return nil + } + bset := make([]uint64, (s.N+63)/64) + any := false + for _, t := range terms { + pl, ok := s.lookup(t) + if !ok { + continue + } + for _, ord := range pl.materializeDocIDs() { + bset[ord>>6] |= uint64(1) << (uint64(ord) & 63) + any = true + } + } + if !any { + return nil + } + docs := make([]int64, 0, 256) + for w, word := range bset { + base := int64(w) << 6 + for word != 0 { + docs = append(docs, base+int64(bits.TrailingZeros64(word))) + word &= word - 1 + } + } + return docs +} + +// mergeStarPostings merges the doc-sorted postings of a prefix expansion into one +// (docs, positions) with per-doc positions sorted (they come from different terms). This +// is the rare embedded-prefix path (a 1-2 char CJK run inside a longer phrase), so the +// map here is bounded and off the hot path. +func (s *Segment) mergeStarPostings(terms []string) ([]int64, [][]int32, bool) { + m := make(map[int64][]int32) + for _, t := range terms { + pl, ok := s.lookup(t) + if !ok { + continue + } + docs := pl.materializeDocIDs() + pos := pl.materializePositions() + for di, ord := range docs { + m[ord] = append(m[ord], pos[di]...) + } + } + if len(m) == 0 { + return nil, nil, false + } + docs := make([]int64, 0, len(m)) + for ord := range m { + docs = append(docs, ord) + } + slices.Sort(docs) + pos := make([][]int32, len(docs)) + for i, ord := range docs { + p := m[ord] + slices.Sort(p) + pos[i] = p + } + return docs, pos, true +} + +// sortedIndexInt64 returns the index of v in the ascending slice a, or -1 if absent. +func sortedIndexInt64(a []int64, v int64) int { + i := sort.Search(len(a), func(i int) bool { return a[i] >= v }) + if i < len(a) && a[i] == v { + return i + } + return -1 +} + +// phraseCursor walks one exact term's Block-Max postings in doc order for the phrase +// intersection, decoding ONE docID block + ONE position block at a time (never the whole +// list). It is the phrase analogue of wandIter, but also carries the block's positions. +type phraseCursor struct { + tp *termPostings + off int32 // this slot's byte offset within the phrase + idx int // current global posting index (0..df-1) + curDocBlk int // block currently decoded into bDocs (-1 = none) + curPosBlk int // block currently decoded into bPos (-1 = none) + blen int // valid entries in bDocs + bDocs []int64 // decoded docIDs of curDocBlk (cap BlockSize); backed by buf + bPos [][]int32 // decoded positions of curPosBlk (cap BlockSize); backed by buf + buf *phraseBuf // pooled backing for bDocs/bPos; returned by releasePhraseCursors +} + +func newPhraseCursor(tp *termPostings, off int32) *phraseCursor { + b := getPhraseBuf() + return &phraseCursor{ + tp: tp, off: off, curDocBlk: -1, curPosBlk: -1, + bDocs: b.docs, + bPos: b.pos, + buf: b, + } +} + +func (c *phraseCursor) atEnd() bool { return c.idx >= c.tp.df() } + +// ensureDocBlock loads only doc IDs. Positional payload stays compressed until all +// phrase cursors reach the same document. +func (c *phraseCursor) ensureDocBlock(b int) { + if b == c.curDocBlk { + return + } + c.blen = c.tp.fillBlockDocs(b, c.bDocs) + c.curDocBlk = b +} + +// ensurePositionBlock loads positions for block b on first use. A cursor can skip many +// doc blocks without ever calling this method. +func (c *phraseCursor) ensurePositionBlock(b int) { + if b == c.curPosBlk { + return + } + c.tp.fillBlockPositions(b, c.bPos) + c.curPosBlk = b +} + +// doc returns the current doc ord, or MaxInt64 when exhausted (sorts last). +func (c *phraseCursor) doc() int64 { + if c.atEnd() { + return math.MaxInt64 + } + c.ensureDocBlock(c.idx / BlockSize) + return c.bDocs[c.idx%BlockSize] +} + +// positions returns the current doc's ascending byte positions. +func (c *phraseCursor) positions() []int32 { + c.ensurePositionBlock(c.idx / BlockSize) + return c.bPos[c.idx%BlockSize] +} + +// skipTo advances (forward only) to the first doc >= target, using the resident +// blockLastDoc to pick the block so non-overlapping blocks are skipped without decoding. +func (c *phraseCursor) skipTo(target int64) { + bl := c.tp.blockLastDoc + b := c.idx / BlockSize + for b < len(bl) && bl[b] < target { + b++ + } + if b >= c.tp.nblk() { + c.idx = c.tp.df() // past the last posting → exhausted + return + } + c.ensureDocBlock(b) + lo, hi := 0, c.blen + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if c.bDocs[mid] < target { + lo = mid + 1 + } else { + hi = mid + } + } + c.idx = b*BlockSize + lo +} + +// matchPhraseCursor runs the block-cursor conjunctive phrase intersection over EXACT +// slots: it drives the rarest slot in doc order and, for each of its docs, skips the +// other cursors to that doc (block-skipping non-overlapping regions) and verifies byte- +// position adjacency. Peak memory is O(nslots × BlockSize) — one decoded block per +// cursor — independent of corpus size, vs the fallback's whole-posting-list materialize. +func (s *Segment) matchPhraseCursor(slots []phraseSlot) []docTf { + cursors := make([]*phraseCursor, len(slots)) + defer releasePhraseCursors(cursors) // recycle block buffers, incl. the early-return path + rare := 0 + for i, sl := range slots { + pl, ok := s.lookup(sl.term) + if !ok { + return nil // a slot no doc satisfies ⇒ the phrase can't occur + } + cursors[i] = newPhraseCursor(pl, sl.off) + if pl.df() < cursors[rare].tp.df() { + rare = i + } + } + rc := cursors[rare] + var hits []docTf + for !rc.atEnd() { + d := rc.doc() + present := true + for j, c := range cursors { + if j == rare { + continue + } + c.skipTo(d) + if c.doc() != d { + present = false + break + } + } + if present { + cnt := 0 + for _, pr := range rc.positions() { + start := pr - slots[rare].off // candidate phrase-start byte position + matched := true + for j := range slots { + if j == rare { + continue + } + if !sortedContainsInt32(cursors[j].positions(), start+slots[j].off) { + matched = false + break + } + } + if matched { + cnt++ + } + } + if cnt > 0 { + hits = append(hits, docTf{d, cnt}) + } + } + rc.idx++ + } + return hits +} + +// idfSquared is MatrixOne's squared inverse document frequency: (log10(N/df))². +// df is clamped to the range [1, n]. +// +// The upper clamp matters: df is summed over PHYSICAL postings (dead + live copies both +// counted, matching bm25's gdf), while n is the LIVE doc count, so heavy update churn on a +// small corpus can push df past n before a MERGE reclaims the dead copies. Without the clamp, +// df==n collapses idf to 0 and df>n squares a NEGATIVE log10(n/df) back into a POSITIVE impact +// — a non-monotonic idf that can rank a dirty base+tail differently from the identical corpus +// after compaction. Clamping keeps idf>=0 and monotonically non-increasing in df; a term that +// appears in (at least) every live doc correctly gets idf 0. n>=1 in every scoring path +// (globalN==0 / empty-segment queries short-circuit before this), so the clamp never divides +// by zero. +func idfSquared(n int64, df int) float64 { + if df < 1 { + df = 1 + } + if n >= 1 && int64(df) > n { + df = int(n) + } + idf := math.Log10(float64(n) / float64(df)) + return idf * idf +} + +// scoreTerm scores one (tf, idf², doc) contribution under the active algorithm. +func (s *Segment) scoreTerm(algo ScoreAlgo, tf, idf2 float64, ord int64, avgDocLen float64) float32 { + if algo == BM25 { + return float32(idf2 * bm25Factor(tf, s.docLen[ord], avgDocLen)) + } + return float32(tf * idf2) // TfIdf +} + +// avgDocLenOrMean returns the segment's AvgDocLen, falling back to the +// single-segment mean when it is unset (a lone segment queried directly). +func (s *Segment) avgDocLenOrMean() float64 { + if s.AvgDocLen != 0 { + return s.AvgDocLen + } + return meanDocLen(s.docLen) +} + +// sortedContainsInt32 reports whether v is in the ascending slice a. +func sortedContainsInt32(a []int32, v int32) bool { + i := sort.Search(len(a), func(i int) bool { return a[i] >= v }) + return i < len(a) && a[i] == v +} + +// bm25Factor is MatrixOne's BM25 tf component: tf·(k1+1)/(tf + k1·(1−b+b·dl/avgdl)). +func bm25Factor(tf float64, docLen int32, avgDocLen float64) float64 { + norm := 1.0 + if avgDocLen > 0 { + norm = 1.0 - bm25B + bm25B*float64(docLen)/avgDocLen + } + return tf * (bm25K1 + 1) / (tf + bm25K1*norm) +} + +// meanDocLen is the single-segment average document length (Σ docLen / N). For a +// multi-segment index AvgDocLen is set at load across all segments (§4); this is +// the fallback when a lone segment is queried directly. +func meanDocLen(docLen []int32) float64 { + if len(docLen) == 0 { + return 0 + } + var sum int64 + for _, d := range docLen { + sum += int64(d) + } + return float64(sum) / float64(len(docLen)) +} diff --git a/pkg/fulltext2/search_cache.go b/pkg/fulltext2/search_cache.go new file mode 100644 index 0000000000000..9c1a0bf79565c --- /dev/null +++ b/pkg/fulltext2/search_cache.go @@ -0,0 +1,517 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "errors" + "math" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// Fulltext2Query is the query payload passed through VectorIndexCache.Search to a +// Fulltext2Search: the raw MATCH pattern, whether it is boolean mode, and the +// relevance formula (BM25/TF-IDF, resolved per query from the ft2_relevancy_algorithm +// session var). The parser is a fixed property of the index (carried in cfg), so it +// is NOT part of the per-query payload — the same cached index serves every query. +type Fulltext2Query struct { + Pattern []byte + Boolean bool + // BagOfWords is IN BM25 MODE: ranked bag-of-words retrieval (each token an OR + // term, no positional phrase), so it works on a POSITION_FREE index. Takes + // precedence over Boolean. + BagOfWords bool + Algo ScoreAlgo + // FilterBytes is the optional serialized docfilter membership — the WHERE-clause + // prefilter pushed down as a runtime filter (built in C from the eligible pks), + // applied INSIDE the search so a pushed LIMIT bounds the filtered set. nil = none. + FilterBytes []byte + // IncludePredsJSON is the optional pushed-down predicate set on INCLUDE columns + // (IncludePredicateSpec JSON), evaluated in Go against the stored per-doc INCLUDE + // values INSIDE the search — so a WHERE on an INCLUDE column needs no second base scan. + // nil = none. ANDed with FilterBytes. + IncludePredsJSON []byte + // ScoreRange is an optional pushed-down relevance interval from an + // AND-reachable MATCH score predicate. Out-of-range scored rows are removed + // before they cross the TVF/join boundary; the plan keeps its own predicate + // for exact SQL semantics. + ScoreRange *ScoreRange +} + +// Fulltext2Search adapts a loaded fulltext2 Index to veccache.VectorIndexSearchIf so +// the positional index shares the VectorIndexCache (load-once, RW-shared, TTL +// eviction) with bm25 and the vector plugins, keyed by its storage table name. +// Before this, fulltext2_search reloaded the whole index (LoadAllBases + +// LoadTailSegments + NewIndex) on EVERY query — ~1s per query at 50K vs bm25's ~3ms. +// A loaded base segment's postings (docID/tf blocks + positions) are views into a +// shared read-only mmap, not the Go heap — reclaimable OS page cache, not GC-scanned; +// Destroy munmaps them (block-postings format, storage.go/segment.go). +type Fulltext2Search struct { + cfg TableConfig + idx *Index + loaded bool + + // Generation captured at Load (same txn snapshot as the loaded data) for the cache's + // cross-CN freshness check (IsStale, called off the housekeeping goroutine): loadedTs = + // MAX(metadata.timestamp) (REBUILD/MERGE), loadedTail = MAX(tag=1 CdcTail chunk_id) (CDC + // append). cnUUID/accountID are the durable handles to re-query in the background. + // genValid is false when capture failed / no resolver (unit tests) → IsStale is a no-op. + cnUUID string + accountID uint32 + loadedTs int64 + loadedTail int64 + genValid bool + loadWaiters atomic.Int64 + pendingTrace *loadTrace + pendingLoadErr error + pendingCancel bool +} + +var errLoadGenerationSuperseded = veccache.NewRetryableLoadError( + moerr.NewInvalidStateNoCtx("fulltext2 load superseded by a newer generation"), +) + +var _ veccache.VectorIndexSearchIf = (*Fulltext2Search)(nil) + +// NewFulltext2Search returns an unloaded search handle; the cache calls Load before +// the first Search. +func NewFulltext2Search(cfg TableConfig) *Fulltext2Search { + return &Fulltext2Search{cfg: cfg} +} + +// Load reads the index from the chunk store: the tag=0 base sub-indexes plus the +// tag=1 CdcTail delta frames (+ delete set), assembled into a queryable Index with +// global stats and per-pk liveness. An index created on an empty table has no tag=0 +// base, so segs may hold only tail segments (or be empty → a loaded, doc-less index). +func (s *Fulltext2Search) Load(sqlproc *sqlexec.SqlProcess) (err error) { + ensureReusableLoadLifecycle() + index := loadReasonKey(s.cfg.DbName, s.cfg.IndexTable) + generation := beginLoadGeneration(index) + defer endLoadGeneration(generation) + reason := LoadMissProcessStart + var reasonGeneration uint64 + if loadObservationEnabled() { + if observed, observedGeneration := peekLoadReason(index); observed != "" { + reason = observed + reasonGeneration = observedGeneration + } + } + trace := newLoadTrace(s.cfg.IndexTable, reason) + canceled := false + defer func() { + if err != nil { + canceled = isLoadCancellationError(err) + loadedBasePool.rollback(generation.owner) + } + // VectorIndexCache finishes the event after it samples the shared + // entry's waiter count, so waiters that arrived during this load are + // included in the diagnostic record. + s.pendingTrace = trace + s.pendingLoadErr = err + s.pendingCancel = canceled + }() + // Fail fast on the QUERY path if the bases' per-doc metadata won't fit the heap + // budget, rather than OOM-killing the CN (which takes down every query on the node). + // This guard is on Load, NOT LoadAllBases, so CompactSegments (MERGE) — the remedy — + // stays exempt and can still load the bases to reclaim dead docs. + var budgetStart time.Time + if trace != nil { + budgetStart = time.Now() + } + if err := checkBaseLoadBudget(sqlproc, s.cfg); err != nil { + if trace != nil { + trace.addInternalSQL(time.Since(budgetStart)) + } + return err + } + if trace != nil { + trace.addInternalSQL(time.Since(budgetStart)) + } + var baseGeneration, observedTail int64 + var generationReady bool + var generationStart time.Time + if trace != nil { + generationStart = time.Now() + } + baseGeneration, observedTail, err = LoadGeneration(sqlproc, s.cfg) + if trace != nil { + trace.addInternalSQL(time.Since(generationStart)) + } + if err == nil { + generationReady = true + } + bases, err := loadAllBases(sqlproc, s.cfg, trace, generation) + if err != nil { + return err + } + if !loadGenerationCurrent(generation) { + freeSegs(bases) + return errLoadGenerationSuperseded + } + var tails []*Segment + var deletes map[any]int64 + var appliedTail int64 + if generationReady { + tails, deletes, appliedTail, err = loadTailWithReuseForGeneration(sqlproc, s.cfg, baseGeneration, observedTail, trace, generation) + } else { + tails, deletes, appliedTail, err = loadTailSegmentsAfter(sqlproc, s.cfg, -1, trace) + } + if err != nil { + freeSegs(bases) // munmap the base segments on a tail-load error (don't leak the mappings) + return err + } + segs := append(bases, tails...) + s.idx = NewIndex(segs, deletes) + s.loaded = true + if loadGenerationCurrent(generation) { + consumeLoadReasonIfCurrent(index, reasonGeneration, generation) + } + trace.setGeneration(baseGeneration, appliedTail) + + // Capture the generation + durable handles for IsStale. Same txn as the load, so the + // captured generation matches the loaded snapshot exactly. On any capture failure genValid + // stays false and IsStale reports the entry as uncheckable-hence-stale (evict + reload to + // retry capture) rather than pinning it in cache forever — see IsStale. + s.cnUUID = sqlproc.GetService() + if acc, e := sqlproc.GetAccountID(); e == nil { + var genStart time.Time + if trace != nil { + genStart = time.Now() + } + ts, _, e2 := LoadGeneration(sqlproc, s.cfg) + if trace != nil { + trace.addInternalSQL(time.Since(genStart)) + } + if e2 == nil { + s.accountID = acc + // Keep the generation actually applied to the assembled Index. If a + // writer committed while loading, IsStale will force a later refresh + // instead of falsely declaring the older snapshot current. + s.loadedTs, s.loadedTail, s.genValid = baseGeneration, appliedTail, true + if !generationReady { + s.loadedTs, s.loadedTail = ts, appliedTail + } + trace.setGeneration(s.loadedTs, s.loadedTail) + } + } + return nil +} + +func isLoadCancellationError(err error) bool { + return errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) || + moerr.IsMoErrCode(err, moerr.ErrQueryInterrupted) +} + +// OnCacheInvalidated is called by the generic cache only on invalidation paths; +// it does not run on a warm query. The bounded registry lets the next load +// classify its miss without changing VectorIndexSearchIf. +func (s *Fulltext2Search) OnCacheInvalidated(reason string) { + if reason == "" { + // A reason-less Remove is used by DROP/restore and by generic cache + // callers. No next-load diagnostic event is needed, but reusable state + // for this index must not survive the cache ownership boundary. + clearReusableLoadGeneration(s.cfg) + return + } + invalidateLoadGeneration(s.cfg, LoadMissReason(reason)) +} + +// SetLoadWaiters is an internal cache hook used only to complete load +// observability. It is not part of the public VectorIndexSearchIf contract. +func (s *Fulltext2Search) SetLoadWaiters(n int64) { + s.loadWaiters.Store(n) +} + +// FinishLoadObservation is called by VectorIndexCache after Load returns and +// SetLoadWaiters has sampled the shared entry. It is intentionally outside +// VectorIndexSearchIf so other algorithms keep their existing contract. +func (s *Fulltext2Search) FinishLoadObservation() { + trace := s.pendingTrace + if trace == nil { + return + } + s.pendingTrace = nil + err := s.pendingLoadErr + canceled := s.pendingCancel + s.pendingLoadErr = nil + s.pendingCancel = false + trace.finish(err, canceled, s.loadWaiters.Load()) +} + +// IsStale reports whether the underlying index has changed since this entry was loaded, by +// comparing the load-time generation to the current one (a REBUILD/MERGE bumps the metadata +// timestamp; a CDC append bumps the tag=1 tail chunk_id). Run on the cache housekeeping +// goroutine (NOT the search path), so it opens its own short auto-commit txn via the durable +// cnUUID/accountID captured at load. +// +// On a query ERROR it returns (true, err): the most likely cause is the index tables were +// dropped/rebuilt out from under us (DROP INDEX, restore), so the dead entry must be +// reclaimed — the next search reloads, or fails cleanly if the index is truly gone. The err +// is surfaced only for logging. (A no-op reload for a genuinely-transient blip is cheaper +// than pinning a possibly-dead entry until TTL.) No captured generation (capture failed at load, +// or no service to re-query) ⇒ (true, nil): the entry cannot self-check freshness, so evict it to +// force a reload that retries capture. Returning (false, nil) would pin a hot entry — whose TTL +// keeps sliding on every search — in cache indefinitely, serving pre-CDC/rebuild data forever; +// the bounded reload (one per freshness sweep) self-heals the moment capture succeeds. +func (s *Fulltext2Search) IsStale() (bool, error) { + if !s.genValid || s.cnUUID == "" { + return true, nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + ts, tail, err := QueryGeneration(ctx, s.cnUUID, s.accountID, s.cfg) + if err != nil { + return true, err + } + stale := ts != s.loadedTs || tail != s.loadedTail + logutil.Debugf("[ft2-isstale] index=%s loadedGen=(ts=%d,tail=%d) curGen=(ts=%d,tail=%d) stale=%v", + s.cfg.IndexTable, s.loadedTs, s.loadedTail, ts, tail, stale) + return stale, nil +} + +// prepare validates the query, resolves the top-k bound, and builds the WHERE prefilter, +// shared by Search and SearchInto. free is ALWAYS non-nil and must be deferred by the caller +// even on error (it frees the C-backed docfilter created here, so it never outlives the +// query — the cached Index is shared, so the filter must be per-query local). empty=true means +// a loaded but doc-less index (matches nothing) — the caller returns an empty result. +func (s *Fulltext2Search) prepare(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig) (q Fulltext2Query, k int, pf *prefilter, free func(), empty bool, err error) { + free = func() {} + if !s.loaded || s.idx == nil { + err = moerr.NewInternalError(proc.GetContext(), "fulltext2 index not loaded") + return + } + if s.idx.NumDocs() == 0 { + empty = true + return + } + var ok bool + if q, ok = query.(Fulltext2Query); !ok { + err = moerr.NewInternalError(proc.GetContext(), "fulltext2 search: invalid query payload") + return + } + // rt.Limit is uint; a value past MaxInt32 (an absurd pushed LIMIT) would wrap negative + // in int(...), and 0 means "no pushed LIMIT" — return the whole result set (the TVF + // paginates). Mirror bm25's clamp. + k = int(rt.Limit) + if rt.Limit > uint(math.MaxInt32) { + k = math.MaxInt32 + } else if k <= 0 { + k = int(s.idx.NumDocs()) + } + if len(q.FilterBytes) > 0 || len(q.IncludePredsJSON) > 0 || q.ScoreRange != nil { + pf = &prefilter{scoreRange: q.ScoreRange} + if len(q.FilterBytes) > 0 { + var filter docfilter.MembershipFilter + if filter, err = docfilter.New(q.FilterBytes); err != nil { + return + } + free = filter.Free // freed via the caller's defer, incl. the error paths below + pf.docFilter = filter + } + if len(q.IncludePredsJSON) > 0 { + var preds []compiledIncludePred + if preds, err = compileIncludePredicates(q.IncludePredsJSON, s.idx.includeTypes(), s.idx.pkType()); err != nil { + return + } + pf.include = preds + } + } + return +} + +// runTopK dispatches the bounded top-k search (ranked bag-of-words vs positional query). +func (s *Fulltext2Search) runTopK(q Fulltext2Query, k int, pf *prefilter) ([]Result, error) { + var ( + results []Result + err error + ) + if q.BagOfWords { + results, err = s.idx.SearchBagOfWords(q.Pattern, s.cfg.Parser, q.Algo, k, pf) + } else { + results, err = s.idx.SearchQuery(q.Pattern, q.Boolean, s.cfg.Parser, q.Algo, k, pf) + } + if err != nil || pf == nil || pf.scoreRange == nil { + return results, err + } + kept := results[:0] + for _, result := range results { + if pf.scoreRange.contains(result.Score) { + kept = append(kept, result) + } + } + return kept, nil +} + +// Search runs the WAND positional query (NL exact-phrase or boolean) and returns +// ([]any pks of the source type, []float64 scores). The covered LIMIT path is served by the +// box-free SearchInto instead; this []any form is the streaming (Emit) path and the legacy +// non-covered return. +func (s *Fulltext2Search) Search(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { + q, k, pf, free, empty, err := s.prepare(proc, query, rt) + defer free() + if err != nil { + return nil, nil, err + } + if empty { + return []any{}, []float64{}, nil + } + + // No-LIMIT streaming path: when the caller passes an Emit callback (the TVF does this + // only for a query with no pushed LIMIT), yield every matching doc in bounded batches — + // no top-K heap, no materialization. Results are handed off through Emit, so return empty + // keys/distances (mirrors bm25). wantInclude tells the stream to carry each doc's INCLUDE + // values (segment order) through Emit's 3rd arg. + if rt.Emit != nil { + wantInclude := len(rt.RequestedIncludeColumns) > 0 + var serr error + if q.BagOfWords { + serr = s.idx.StreamBagOfWords(q.Pattern, s.cfg.Parser, q.Algo, pf, wantInclude, rt.Emit) + } else { + serr = s.idx.StreamQuery(q.Pattern, q.Boolean, s.cfg.Parser, q.Algo, pf, wantInclude, rt.Emit) + } + if serr != nil { + return nil, nil, serr + } + return []any{}, []float64{}, nil + } + + results, err := s.runTopK(q, k, pf) + if err != nil { + return nil, nil, err + } + keysOut := make([]any, len(results)) + dist := make([]float64, len(results)) + for i, r := range results { + keysOut[i] = r.Pk + dist[i] = float64(r.Score) + } + return keysOut, dist, nil +} + +// SearchInto is the box-free LIMIT-path twin of Search: it fills the caller-owned out +// SearchOutput (pk column, scores, covered INCLUDE columns) instead of returning boxed []any +// keys — so a warm query allocates nothing for its results (the TVF pools out and Resets it +// per query). out.Keys is the pk in ColumnBuffer form (box-free for every pk type), out.Dists +// the float32 scores, out.Include one nullable buffer per FULL index INCLUDE column (populated +// only when rt.RequestedIncludeColumns is set; the TVF projects what it needs by segPos). +func (s *Fulltext2Search) SearchInto(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, out *vectorindex.SearchOutput) error { + if out == nil { + return moerr.NewInternalError(proc.GetContext(), "fulltext2 SearchInto: nil out") + } + q, k, pf, free, empty, err := s.prepare(proc, query, rt) + defer free() + if err != nil { + return err + } + // Reset the caller-owned buffers (reused across queries → alloc-free after warmup). + pkType := s.idx.pkType() + if out.Keys == nil { + out.Keys = &vectorindex.ColumnBuffer{} + } else { + out.Keys.Reset() + } + out.Keys.Type = types.T(pkType) + out.Dists = out.Dists[:0] + if empty { + out.Include = out.Include[:0] + return nil + } + + results, err := s.runTopK(q, k, pf) + if err != nil { + return err + } + // pk column (box-free re-encode of each already-materialized boxed pk) + scores. + w, fixed := fixedPkByteWidth(pkType) + if !fixed { + w = 0 + } + for i := range results { + appendAnyPk(out.Keys, w, results[i].Pk) + out.Dists = append(out.Dists, results[i].Score) + } + return s.fillInclude(rt, results, out) +} + +// fillInclude populates out.Include with one nullable ColumnBuffer per FULL index INCLUDE +// column (segment order), covering the whole result set — so a covered LIMIT query reads its +// projected INCLUDE columns straight from the index (no base-table JOIN, no map[string][]any, +// no per-row reflection append). The TVF maps its requested output columns to segment +// positions (by includeOut.segPos), exactly as the streaming consumer does. No-op (empties +// out.Include) when no include columns were requested. The buffers are REUSED across queries +// (Reset in place) so includes are alloc-free after warmup, like out.Keys/out.Dists. +// +// Source note: this re-encodes each result's already-materialized boxed Include value — the +// top-k's per-winner decodeInclude boxing (O(limit), bounded) is retained; what this removes +// is the transport map + the reflection AppendAny. +func (s *Fulltext2Search) fillInclude(rt vectorindex.RuntimeConfig, results []Result, out *vectorindex.SearchOutput) error { + if len(rt.RequestedIncludeColumns) == 0 { + out.Include = out.Include[:0] + return nil + } + it := s.idx.includeTypes() + if cap(out.Include) < len(it) { + out.Include = make([]*vectorindex.ColumnBuffer, len(it)) + } else { + out.Include = out.Include[:len(it)] + } + for c := range it { + if out.Include[c] == nil { + out.Include[c] = &vectorindex.ColumnBuffer{} + } else { + out.Include[c].Reset() + } + out.Include[c].Type = types.T(it[c]) + } + for r := range results { + inc := results[r].Include + for c := range out.Include { + var v any + if c < len(inc) { + v = inc[c] + } + if err := appendAnyInclude(out.Include[c], v); err != nil { + return err + } + } + } + return nil +} + +// SearchFloat32 is unsupported (fulltext scores are float64; the vector float32 +// fast-path does not apply). +func (s *Fulltext2Search) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { + return moerr.NewInternalError(proc.GetContext(), "fulltext2 search: SearchFloat32 not supported") +} + +// Destroy releases this generation's base/tail views. Immutable base mappings +// remain alive in the bounded pool while another generation can reuse them; +// the cache holds the write lock, so no search is in flight on this generation. +func (s *Fulltext2Search) Destroy() { + s.FinishLoadObservation() + s.idx.Free() + s.idx = nil + s.loaded = false +} diff --git a/pkg/fulltext2/search_cache_test.go b/pkg/fulltext2/search_cache_test.go new file mode 100644 index 0000000000000..049ae32b50c53 --- /dev/null +++ b/pkg/fulltext2/search_cache_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "sync" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestConcurrentSearchQuery is the concurrency contract behind Fulltext2Search / +// VectorIndexCache: a single loaded Index is shared (RW-locked) across concurrent +// queries, so SearchQuery must be pure-read — no mutation of segments, the FST, the +// loaded posting slices, or the liveness maps. It builds a loaded (serialized -> +// deserialized, i.e. FST + materialized `loaded` path) Index and hammers it from +// many goroutines with NL-phrase, boolean, and prefix queries at once, asserting +// every goroutine gets the SAME result as a single-threaded baseline. Run under +// -race, a shared write on the query path trips the detector. +func TestConcurrentSearchQuery(t *testing.T) { + // A loaded multi-segment Index (base + a tail delta) so liveness/global-stats and + // both the FST get() and the prefix Iterator() paths are exercised concurrently. + bb := NewBuilder("base", int32(types.T_int64)) + feed(t, bb, int64(0), "quick", "brown", "fox") + feed(t, bb, int64(1), "quick", "brown", "dog") + feed(t, bb, int64(2), "lazy", "fox", "sleeps") + base := loadedSeg(t, bb) + base.Recency = 0 + + tb := NewBuilder("tail", int32(types.T_int64)) + feed(t, tb, int64(3), "brown", "bear", "lazy") + feed(t, tb, int64(4), "quick", "quick", "fox") + tail := loadedSeg(t, tb) + tail.Recency = 1 + + idx := NewIndex([]*Segment{base, tail}, nil) + + type query struct { + pattern string + boolean bool + } + queries := []query{ + {"quick brown fox", false}, // NL exact phrase (FST get) + {"fox", false}, // single term + {"+quick +fox", true}, // boolean AND + {"+quick -fox", true}, // boolean AND-NOT + {"quic*", true}, // boolean prefix (FST Iterator) + {"lazy", false}, // term hitting both segments + } + + // Single-threaded baseline: the canonical (pk-set) answer for each query. + baseline := make([]map[any]struct{}, len(queries)) + for i, q := range queries { + res, err := idx.SearchQuery([]byte(q.pattern), q.boolean, ParserDefault, BM25, 100, nil) + require.NoError(t, err, "baseline %q", q.pattern) + m := make(map[any]struct{}, len(res)) + for _, r := range res { + m[r.Pk] = struct{}{} + } + baseline[i] = m + } + + const goroutines, iters = 24, 200 + var wg sync.WaitGroup + errCh := make(chan error, goroutines) + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for it := 0; it < iters; it++ { + q := queries[it%len(queries)] + res, err := idx.SearchQuery([]byte(q.pattern), q.boolean, ParserDefault, BM25, 100, nil) + if err != nil { + errCh <- fmt.Errorf("query %q: %w", q.pattern, err) + return + } + want := baseline[it%len(queries)] + if len(res) != len(want) { + errCh <- fmt.Errorf("query %q: got %d hits, want %d", q.pattern, len(res), len(want)) + return + } + for _, r := range res { + if _, ok := want[r.Pk]; !ok { + errCh <- fmt.Errorf("query %q: unexpected pk %v", q.pattern, r.Pk) + return + } + } + } + }() + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Fatal(err) + } +} diff --git a/pkg/fulltext2/search_cache_unit_test.go b/pkg/fulltext2/search_cache_unit_test.go new file mode 100644 index 0000000000000..01b2b1d68cbcd --- /dev/null +++ b/pkg/fulltext2/search_cache_unit_test.go @@ -0,0 +1,679 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "math" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/stretchr/testify/require" +) + +func newSearchProc(t *testing.T) *sqlexec.SqlProcess { + return sqlexec.NewSqlProcess(testutil.NewProc(t)) +} + +// loadedSearch builds a Fulltext2Search whose Index is assembled in memory (via the +// serialize→deserialize loadedSeg path), bypassing Load/DB. +func loadedSearch(t *testing.T) *Fulltext2Search { + bb := NewBuilder("base", int32(types.T_int64)) + feed(t, bb, int64(0), "quick", "brown", "fox") + feed(t, bb, int64(1), "quick", "brown", "dog") + feed(t, bb, int64(2), "lazy", "fox", "sleeps") + seg := loadedSeg(t, bb) + s := NewFulltext2Search(TableConfig{IndexTable: "__store", Parser: ParserDefault}) + s.idx = NewIndex([]*Segment{seg}, nil) + s.loaded = true + return s +} + +func TestFulltext2SearchNewAndUnloaded(t *testing.T) { + proc := newSearchProc(t) + s := NewFulltext2Search(TableConfig{IndexTable: "__store"}) + require.Equal(t, "__store", s.cfg.IndexTable) + require.False(t, s.loaded) + + // Search before Load → "not loaded". + _, _, err := s.Search(proc, Fulltext2Query{Pattern: []byte("fox")}, vectorindex.RuntimeConfig{}) + require.ErrorContains(t, err, "not loaded") + + // SearchFloat32 is unsupported. + require.ErrorContains(t, s.SearchFloat32(proc, nil, vectorindex.RuntimeConfig{}, nil, nil), "not supported") +} + +func TestFulltext2SearchLoadFailurePreservesOtherGeneration(t *testing.T) { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + t.Cleanup(func() { + loadedBasePool.clearAll() + loadedTailPool.clearAll() + }) + + cfg := testStorageCfg() + baseKey := baseKey{index: "db.__store", id: "base-0", checksum: "sum", filesize: 1} + view, err := loadedBasePool.acquire(context.Background(), baseKey, func() (*Segment, error) { + return NewSegment("base-0", 0), nil + }, 0) + require.NoError(t, err) + view.Free() + tailViews, _ := loadedTailPool.installAndAcquire("db.__store", newTailState(1, 1, []*Segment{NewSegment("tail-1", 0)}, nil)) + freeSegs(tailViews) + + sp, mp := mockSqlProc(t) + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, int64(1)<<50)}}, nil + }) + s := NewFulltext2Search(cfg) + require.Error(t, s.Load(sp)) + + loadedBasePool.mu.Lock() + baseEntries := len(loadedBasePool.entries) + loadedBasePool.mu.Unlock() + loadedTailPool.mu.Lock() + tailStates := len(loadedTailPool.states) + loadedTailPool.mu.Unlock() + require.Equal(t, 1, baseEntries) + require.Equal(t, 1, tailStates) +} + +func TestFulltext2SearchLoadConsumesInvalidationReasonAfterSuccess(t *testing.T) { + for _, reason := range []LoadMissReason{LoadMissCDCFlush, LoadMissMerge, LoadMissRebuild} { + t.Run(string(reason), func(t *testing.T) { + sp := &sqlexec.SqlProcess{SqlCtx: sqlexec.NewSqlContext(context.Background(), "cn-1", nil, 7, nil)} + mp := mpool.MustNewZero() + cfg := testStorageCfg() + var events []LoadEvent + restore := setLoadObserver(func(event LoadEvent) { events = append(events, event) }) + defer restore() + + pendingLoadReasons.Lock() + oldReasons := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + loadGenerations.Lock() + oldGenerations := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = oldReasons + pendingLoadReasons.Unlock() + loadGenerations.Lock() + loadGenerations.m = oldGenerations + loadGenerations.Unlock() + }) + + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + switch { + case strings.Contains(sql, "CAST(COALESCE"): + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + case strings.Contains(sql, "MAX(timestamp)"): + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 11)}}, nil + case strings.Contains(sql, "MAX(chunk_id)"): + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 22)}}, nil + case strings.Contains(sql, "LENGTH("): + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + default: + return executor.Result{Mp: mp}, nil + } + }) + invalidateLoadGeneration(cfg, reason) + + s := NewFulltext2Search(cfg) + require.NoError(t, s.Load(sp)) + s.FinishLoadObservation() + require.Len(t, events, 1) + require.Equal(t, reason, events[0].MissReason) + require.Equal(t, int64(11), events[0].BaseGeneration) + require.Equal(t, int64(22), events[0].TailGeneration) + require.True(t, events[0].LoadSuccess) + gotReason, generation := peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Empty(t, gotReason) + require.Zero(t, generation) + s.Destroy() + }) + } +} + +func TestFulltext2SearchLoadClassifiesQueryInterruptionAsCancel(t *testing.T) { + sp := &sqlexec.SqlProcess{SqlCtx: sqlexec.NewSqlContext(context.Background(), "cn-1", nil, 7, nil)} + var event LoadEvent + restore := setLoadObserver(func(got LoadEvent) { event = got }) + defer restore() + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{}, moerr.NewQueryInterrupted(context.Background()) + }) + + s := NewFulltext2Search(testStorageCfg()) + require.Error(t, s.Load(sp)) + s.FinishLoadObservation() + require.True(t, event.LoadCancel) + require.False(t, event.LoadError) + require.False(t, event.LoadSuccess) +} + +func TestFulltext2SearchLoadRetainsInvalidationReasonAfterFailure(t *testing.T) { + for _, reason := range []LoadMissReason{LoadMissCDCFlush, LoadMissMerge, LoadMissRebuild} { + t.Run(string(reason), func(t *testing.T) { + sp := &sqlexec.SqlProcess{SqlCtx: sqlexec.NewSqlContext(context.Background(), "cn-1", nil, 7, nil)} + cfg := testStorageCfg() + var events []LoadEvent + restore := setLoadObserver(func(got LoadEvent) { events = append(events, got) }) + defer restore() + + pendingLoadReasons.Lock() + oldReasons := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + loadGenerations.Lock() + oldGenerations := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = oldReasons + pendingLoadReasons.Unlock() + loadGenerations.Lock() + loadGenerations.m = oldGenerations + loadGenerations.Unlock() + }) + + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{}, moerr.NewQueryInterrupted(context.Background()) + }) + invalidateLoadGeneration(cfg, reason) + + s := NewFulltext2Search(cfg) + require.Error(t, s.Load(sp)) + s.FinishLoadObservation() + require.Error(t, s.Load(sp)) + s.FinishLoadObservation() + + require.Len(t, events, 2) + require.Equal(t, reason, events[0].MissReason) + require.Equal(t, reason, events[1].MissReason) + require.True(t, events[0].LoadCancel) + require.True(t, events[1].LoadCancel) + s.Destroy() + }) + } +} + +func TestFulltext2SearchInvalidationEvictionRecordsOneReason(t *testing.T) { + cleanupObserver := setLoadObserver(func(LoadEvent) {}) + defer cleanupObserver() + + pendingLoadReasons.Lock() + previousReasons := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = previousReasons + pendingLoadReasons.Unlock() + }) + + previousCache := veccache.Cache + veccache.Cache = veccache.NewVectorIndexCache() + t.Cleanup(func() { veccache.Cache = previousCache }) + + cfg := testStorageCfg() + old := NewFulltext2Search(cfg) + old.idx = NewIndex(nil, nil) + old.loaded = true + oldEntry := &veccache.VectorIndexSearch{Algo: old} + oldEntry.Cond = sync.NewCond(oldEntry.Mutex.RLocker()) + veccache.Cache.IndexMap.Store(cfg.IndexTable, oldEntry) + + // Production mutation order: record the reason before evicting the local + // entry. Hold the old entry lock so Remove has deleted the old map entry but + // is blocked before destruction; a replacement can therefore start while + // the old eviction is still in flight. + oldEntry.Mutex.Lock() + old.OnCacheInvalidated(string(LoadMissCDCFlush)) + removed := make(chan struct{}) + go func() { + veccache.Cache.Remove(cfg.IndexTable) + close(removed) + }() + require.Eventually(t, func() bool { + _, loaded := veccache.Cache.IndexMap.Load(cfg.IndexTable) + return !loaded + }, time.Second, time.Millisecond) + replacement := NewFulltext2Search(cfg) + replacement.idx = NewIndex(nil, nil) + replacement.loaded = true + replacementEntry := &veccache.VectorIndexSearch{Algo: replacement} + replacementEntry.Cond = sync.NewCond(replacementEntry.Mutex.RLocker()) + veccache.Cache.IndexMap.Store(cfg.IndexTable, replacementEntry) + + reason, generation := peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Equal(t, LoadMissCDCFlush, reason) + require.NotZero(t, generation) + consumeLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable), generation) + oldEntry.Mutex.Unlock() + select { + case <-removed: + case <-time.After(time.Second): + t.Fatal("old cache eviction did not finish") + } + reason, generation = peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Empty(t, reason) + require.Zero(t, generation) + veccache.Cache.Remove(cfg.IndexTable) +} + +func TestHouseKeepingPublishesGenerationBeforeReplacementLoad(t *testing.T) { + cleanupObserver := setLoadObserver(func(LoadEvent) {}) + defer cleanupObserver() + + previousCache := veccache.Cache + veccache.Cache = veccache.NewVectorIndexCache() + defer func() { veccache.Cache = previousCache }() + + loadGenerations.Lock() + previousGenerations := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + defer func() { + loadGenerations.Lock() + loadGenerations.m = previousGenerations + loadGenerations.Unlock() + }() + + cfg := testStorageCfg() + old := NewFulltext2Search(cfg) + old.idx = NewIndex(nil, nil) + old.loaded = true + oldEntry := &veccache.VectorIndexSearch{Algo: old} + oldEntry.Cond = sync.NewCond(oldEntry.Mutex.RLocker()) + oldEntry.ExpireAt.Store(time.Now().Add(-time.Second).UnixMicro()) + veccache.Cache.IndexMap.Store(cfg.IndexTable, oldEntry) + + oldEntry.Mutex.Lock() + done := make(chan struct{}) + go func() { + veccache.Cache.HouseKeeping() + close(done) + }() + require.Eventually(t, func() bool { + _, loaded := veccache.Cache.IndexMap.Load(cfg.IndexTable) + return !loaded + }, time.Second, time.Millisecond) + reason, observedGeneration := peekLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.Equal(t, LoadMissTTLExpired, reason) + require.NotZero(t, observedGeneration) + consumeLoadReason(loadReasonKey(cfg.DbName, cfg.IndexTable), observedGeneration) + + replacement := NewFulltext2Search(cfg) + replacement.idx = NewIndex(nil, nil) + replacement.loaded = true + replacementEntry := &veccache.VectorIndexSearch{Algo: replacement} + replacementEntry.Cond = sync.NewCond(replacementEntry.Mutex.RLocker()) + veccache.Cache.IndexMap.Store(cfg.IndexTable, replacementEntry) + + loadGen := beginLoadGeneration(loadReasonKey(cfg.DbName, cfg.IndexTable)) + require.True(t, loadGenerationCurrent(loadGen)) + endLoadGeneration(loadGen) + + oldEntry.Mutex.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("housekeeping eviction did not finish") + } + require.True(t, loadGenerationCurrent(loadGen)) + veccache.Cache.Remove(cfg.IndexTable) +} + +func TestFulltext2SupersededLoadIsRetryable(t *testing.T) { + require.True(t, veccache.IsRetryableLoadError(errLoadGenerationSuperseded)) + require.Contains(t, errLoadGenerationSuperseded.Error(), "fulltext2 load superseded by a newer generation") +} + +func TestVectorIndexCacheRetriesSupersededFulltext2Load(t *testing.T) { + for _, tc := range []struct { + name string + call func(*sqlexec.SqlProcess, *Fulltext2Search, TableConfig) error + }{ + { + name: "Search", + call: func(proc *sqlexec.SqlProcess, loader *Fulltext2Search, cfg TableConfig) error { + _, _, err := veccache.Cache.Search(proc, cfg.IndexTable, loader, + Fulltext2Query{Pattern: []byte("x")}, vectorindex.RuntimeConfig{Limit: 1}) + return err + }, + }, + { + name: "SearchInto", + call: func(proc *sqlexec.SqlProcess, loader *Fulltext2Search, cfg TableConfig) error { + return veccache.Cache.SearchInto(proc, cfg.IndexTable, loader, + Fulltext2Query{Pattern: []byte("x")}, vectorindex.RuntimeConfig{Limit: 1}, &vectorindex.SearchOutput{}) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + previousCache := veccache.Cache + veccache.Cache = veccache.NewVectorIndexCache() + t.Cleanup(func() { veccache.Cache = previousCache }) + + pendingLoadReasons.Lock() + previousReasons := pendingLoadReasons.m + pendingLoadReasons.m = make(map[string]pendingLoadReason) + pendingLoadReasons.Unlock() + t.Cleanup(func() { + pendingLoadReasons.Lock() + pendingLoadReasons.m = previousReasons + pendingLoadReasons.Unlock() + }) + loadGenerations.Lock() + previousGenerations := loadGenerations.m + loadGenerations.m = make(map[string]loadGenerationState) + loadGenerations.Unlock() + t.Cleanup(func() { + loadGenerations.Lock() + loadGenerations.m = previousGenerations + loadGenerations.Unlock() + }) + + cfg := testStorageCfg() + proc, mp := mockSqlProc(t) + var invalidated atomic.Bool + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + if strings.Contains(sql, "index_id") && strings.Contains(sql, "__meta") { + if invalidated.CompareAndSwap(false, true) { + // This is the production publication-before-remove window: the + // loading cache entry still owns the key while the generation bump + // becomes visible to its Fulltext2Search.Load. + NewFulltext2Search(cfg).OnCacheInvalidated(string(LoadMissCDCFlush)) + } + return executor.Result{Mp: mp}, nil + } + if strings.Contains(sql, "chunk_id, data") { + return executor.Result{Mp: mp}, nil + } + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + }) + + loader := NewFulltext2Search(cfg) + require.NoError(t, tc.call(proc, loader, cfg)) + require.True(t, invalidated.Load()) + veccache.Cache.Remove(cfg.IndexTable) + }) + } +} + +// TestStaleGenSqls pins the cache-freshness generation queries: MAX(timestamp) over the +// metadata table (REBUILD/MERGE signal) and MAX(chunk_id) over the tag=1 CdcTail (CDC-append +// signal), scoped to (CdcTailId, tag=1) so a base sub-index cannot mask a fresh append. +func TestStaleGenSqls(t *testing.T) { + cfg := TableConfig{DbName: "db", IndexTable: "__store", MetadataTable: "__meta"} + tsSQL, tailSQL := StaleGenSqls(cfg) + require.Contains(t, tsSQL, "MAX(timestamp)") + require.Contains(t, tsSQL, "`db`.`__meta`") + require.Contains(t, tailSQL, "MAX(chunk_id)") + require.Contains(t, tailSQL, "`db`.`__store`") + require.Contains(t, tailSQL, "tag = 1") // tag=Tag_CdcEvents + require.Contains(t, tailSQL, vectorindex.CdcTailId) // scoped to the single CDC tail +} + +// TestIsStaleUncheckableEvicts: the index loaded fine (loadedSearch assembles segments in memory) +// but no generation was captured (genValid=false — the load-time capture failed). IsStale must +// report stale, NOT a no-op: an uncheckable entry whose TTL keeps sliding on every search would +// otherwise serve pre-CDC/rebuild data forever. Reporting stale forces a bounded evict+reload +// that retries capture and self-heals once it succeeds. +func TestIsStaleUncheckableEvicts(t *testing.T) { + s := loadedSearch(t) + require.False(t, s.genValid) // loaded, but generation never captured + stale, err := s.IsStale() + require.NoError(t, err) + require.True(t, stale, "an entry that can't self-check freshness must be evicted, not pinned") +} + +func TestFulltext2SearchEmptyIndex(t *testing.T) { + proc := newSearchProc(t) + s := NewFulltext2Search(TableConfig{IndexTable: "__store", Parser: ParserDefault}) + s.idx = NewIndex(nil, nil) // loaded but doc-less + s.loaded = true + + keys, dists, err := s.Search(proc, Fulltext2Query{Pattern: []byte("fox")}, vectorindex.RuntimeConfig{}) + require.NoError(t, err) + require.Empty(t, keys) + require.Empty(t, dists) +} + +func TestFulltext2SearchInvalidPayload(t *testing.T) { + proc := newSearchProc(t) + s := loadedSearch(t) + defer s.Destroy() + + _, _, err := s.Search(proc, "not a query", vectorindex.RuntimeConfig{}) + require.ErrorContains(t, err, "invalid query payload") +} + +func TestFulltext2SearchTopK(t *testing.T) { + proc := newSearchProc(t) + s := loadedSearch(t) + defer s.Destroy() + + // single-term NL query with a pushed LIMIT. + keys, dists, err := s.Search(proc, Fulltext2Query{Pattern: []byte("fox"), Algo: BM25}, + vectorindex.RuntimeConfig{Limit: 10}) + require.NoError(t, err) + ks, ok := keys.([]any) + require.True(t, ok) + require.Len(t, dists, len(ks)) + require.NotEmpty(t, ks) // "fox" hits docs 0 and 2 + + // k <= 0 (no pushed LIMIT) falls back to NumDocs. + keys, _, err = s.Search(proc, Fulltext2Query{Pattern: []byte("fox"), Algo: BM25}, + vectorindex.RuntimeConfig{Limit: 0}) + require.NoError(t, err) + require.NotEmpty(t, keys.([]any)) + + // an absurd LIMIT past MaxInt32 is clamped, not wrapped negative. + keys, _, err = s.Search(proc, Fulltext2Query{Pattern: []byte("fox"), Algo: BM25}, + vectorindex.RuntimeConfig{Limit: uint(math.MaxInt32) + 100}) + require.NoError(t, err) + require.NotEmpty(t, keys.([]any)) + + // bag-of-words (IN BM25 MODE) path. + keys, _, err = s.Search(proc, Fulltext2Query{Pattern: []byte("quick fox"), BagOfWords: true, Algo: BM25}, + vectorindex.RuntimeConfig{Limit: 10}) + require.NoError(t, err) + require.NotEmpty(t, keys.([]any)) +} + +func TestFulltext2SearchStreamingEmit(t *testing.T) { + proc := newSearchProc(t) + s := loadedSearch(t) + defer s.Destroy() + + // Emit set + no pushed LIMIT → streaming: results handed off via Emit, empty return. + for _, bagOfWords := range []bool{false, true} { + emitted := 0 + emit := func(o *vectorindex.SearchOutput) error { + emitted += o.Keys.N + PutColumnBuffer(o.Keys) // recycle like the real consumer + return nil + } + keys, dists, err := s.Search(proc, + Fulltext2Query{Pattern: []byte("fox"), BagOfWords: bagOfWords, Algo: BM25}, + vectorindex.RuntimeConfig{Emit: emit}) + require.NoError(t, err) + require.Empty(t, keys) + require.Empty(t, dists) + require.Positive(t, emitted, "bagOfWords=%v should emit docs", bagOfWords) + } +} + +func TestFulltext2SearchDestroy(t *testing.T) { + s := loadedSearch(t) + + // The cached config is immutable for the entry's lifetime (no UpdateConfig hook — + // a config change evicts the entry), so Search is pure-read; here we just pin that + // the constructed cfg is what Load queries with and that Destroy tears down cleanly. + require.Equal(t, ParserDefault, s.cfg.Parser) + + // Destroy frees and clears the loaded index. + s.Destroy() + require.Nil(t, s.idx) + require.False(t, s.loaded) +} + +// TestLoadGenerationHappy stubs the package runSql to cover the fulltext2 generation reader +// (StaleGenSqls + resultScalarInt64 + the two-read LoadGeneration body). +func TestLoadGenerationHappy(t *testing.T) { + mp := mpool.MustNewZero() + old := runSql + defer func() { runSql = old }() + n := 0 + runSql = func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + n++ + bat := batch.NewWithSize(1) + bat.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + v := int64(11) + if n == 2 { + v = 22 + } + require.NoError(t, vector.AppendFixed[int64](bat.Vecs[0], v, false, mp)) + bat.SetRowCount(1) + return executor.Result{Mp: mp, Batches: []*batch.Batch{bat}}, nil + } + ts, tail, err := LoadGeneration(nil, TableConfig{DbName: "db", MetadataTable: "m", IndexTable: "s"}) + require.NoError(t, err) + require.Equal(t, int64(11), ts) // MAX(timestamp) + require.Equal(t, int64(22), tail) // MAX(chunk_id) tag=1 +} + +// TestLoadGenerationRecover: if the generation read panics (e.g. a background housekeeping call +// hits a torn-down executor), LoadGeneration must recover it into an error — never let it crash +// the caller. The caller then leaves genValid=false, and IsStale evicts to retry (see +// TestIsStaleUncheckableEvicts). +func TestLoadGenerationRecover(t *testing.T) { + old := runSql + defer func() { runSql = old }() + runSql = func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + panic("simulated executor teardown") + } + _, _, err := LoadGeneration(nil, TableConfig{DbName: "db", MetadataTable: "m", IndexTable: "s"}) + require.Error(t, err) + require.Contains(t, err.Error(), "LoadGeneration recovered") +} + +// TestFulltext2IsStaleQueryError: with a captured generation but an unresolvable CN service, the +// background QueryGeneration read errors (its recover turns the ServiceRuntime panic into an +// error), and IsStale treats a query error as stale so a dropped/rebuilt index's dead cache entry +// is reclaimed — while surfacing the error for logging. +func TestFulltext2IsStaleQueryError(t *testing.T) { + s := NewFulltext2Search(TableConfig{DbName: "db", MetadataTable: "m", IndexTable: "s"}) + s.genValid = true + s.cnUUID = "no-such-cn-uuid" + stale, err := s.IsStale() + require.Error(t, err) + require.True(t, stale) +} + +// TestFulltext2SearchInto pins the box-free LIMIT path: SearchInto fills the caller-owned +// SearchOutput (pk column, float32 scores, one nullable ColumnBuffer per FULL INCLUDE column) +// — box-free and reused across calls. incIdx has 5 docs all matching "x", includes +// [status varchar, prio int64] with a NULL status (pk4). +func TestFulltext2SearchInto(t *testing.T) { + proc := newSearchProc(t) + idx := incIdx(t) + s := &Fulltext2Search{idx: idx, loaded: true, cfg: TableConfig{IndexTable: "__store", Parser: ParserDefault}} + mp := mpool.MustNewZero() + + out := &vectorindex.SearchOutput{} + rt := vectorindex.RuntimeConfig{Limit: 10, RequestedIncludeColumns: []string{"status", "prio"}} + require.NoError(t, s.SearchInto(proc, Fulltext2Query{Pattern: []byte("x"), Algo: BM25}, rt, out)) + + require.Equal(t, 5, out.Keys.N) // all 5 docs contain "x" + require.Len(t, out.Dists, 5) + require.Len(t, out.Include, 2) // status, prio (FULL include order) + + // Decode the box-free buffers into vectors and zip by row (Keys[i] <-> Include[*][i]) into + // a pk -> (status, prio) map (result order is score-desc; equal scores are unspecified). + keyVec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(out.Keys, keyVec, mp)) + statusVec := vector.NewVec(types.T_varchar.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(out.Include[0], statusVec, mp)) + prioVec := vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(out.Include[1], prioVec, mp)) + + pks := vector.MustFixedColWithTypeCheck[int64](keyVec) + prios := vector.MustFixedColWithTypeCheck[int64](prioVec) + require.Len(t, pks, 5) + type sv struct { + status any + prio int64 + } + got := map[int64]sv{} + for i := range pks { + var st any + if !statusVec.IsNull(uint64(i)) { + st = statusVec.GetStringAt(i) + } + got[pks[i]] = sv{st, prios[i]} + } + require.Equal(t, sv{"active", int64(10)}, got[1]) + require.Equal(t, sv{"inactive", int64(20)}, got[2]) + require.Equal(t, sv{"active", int64(30)}, got[3]) + require.Equal(t, sv{nil, int64(40)}, got[4]) // NULL status preserved + require.Equal(t, sv{"archived", int64(5)}, got[5]) + + // Reuse: a second SearchInto Resets out and refills (no stale rows accumulated). + require.NoError(t, s.SearchInto(proc, Fulltext2Query{Pattern: []byte("x"), Algo: BM25}, rt, out)) + require.Equal(t, 5, out.Keys.N) + require.Len(t, out.Dists, 5) + + // No requested INCLUDE columns → out.Include emptied. + require.NoError(t, s.SearchInto(proc, Fulltext2Query{Pattern: []byte("x"), Algo: BM25}, + vectorindex.RuntimeConfig{Limit: 10}, out)) + require.Equal(t, 5, out.Keys.N) + require.Empty(t, out.Include) + + // nil out → error, not a nil-deref. + require.Error(t, s.SearchInto(proc, Fulltext2Query{Pattern: []byte("x")}, rt, nil)) +} + +// TestFulltext2SearchIntoNotLoaded / empty: the two prepare() early-outs on the SearchInto path. +func TestFulltext2SearchIntoNotLoadedAndEmpty(t *testing.T) { + proc := newSearchProc(t) + out := &vectorindex.SearchOutput{} + + // not loaded → error. + s := NewFulltext2Search(TableConfig{IndexTable: "__store"}) + require.ErrorContains(t, s.SearchInto(proc, Fulltext2Query{Pattern: []byte("x")}, vectorindex.RuntimeConfig{Limit: 10}, out), "not loaded") + + // loaded but empty index → no rows, no error, out emptied. + s2 := &Fulltext2Search{idx: NewIndex(nil, nil), loaded: true, cfg: TableConfig{IndexTable: "__store"}} + require.NoError(t, s2.SearchInto(proc, Fulltext2Query{Pattern: []byte("x")}, vectorindex.RuntimeConfig{Limit: 10}, out)) +} diff --git a/pkg/fulltext2/search_test.go b/pkg/fulltext2/search_test.go new file mode 100644 index 0000000000000..f5ebabf8713f3 --- /dev/null +++ b/pkg/fulltext2/search_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// A small fulltext corpus (int64 pk -> text), tokenized with the dict-free +// SimpleTokenizer — the same tokenizer used for both docs and queries so phrase +// adjacency is consistent. +func fulltextCorpus(t *testing.T) *Segment { + t.Helper() + docs := []Doc{ + {int64(1), []byte("the quick brown fox jumps over the lazy dog")}, // len 9 + {int64(2), []byte("a quick brown fox")}, // len 4 + {int64(3), []byte("the lazy dog sleeps all day")}, // len 6 + {int64(4), []byte("quick brown fox jumps high")}, // len 5 + {int64(5), []byte("the dog and the fox")}, // len 5 + } + s, err := BuildSegmentFromDocs("seg", int32(types.T_int64), docs, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + return s +} + +func pkslice(rs []Result) []any { + out := make([]any, len(rs)) + for i, r := range rs { + out[i] = r.Pk + } + return out +} + +func query(t *testing.T, s *Segment, q string, algo ScoreAlgo, k int) []any { + t.Helper() + rs, err := s.SearchText([]byte(q), tokenizer.NewSimpleTokenizer(), algo, k) + require.NoError(t, err) + return pkslice(rs) +} + +// TestNLPhraseMatching: MATCH(col) AGAINST('...') in NL mode = exact contiguous +// phrase. Asserts the matched doc SET across representative fulltext cases. +func TestNLPhraseMatching(t *testing.T) { + s := fulltextCorpus(t) + + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, query(t, s, "quick brown fox", TfIdf, 10)) + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, query(t, s, "brown fox", TfIdf, 10)) + require.ElementsMatch(t, []any{int64(1), int64(3)}, query(t, s, "lazy dog", TfIdf, 10)) + require.ElementsMatch(t, []any{int64(1), int64(4)}, query(t, s, "fox jumps", TfIdf, 10)) + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4), int64(5)}, query(t, s, "fox", TfIdf, 10)) + + // "the fox" is contiguous only in doc 5 ("...the fox"); doc 1 has "the" and + // "fox" but not adjacent. + require.Equal(t, []any{int64(5)}, query(t, s, "the fox", TfIdf, 10)) + + // non-contiguous phrase — "quick" and "fox" are never adjacent (brown between) + require.Empty(t, query(t, s, "quick fox", TfIdf, 10)) + // absent term + require.Empty(t, query(t, s, "dog cat", TfIdf, 10)) + require.Empty(t, query(t, s, "unicorn", TfIdf, 10)) +} + +// TestTfIdfOrdering: equal-tf hits tie-break by ascending ord (deterministic). +func TestTfIdfOrdering(t *testing.T) { + s := fulltextCorpus(t) + // The three docs contain the phrase once each → equal TfIdf, so order is unspecified. + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, query(t, s, "quick brown fox", TfIdf, 10)) +} + +// TestBM25Ordering: BM25 length-normalizes, so among equal-tf phrase hits the +// shorter document ranks higher: doc2(len4) > doc4(len5) > doc1(len9). +func TestBM25Ordering(t *testing.T) { + s := fulltextCorpus(t) + require.Equal(t, []any{int64(2), int64(4), int64(1)}, query(t, s, "quick brown fox", BM25, 10)) +} + +// TestTopK caps the result count. +func TestTopK(t *testing.T) { + s := fulltextCorpus(t) + require.Len(t, query(t, s, "fox", TfIdf, 2), 2) // 4 hits, top-2 + require.Len(t, query(t, s, "fox", TfIdf, 10), 4) +} + +// TestSearchOnLoadedSegment: build → serialize → deserialize → query, so the +// loaded (FST + ordinal postings) path produces identical results. +func TestSearchOnLoadedSegment(t *testing.T) { + data, err := fulltextCorpus(t).Serialize() + require.NoError(t, err) + loaded, err := Deserialize("seg", bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + require.ElementsMatch(t, []any{int64(1), int64(2), int64(4)}, // equal TfIdf → order unspecified + query(t, loaded, "quick brown fox", TfIdf, 10)) + require.Equal(t, []any{int64(5)}, query(t, loaded, "the fox", TfIdf, 10)) + require.Empty(t, query(t, loaded, "quick fox", TfIdf, 10)) +} diff --git a/pkg/fulltext2/segment.go b/pkg/fulltext2/segment.go new file mode 100644 index 0000000000000..8754d0f222258 --- /dev/null +++ b/pkg/fulltext2/segment.go @@ -0,0 +1,784 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "os" + "sort" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +const ( + // MaxCappedTf caps stored term frequency at one byte (matches classic + // fulltext's tf cap and bm25's MaxCappedTf) — long docs with a hugely + // repeated term saturate rather than skewing the score. + MaxCappedTf = 255 + + // BlockSize is the doc count per block-max skip block (matches bm25's + // BlockSize). One (lastDoc, maxTf, minDocLen) triple is derived per block at + // load, letting WAND skip whole blocks whose score bound can't make top-k. + BlockSize = 128 +) + +// termPostings is the in-memory posting list for one term, ordered by doc ord. +// +// It mirrors bm25's termPostings but carries POSITIONS (per doc, the token +// offsets of this term) — the positional payload phrase / NL adjacency and +// ngram reassembly need, which the position-free bm25 engine omits. The score +// upper-bound inputs (maxTf, minDocLen, block-max) are stored RAW and +// idf/avgdl-free so one segment serves both TF-IDF and BM25; the active scorer +// derives its max-impact bound at query time. +type termPostings struct { + docIDs []int64 // BUILD-side doc ords, ascending, len == df; nil on a LOADED segment + tfs []uint8 // BUILD-side parallel capped tf (<= MaxCappedTf); nil on a LOADED segment + positions [][]int32 // BUILD-side per-doc positions (len == df); nil on a LOADED segment + + // ndoc is the document frequency (df). On a LOADED segment docIDs/tfs are NOT + // expanded — they live block-compressed in blockData (a view into the mmap) — + // so ndoc is the authoritative count. On a build-side segment it equals + // len(docIDs) (set by deriveTermStats). + ndoc int + + // LOADED-side docID/tf blocks: this term's postings kept COMPRESSED as + // BlockSize-doc blocks (per block: docID gaps as delta+varint from the previous + // block's last ord, then the block's raw tf bytes), a view into the segment's + // mmap. docIDs are the largest resident section (~46% at load), needed in full + // ONLY by phrase verification / MERGE / boolean — WAND ranking touches only the + // blocks its block-max walk lands on. So they are NOT expanded at load; the WAND + // cursor decodes one block on demand (fillBlock), and the cold paths materialize + // transiently (materializeDocIDs/materializeTfs). blockOff[b] is block b's byte + // offset within blockData (len nblk+1, cumulative). nil on a build-side segment. + blockData []byte + blockOff []int64 + + // LOADED-side positions (Deserialize): this term's positions kept COMPRESSED + // (delta+varint, per doc: pc + position gaps), a view into the segment's + // resident positions buffer. Positions are the largest section (~48%) and are + // needed ONLY for phrase verification / MERGE — never for WAND ranking — so they + // are NOT expanded into RAM at load; callers decode on demand via + // materializePositions() (whole list) or fillBlockPositions() (one block, for the + // phrase cursor). nil on a build-side segment. + posRaw []byte + // blockPosOff[b] is block b's byte offset within posRaw (len nblk+1, cumulative), + // so a block's positions are seekable without decoding the prior blocks — the + // position analogue of blockOff. nil on a build-side segment (positions is flat there). + blockPosOff []int64 + + // Term-level score-UB inputs (raw, scorer-agnostic). + maxTf uint8 // max tf over all postings + minDocLen int32 // min doc length over all postings + + // Block-Max skip-block metadata, one entry per ceil(df/BlockSize). Computed at + // BUILD (deriveTermStats) and STORED in the ranking directory, read back RESIDENT + // at load (never re-derived from the compressed blocks). Raw / scorer-agnostic. + blockLastDoc []int64 // last (max) ord in each block + blockMaxTf []uint8 // max tf in each block + blockMinDocLn []int32 // min doc length in each block +} + +// df is the document frequency (number of docs containing the term). +func (p *termPostings) df() int { + if p.docIDs != nil { + return len(p.docIDs) // build-side (authoritative before ndoc is set) + } + return p.ndoc +} + +// nblk is the number of Block-Max skip blocks (== ceil(df/BlockSize)). +func (p *termPostings) nblk() int { return len(p.blockLastDoc) } + +// blockLen is the number of postings in block b (BlockSize, or the remainder for +// the last block). +func (p *termPostings) blockLen(b int) int { + n := p.df() - b*BlockSize + if n > BlockSize { + n = BlockSize + } + return n +} + +// fillBlock decodes block b's docIDs and tfs into outDocs/outTfs (each cap >= +// BlockSize) and returns the block length. Build-side copies the flat slices; +// loaded-side varint-decodes the block from blockData (the mmap view): docID gaps +// accumulate from the previous block's last ord (resident blockLastDoc[b-1]), then +// the raw tf bytes follow. The WAND cursor calls this once per block it lands on; +// materializeDocIDs/Tfs call it per block to rebuild the flat arrays. +func (p *termPostings) fillBlock(b int, outDocs []int64, outTfs []uint8) int { + blen := p.blockLen(b) + if p.docIDs != nil { // build-side: copy from the flat arrays + lo := b * BlockSize + copy(outDocs[:blen], p.docIDs[lo:lo+blen]) + copy(outTfs[:blen], p.tfs[lo:lo+blen]) + return blen + } + data := p.blockData[p.blockOff[b]:p.blockOff[b+1]] + var prev int64 + if b > 0 { + prev = p.blockLastDoc[b-1] + } + off := 0 + for i := 0; i < blen; i++ { + // Guard corrupt/torn block bytes that survive the CRC: a malformed docID varint + // (n<=0) or one that over-consumes would otherwise make the tf copy below slice + // out of range and panic the query goroutine. Return the docs decoded so far — + // the sibling fillBlockPositions degrades the same way rather than crashing. + if off >= len(data) { + return i + } + g, n := binary.Uvarint(data[off:]) + if n <= 0 { + return i + } + off += n + prev += int64(g) + outDocs[i] = prev + } + // tfs are blen raw bytes after the docID gaps; on corruption they may not all be + // present. Copy only what the block actually holds (never past len(data)). + if off+blen > len(data) { + blen = len(data) - off + } + copy(outTfs[:blen], data[off:off+blen]) + return blen +} + +// fillBlockDocs decodes only block b's doc IDs. Phrase intersection never reads term +// frequency, so using fillBlockDocs there avoids a redundant tf copy for every block the +// doc cursor visits. WAND keeps using fillBlock because it scores with tf. +func (p *termPostings) fillBlockDocs(b int, outDocs []int64) int { + blen := p.blockLen(b) + if p.docIDs != nil { + lo := b * BlockSize + copy(outDocs[:blen], p.docIDs[lo:lo+blen]) + return blen + } + data := p.blockData[p.blockOff[b]:p.blockOff[b+1]] + var prev int64 + if b > 0 { + prev = p.blockLastDoc[b-1] + } + off := 0 + for i := 0; i < blen; i++ { + if off >= len(data) { + return i + } + g, n := binary.Uvarint(data[off:]) + if n <= 0 { + return i + } + off += n + prev += int64(g) + outDocs[i] = prev + } + return blen +} + +// fillBlockPositions decodes block b's per-doc token positions into out[:blen], +// returning the block length. loaded-side varint-decodes only block b's slice of posRaw +// (via blockPosOff) — the per-block analogue of fillBlock, so the phrase cursor holds ONE +// block's positions (O(BlockSize)) instead of the whole list. +// +// out[i] is REUSED in place (out[i] = append(out[i][:0], …)): out is a pooled phraseBuf +// buffer (getPhraseBuf), and reallocating a fresh []int32 per doc per block both churned +// the allocator under concurrent phrase load and left the pool holding stale per-doc +// arrays. Reusing keeps each slot at its high-water capacity and makes the pool actually +// pool. The build-side branch COPIES the flat slice into the owned out[i] (rather than +// aliasing p.positions) so a buffer reused later on a loaded segment can never write back +// into build data. matchPhrase reads positions via range / sortedContainsInt32, both of +// which treat a nil and an empty slice identically, so the degenerate (no-data / corrupt) +// slots leave out[i] empty rather than nil. +func (p *termPostings) fillBlockPositions(b int, out [][]int32) int { + blen := p.blockLen(b) + if p.positions != nil { // build-side: copy the flat slice into the buffer's own arrays + lo := b * BlockSize + for i := 0; i < blen; i++ { + out[i] = append(out[i][:0], p.positions[lo+i]...) + } + return blen + } + data := p.posRaw[p.blockPosOff[b]:p.blockPosOff[b+1]] + off := 0 + for i := 0; i < blen; i++ { + out[i] = out[i][:0] // reuse the pooled backing array; empty ≡ nil for the reader + if off >= len(data) { + continue + } + pc, n := binary.Uvarint(data[off:]) + if n <= 0 { + continue + } + off += n + if pc > uint64(len(data)-off) { // corrupt guard (each gap is >= 1 byte) + pc = uint64(len(data) - off) + } + var pp int32 + for m := uint64(0); m < pc; m++ { + g, k := binary.Uvarint(data[off:]) + if k <= 0 { + break + } + off += k + pp += int32(g) + out[i] = append(out[i], pp) + } + } + return blen +} + +// materializeDocIDs returns this term's full ascending doc ords (df entries): the +// build-side flat slice as-is, or a transient decode of every loaded block. Cold +// paths (phrase / MERGE / boolean) that scan all postings call this ONCE; WAND +// ranking never does (it decodes only the blocks its walk lands on). +func (p *termPostings) materializeDocIDs() []int64 { + if p.docIDs != nil { + return p.docIDs + } + out := make([]int64, p.ndoc) + var scratch [BlockSize]uint8 + for b := 0; b < p.nblk(); b++ { + lo := b * BlockSize + p.fillBlock(b, out[lo:], scratch[:]) + } + return out +} + +// materializeTfs returns this term's full capped tf bytes (df entries): the +// build-side flat slice as-is, or a transient decode of every loaded block. +func (p *termPostings) materializeTfs() []uint8 { + if p.tfs != nil { + return p.tfs + } + out := make([]uint8, p.ndoc) + var scratch [BlockSize]int64 + for b := 0; b < p.nblk(); b++ { + lo := b * BlockSize + p.fillBlock(b, scratch[:], out[lo:]) + } + return out +} + +// materializePositions decodes this term's per-doc token positions (df entries): +// the build-side stored [][]int32 as-is, or the loaded-side compressed posRaw +// (delta+varint) transiently. Positions are NOT resident on a loaded segment, so a +// caller that needs them (phrase verification, MERGE) materializes ONCE per query +// and discards — WAND ranking never calls this. Panics on a build-side... no: a +// build-side segment returns tp.positions directly. +func (p *termPostings) materializePositions() [][]int32 { + if p.positions != nil { + return p.positions + } + df := p.df() + out := make([][]int32, df) + off := 0 + for i := 0; i < df; i++ { + if off >= len(p.posRaw) { + break // truncated/corrupt (defense-in-depth on already-checksummed data) + } + pc, n := binary.Uvarint(p.posRaw[off:]) + if n <= 0 { + break + } + off += n + // A doc's position count cannot exceed the remaining bytes (each gap is >= 1 + // varint byte), so a garbage pc can't make() an unbounded slice. + if pc > uint64(len(p.posRaw)-off) { + pc = uint64(len(p.posRaw) - off) + } + doc := make([]int32, pc) + var pp int32 + for m := uint64(0); m < pc; m++ { + g, k := binary.Uvarint(p.posRaw[off:]) + if k <= 0 { + break + } + off += k + pp += int32(g) + doc[m] = pp + } + out[i] = doc + } + return out +} + +// Segment is one loadable in-memory fulltext v2 index unit — a tag=0 base sub or +// a tag=1 CDC tail frame (§4 of fulltext2.md). It is the positional analogue of +// bm25's WandModel: same docmap (ord → pk, docLen) and avgDocLen, but its term +// dict is keyed by the actual indexed term STRING and kept sorted so one +// structure serves both O(log n) lookup and prefix enumeration (`word*`). +type Segment struct { + Id string + N int64 // number of documents (= len(pks)) + PkType int32 // types.T of the source primary key (output decode + membership) + AvgDocLen float64 // average doc length; set at LOAD across all loaded segments + + // Recency orders segments for liveness when the same pk lands in several + // (UPDATE / reinsert / a stale base copy) — only the highest-Recency copy is + // live. Same semantics as bm25's WandModel.Recency (chunk_id for a tag=1 + // tail delta, metadata.recency for a tag=0 base). + Recency int64 + + // pks is the BUILD-side ord->pk slice (set while building an in-memory segment). + // On a LOADED segment it is nil: instead of materializing N boxed `any` pks (~24 B + // each — the O(docs) resident floor after the liveLoc removal), decodeDocmap keeps + // pkOffsets (4 B/doc, the byte offset of each pk in the docmap bytes) + pkRaw (a + // VIEW into those bytes, mmap-backed for a base), and pk(ord) decodes on demand. + pks []any + pkOffsets []int32 // loaded-side: ord -> offset of pk's length prefix in pkRaw + pkRaw []byte // loaded-side: the docmap bytes (view; pks live here len-prefixed) + docLen []int32 // ord -> document length (token count), for BM25 + + // INCLUDE columns: actual per-doc scalar values stored in the docmap (ord-aligned), + // used for in-index prefiltering and covering projection so a covered query needs no + // JOIN back to the base table. Encoded with the same pk codec (fixed-width dense for + // numerics, [u32 len][content] for varchar/char). All slices are nil when the index has + // no INCLUDE columns. + // + // Build-side: includeVals[ord] is the row's per-column values ([]any of len(includeTypes)); + // a nil element is SQL NULL. Loaded-side: includeVals is nil, and + // includeOffsets[ord*nIncl+col] is the byte offset (into includeRaw, a view over the + // docmap bytes == pkRaw) of column col's [nullFlag][u32 len][content] entry, decoded on + // demand by includeVal(ord, col). + // The docmap include section splits columns into a dense stride-addressed FIXED region + // (the integer family — no per-doc offset needed, computed by stride) and an + // offset-addressed VARLENA region (varchar/char). So a numeric-only INCLUDE index keeps + // ZERO resident offset entries; only varlena columns cost a resident offset table. + includeTypes []int32 // types.T of each INCLUDE column (len = nIncl, col order) + includeVals [][]any // build-side: ord -> per-column values (nil element = NULL) + includeRaw []byte // loaded-side: docmap bytes view (== pkRaw), mmap-backed on a base + includeLay includeLayout // loaded-side: derived per-column fixed/varlena addressing + // loaded-side: byte offset of the FIXED region within includeRaw; the VARLENA offset table + // (ord*nVarlena+varlenaIdx -> offset). includeVarOffsets is nil when all columns are fixed. + includeFixedStart int + includeVarOffsets []int32 + + // term dict, BUILD-side representation — dictionary-free, keyed by the + // indexed term string. `terms` accumulates postings during build and gives + // O(1) exact lookup; `sortedTerms` is the ascending key list. On serialize + // these feed buildTermDictFST (termdict.go), whose vellum FST is the compact + // on-disk / loaded form used for query lookup + prefix. Kept consistent: + // every key in `terms` appears once in `sortedTerms`. + terms map[string]*termPostings + sortedTerms []string + + // term dict, LOADED-side representation — set by Deserialize, nil on a + // build-side segment. `dict` is the vellum FST mapping term → the BYTE OFFSET of + // that term's self-contained directory entry in `ranking`. A loaded segment does + // NOT expand any term at load: query lookup (LookupLoaded) decodes just the touched + // term's directory entry from `ranking` on demand and points its blocks/positions at + // `blocks`/`positions` — so the resident directory heap is O(the current query), not + // O(vocabulary). `ranking`/`blocks`/`positions` are views into the mmap/blob (kept + // alive by mmapData or GC). The build-side `terms` map is left nil. + dict *termDict + ranking, blocks, positions []byte + + // mmapData is the shared read-only mmap of a base segment's on-disk file: the + // FST, the compressed docID/tf blocks, and the compressed positions section are + // all views into it (page-cache-backed, reclaimable, shared by all concurrent + // queries — no copy, no off-heap). mmapPath is that file (empty for the anonymous + // SSD file, whose inode is freed by munmap). Both are released by Free() under the + // cache's eviction write-lock (no reader in flight). nil on a build-side or + // in-memory (tail) segment, whose bytes are GC-managed Go slices. + mmapData []byte + mmapPath string + // ownedBytes is the unique serialized archive retained by a loaded segment. + // The tail pool counts this once because pkRaw, FST, ranking, blocks, and + // positions are views into the same archive. + ownedBytes int64 + lease *segmentLease // non-nil for a view into the immutable base pool +} + +// numDocs is the segment's document count, valid for both a build-side segment (== len(pks)) +// and a loaded segment (== N, where pks is nil and pk(ord) decodes from pkRaw). +func (s *Segment) numDocs() int { return int(s.N) } + +// pk returns the original pk value at ord. Build-side segments read the resident pks +// slice; loaded segments decode on demand from pkRaw at pkOffsets[ord] (a length-prefixed +// pk), so no O(docs) []any is held resident. pkType is validated once at decodeDocmap, so +// decodePk cannot fail here for a well-formed segment. +func (s *Segment) pk(ord int64) any { + if s.pks != nil { + return s.pks[ord] + } + off := int(s.pkOffsets[ord]) + l := int(binary.LittleEndian.Uint32(s.pkRaw[off:])) + v, _ := decodePk(s.PkType, s.pkRaw[off+4:off+4+l]) + return v +} + +// includeLayout is the derived per-column addressing for the docmap include section: a dense +// stride-addressed FIXED region (the integer family) plus an offset-addressed VARLENA region +// (varchar/char). Fixed columns need NO per-doc offset — their value sits at a computed +// (ord*fixedStride + maskBytes + fixedValOff[col]) position — so only varlena columns cost a +// resident offset table. Derived once from the column types (build encode + load decode). +type includeLayout struct { + fixedIdx []int // per col: index among fixed cols (0..nFixed-1), or -1 if varlena + varlenaIdx []int // per col: index among varlena cols (0..nVarlena-1), or -1 if fixed + fixedValOff []int // per col: byte offset of its value within a row's fixed value region (fixed cols) + fixedWidth []int // per col: byte width (fixed cols) + nFixed int + nVarlena int + maskBytes int // ceil(nFixed/8): the per-row NULL bitmask for fixed cols + fixedStride int // maskBytes + sum(fixedWidth): bytes per doc in the fixed region +} + +// computeIncludeLayout derives the fixed/varlena addressing from the INCLUDE column types. +func computeIncludeLayout(types []int32) includeLayout { + l := includeLayout{ + fixedIdx: make([]int, len(types)), + varlenaIdx: make([]int, len(types)), + fixedValOff: make([]int, len(types)), + fixedWidth: make([]int, len(types)), + } + valOff := 0 + for c, t := range types { + l.fixedIdx[c], l.varlenaIdx[c] = -1, -1 + if w, ok := fixedPkByteWidth(t); ok { + l.fixedIdx[c] = l.nFixed + l.fixedWidth[c] = w + l.fixedValOff[c] = valOff + valOff += w + l.nFixed++ + } else { + l.varlenaIdx[c] = l.nVarlena + l.nVarlena++ + } + } + l.maskBytes = (l.nFixed + 7) / 8 + l.fixedStride = l.maskBytes + valOff + return l +} + +// nIncludeCols is the number of INCLUDE columns carried by this segment (0 = none). +func (s *Segment) nIncludeCols() int { return len(s.includeTypes) } + +// decodeInclude returns the doc's INCLUDE column values (all columns, in index order; a nil +// element = SQL NULL), or nil when the segment has no INCLUDE columns. Carried out with a +// search Result so a covering query needs no base-table JOIN. +func (s *Segment) decodeInclude(ord int64) []any { + n := len(s.includeTypes) + if n == 0 { + return nil + } + out := make([]any, n) + for c := 0; c < n; c++ { + if v, isNull, err := s.includeVal(ord, c); err == nil && !isNull { + out[c] = v + } + } + return out +} + +// includeRawAt locates INCLUDE column col at ord in a LOADED segment (includeVals == nil) +// and returns the raw value bytes — for a FIXED column the width bytes, for a VARLENA +// column the content WITHOUT its [nullFlag][u32 len] prefix — whether the value is SQL NULL, +// and an error. It is the single decode primitive shared by includeVal (which then decodePk's +// the bytes) and appendIncludeTo (which copies them box-free). +// +// Belt-and-suspenders bounds (F3): every computed offset into includeRaw / includeVarOffsets +// is validated against the slice length, and a malformed/out-of-range segment returns an +// error rather than panicking on an out-of-range slice. Callers must pass col in [0,nIncl); +// includeRawAt still re-checks it defensively. The returned byte slice aliases includeRaw +// (mmap-backed on a loaded base segment) — the caller must copy before the segment is freed. +func (s *Segment) includeRawAt(ord int64, col int) (content []byte, isNull bool, err error) { + if col < 0 || col >= len(s.includeTypes) || ord < 0 { + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include access out of range (ord=%d col=%d)", ord, col) + } + lay := &s.includeLay + n := len(s.includeRaw) + if fi := lay.fixedIdx[col]; fi >= 0 { + // FIXED region: value at a stride-computed position, NULL via the per-row bitmask. + rowBase := s.includeFixedStart + int(ord)*lay.fixedStride + if rowBase < 0 || rowBase+lay.fixedStride > n { + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include fixed row out of range (ord=%d col=%d)", ord, col) + } + if s.includeRaw[rowBase+fi/8]&(1<<(uint(fi)&7)) != 0 { + return nil, true, nil + } + valOff := rowBase + lay.maskBytes + lay.fixedValOff[col] + w := lay.fixedWidth[col] + if valOff < 0 || valOff+w > n { // within rowBase+fixedStride, but re-check the exact span + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include fixed value out of range (ord=%d col=%d)", ord, col) + } + return s.includeRaw[valOff : valOff+w], false, nil + } + // VARLENA region: offset-addressed [nullFlag][u32 len][content]. + oi := int(ord)*lay.nVarlena + lay.varlenaIdx[col] + if oi < 0 || oi >= len(s.includeVarOffsets) { + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include varlena offset out of range (ord=%d col=%d)", ord, col) + } + off := int(s.includeVarOffsets[oi]) + if off < 0 || off >= n { + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include varlena entry out of range (ord=%d col=%d)", ord, col) + } + if s.includeRaw[off] == includeNull { + return nil, true, nil + } + if off+5 > n { // [nullFlag(1)][u32 len(4)] + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include varlena header out of range (ord=%d col=%d)", ord, col) + } + l := int(binary.LittleEndian.Uint32(s.includeRaw[off+1:])) + if l < 0 || off+5+l > n { + return nil, false, moerr.NewInternalErrorNoCtxf("fulltext2: include varlena content out of range (ord=%d col=%d)", ord, col) + } + return s.includeRaw[off+5 : off+5+l], false, nil +} + +// includeVal returns the value of INCLUDE column col at ord and whether it is SQL NULL. +// Build-side reads the resident includeVals; a loaded segment decodes on demand via +// includeRawAt (bounds-checked) + decodePk. Returns (nil, true, nil) when the segment has +// no INCLUDE columns. +func (s *Segment) includeVal(ord int64, col int) (any, bool, error) { + nIncl := len(s.includeTypes) + if nIncl == 0 || col < 0 || col >= nIncl { + return nil, true, nil + } + if s.includeVals != nil { // build-side + var v any + if int(ord) < len(s.includeVals) && col < len(s.includeVals[ord]) { + v = s.includeVals[ord][col] + } + return v, v == nil, nil + } + content, isNull, err := s.includeRawAt(ord, col) + if err != nil || isNull { + return nil, isNull, err + } + v, err := decodePk(s.includeTypes[col], content) + if err != nil { + return nil, false, err + } + return v, false, nil +} + +// appendIncludeTo decodes INCLUDE column col at ord straight into k in the box-free +// ColumnBuffer form (the mirror of appendPkTo, extended for NULLs): a fixed-width column +// contributes its width bytes, a varlena column a [u32 len][content] entry, and a NULL is +// recorded in k.Nulls with a well-formed placeholder (width zero-bytes / [u32 0]) so the +// Data cursor stays aligned for AppendColumnBuffer. k.Type MUST be col's type. Returns an +// error only on a malformed/out-of-range segment (includeRawAt bounds); the caller stops +// the stream so a torn batch is never emitted. +func (s *Segment) appendIncludeTo(k *vectorindex.ColumnBuffer, col int, ord int64) error { + fixedW, fixed := fixedPkByteWidth(int32(k.Type)) + if s.includeVals != nil { // build-side: re-encode the already-boxed value (no NEW box) + var v any + if int(ord) < len(s.includeVals) && col >= 0 && col < len(s.includeVals[ord]) { + v = s.includeVals[ord][col] + } + if v == nil { + cbAppendNull(k, fixedW, fixed) + return nil + } + content, err := encodePk(int32(k.Type), v) + if err != nil { + return err + } + cbAppendVal(k, content, fixedW, fixed) + return nil + } + content, isNull, err := s.includeRawAt(ord, col) + if err != nil { + return err + } + if isNull { + cbAppendNull(k, fixedW, fixed) + return nil + } + cbAppendVal(k, content, fixedW, fixed) + return nil +} + +// cbAppendVal appends a non-NULL value's bytes to a nullable ColumnBuffer: fixed types the +// content verbatim (exactly fixedW bytes), varlena types a [u32 len][content] entry. It +// keeps k.Nulls parallel to k.N once Nulls has been materialized (a prior NULL). +func cbAppendVal(k *vectorindex.ColumnBuffer, content []byte, fixedW int, fixed bool) { + if fixed { + k.Data = append(k.Data, content...) // exactly fixedW bytes + } else { + var lp [4]byte + binary.LittleEndian.PutUint32(lp[:], uint32(len(content))) + k.Data = append(k.Data, lp[:]...) + k.Data = append(k.Data, content...) + } + if k.Nulls != nil { + k.Nulls = append(k.Nulls, false) + } + k.N++ +} + +// appendAnyInclude appends a boxed INCLUDE value (or nil ⇒ SQL NULL) to a nullable +// ColumnBuffer, re-encoding via encodePk — the boxed-source analogue of appendIncludeTo, +// used by the LIMIT path whose top-k already materialized boxed Result.Include values. +// k.Type MUST be the column's type. An encode failure is RETURNED (not silently coerced to +// NULL): the value should be encodable (INCLUDE types are validated at CREATE), so a failure +// means real corruption — surface it, matching the streaming path (appendIncludeTo) which +// stops the stream on the same error rather than projecting a wrong NULL. +func appendAnyInclude(k *vectorindex.ColumnBuffer, v any) error { + fixedW, fixed := fixedPkByteWidth(int32(k.Type)) + if v == nil { + cbAppendNull(k, fixedW, fixed) + return nil + } + content, err := encodePk(int32(k.Type), v) + if err != nil { + return err + } + cbAppendVal(k, content, fixedW, fixed) + return nil +} + +// cbAppendNull appends a SQL-NULL element: a well-formed placeholder in Data (width +// zero-bytes / a [u32 0] entry) so the Data cursor stays aligned, plus a true flag in +// k.Nulls (lazily materialized, backfilling the prior all-non-null elements). +func cbAppendNull(k *vectorindex.ColumnBuffer, fixedW int, fixed bool) { + if k.Nulls == nil { + k.Nulls = make([]bool, k.N, k.N+1) // backfill prior non-nulls as false + } + if fixed { + k.Data = append(k.Data, make([]byte, fixedW)...) + } else { + var lp [4]byte // [u32 0]: zero-length content + k.Data = append(k.Data, lp[:]...) + } + k.Nulls = append(k.Nulls, true) + k.N++ +} + +// Free releases a loaded segment's mmap (and its backing file, if linked). Safe on +// a build-side / in-memory tail segment (nil mmapData → no-op) and idempotent. After +// Free the segment must not be queried; the VectorIndexCache holds the write lock +// when calling Destroy, and single-shot loaders (compact) Free after they finish +// reading. The loaded posting blocks (blockData) are views into mmapData, so +// munmap reclaims them — there is no off-heap buffer to deallocate. +func (s *Segment) Free() { + if s.lease != nil { + lease := s.lease + s.lease = nil + // The view does not own the mmap. Clear its aliases as well so a + // defensive second Free cannot munmap the shared mapping directly. + s.mmapData, s.mmapPath = nil, "" + s.ranking, s.blocks, s.positions = nil, nil, nil + lease.release() + return + } + s.freeOwned() +} + +func (s *Segment) freeOwned() { + if s.mmapData != nil { + _ = munmap(s.mmapData) + s.mmapData = nil + } + if s.mmapPath != "" { + _ = os.Remove(s.mmapPath) + s.mmapPath = "" + } + s.ranking, s.blocks, s.positions = nil, nil, nil + s.ownedBytes = 0 +} + +// freeSegs frees every segment's off-heap buffers (nil-safe on build-side segs). +func freeSegs(segs []*Segment) { + for _, s := range segs { + if s != nil { + s.Free() + } + } +} + +// NewSegment returns an empty segment with the given id and pk type. Postings +// are added by the builder (a later slice); this is the shared zero value used +// by both the build sink and the loader. +func NewSegment(id string, pkType int32) *Segment { + return &Segment{ + Id: id, + PkType: pkType, + terms: make(map[string]*termPostings), + } +} + +// NumDocs returns the document count in this segment. +func (s *Segment) NumDocs() int64 { return s.N } + +// NumTerms returns the number of distinct terms in this segment. +func (s *Segment) NumTerms() int { return len(s.sortedTerms) } + +// Lookup returns the posting list for an exact term, or (nil, false) if the term +// is not in this segment. O(1). +func (s *Segment) Lookup(term string) (*termPostings, bool) { + p, ok := s.terms[term] + return p, ok +} + +// forEachPosting calls fn for every (term, posting-list) in the segment, whether +// build-side (terms map) or loaded-side (FST dict + loaded slices). Used by MERGE +// compaction to reconstruct each doc's terms from the positional postings. +func (s *Segment) forEachPosting(fn func(term string, tp *termPostings)) error { + if s.dict == nil { + for term, tp := range s.terms { + fn(term, tp) + } + return nil + } + // Stream the FST (every term, ascending) instead of materializing the whole + // vocabulary as a []string — MERGE reconstruction over a large index must not add an + // O(vocabulary) string spike on top of the reconstruction buckets. + return s.dict.forEachTerm(func(term string) error { + if tp, ok := s.LookupLoaded(term); ok { + fn(term, tp) + } + return nil + }) +} + +// PrefixRange returns the sorted terms of this segment that start with prefix, +// in ascending order — the enumeration the `word*` boolean operator expands into +// a disjunctive slot. An empty prefix returns all terms. O(log n) to locate the +// range start, then linear in the number of matches. +// +// It relies solely on the sorted key list, so it works uniformly for gojieba +// words and ngram bigrams (both live in the same dictionary-free term dict). +func (s *Segment) PrefixRange(prefix string) []string { + if prefix == "" { + return s.sortedTerms + } + // First index whose term >= prefix. + lo := sort.SearchStrings(s.sortedTerms, prefix) + hi := lo + for hi < len(s.sortedTerms) && hasPrefix(s.sortedTerms[hi], prefix) { + hi++ + } + return s.sortedTerms[lo:hi] +} + +// hasPrefix reports whether s begins with prefix. (Local rather than +// strings.HasPrefix to keep the hot prefix-scan allocation-free and explicit.) +func hasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} + +// setTerms installs the term dict from a built/loaded map and (re)derives the +// sorted key list. Used by the builder and the loader; kept here so the +// terms/sortedTerms consistency invariant lives in one place. +func (s *Segment) setTerms(terms map[string]*termPostings) { + s.terms = terms + s.sortedTerms = make([]string, 0, len(terms)) + for t := range terms { + s.sortedTerms = append(s.sortedTerms, t) + } + sort.Strings(s.sortedTerms) +} diff --git a/pkg/fulltext2/segment_test.go b/pkg/fulltext2/segment_test.go new file mode 100644 index 0000000000000..d6835e86b15c0 --- /dev/null +++ b/pkg/fulltext2/segment_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func segmentWithTerms(terms ...string) *Segment { + s := NewSegment("seg0", 0) + m := make(map[string]*termPostings, len(terms)) + for i, t := range terms { + // distinct df per term so we can assert Lookup returns the right posting + m[t] = &termPostings{docIDs: make([]int64, i+1)} + } + s.setTerms(m) + return s +} + +// TestSetTermsSorted: setTerms derives an ascending key list consistent with the +// map (the invariant PrefixRange's binary search relies on). +func TestSetTermsSorted(t *testing.T) { + s := segmentWithTerms("banana", "apple", "cherry", "apricot") + require.Equal(t, 4, s.NumTerms()) + require.True(t, sort.StringsAreSorted(s.sortedTerms), "sortedTerms must be ascending") + require.Equal(t, []string{"apple", "apricot", "banana", "cherry"}, s.sortedTerms) + // every sorted key resolves in the map + for _, term := range s.sortedTerms { + _, ok := s.Lookup(term) + require.True(t, ok, term) + } +} + +// TestLookup: exact hit returns the term's own posting; miss returns false. +func TestLookup(t *testing.T) { + s := segmentWithTerms("apple", "apricot") + p, ok := s.Lookup("apple") + require.True(t, ok) + require.Equal(t, 1, p.df()) // "apple" was index 0 -> df 1 + p, ok = s.Lookup("apricot") + require.True(t, ok) + require.Equal(t, 2, p.df()) + _, ok = s.Lookup("avocado") + require.False(t, ok) + _, ok = s.Lookup("") + require.False(t, ok) +} + +// TestPrefixRange: the `word*` enumeration — prefix hit, boundary, no-match, and +// empty prefix (all terms). +func TestPrefixRange(t *testing.T) { + s := segmentWithTerms("app", "apple", "apply", "banana", "band", "bat") + + require.Equal(t, []string{"app", "apple", "apply"}, s.PrefixRange("app")) + require.Equal(t, []string{"banana", "band"}, s.PrefixRange("ban")) + require.Equal(t, []string{"bat"}, s.PrefixRange("bat")) + // prefix matches a whole run + require.Equal(t, []string{"banana", "band", "bat"}, s.PrefixRange("b")) + // no match — before, between, and after the key range + require.Empty(t, s.PrefixRange("aa")) + require.Empty(t, s.PrefixRange("az")) + require.Empty(t, s.PrefixRange("z")) + // empty prefix returns everything, in order + require.Equal(t, s.sortedTerms, s.PrefixRange("")) +} + +func TestPrefixRangeEmptySegment(t *testing.T) { + s := NewSegment("empty", 0) + require.Equal(t, 0, s.NumTerms()) + require.Empty(t, s.PrefixRange("x")) + require.Empty(t, s.PrefixRange("")) +} diff --git a/pkg/fulltext2/serialize.go b/pkg/fulltext2/serialize.go new file mode 100644 index 0000000000000..6ffdaf244bf99 --- /dev/null +++ b/pkg/fulltext2/serialize.go @@ -0,0 +1,722 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "archive/tar" + "bytes" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// Tar member names — one archive per segment (same multi-member shape as bm25). +const ( + memberDocmap = "docmap" // pkType + N + ord->pk + ord->docLen + memberTermDict = "termdict" // the vellum FST: term -> ordinal + memberPostings = "postings" // ranking DIRECTORY: per-term df, per-block max/skip meta (resident) + memberBlocks = "blocks" // per-term docID/tf blocks (delta+varint; mmap, block-decoded on demand) + memberPositions = "positions" // per-term compressed positions (phrase-only, lazy) +) + +// Checksum returns the CRC32 (IEEE) of the serialized bytes (matches bm25's +// Checksum so the shared storage/CDC framing verifies both engines uniformly). +func Checksum(b []byte) uint32 { return crc32.ChecksumIEEE(b) } + +// Serialize encodes the (build-side) segment into a tar archive of three +// members: docmap, the FST term dict, and the positional postings. Terms are +// written in sorted order and the FST maps each term to its ordinal (index into +// the postings member), so load resolves term -> ordinal -> posting list. +func (s *Segment) Serialize() ([]byte, error) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + docmap, err := s.encodeDocmap() + if err != nil { + return nil, err + } + if err = writeMember(tw, memberDocmap, docmap); err != nil { + return nil, err + } + + fst, ranking, blocks, positions, err := s.encodeTermsAndPostings() + if err != nil { + return nil, err + } + if err = writeMember(tw, memberTermDict, fst); err != nil { + return nil, err + } + if err = writeMember(tw, memberPostings, ranking); err != nil { + return nil, err + } + if err = writeMember(tw, memberBlocks, blocks); err != nil { + return nil, err + } + if err = writeMember(tw, memberPositions, positions); err != nil { + return nil, err + } + + if err = tw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Deserialize parses a tar archive produced by Serialize into a loaded, queryable +// segment (dict + loaded[] postings, NOT the build-side terms map). id is the +// segment id (carried from storage, not stored in the archive). AvgDocLen is left +// unset — it is a cross-segment aggregate computed when all segments are loaded +// (§4), not a per-segment value. +func Deserialize(id string, r io.Reader) (*Segment, error) { + // In-memory path (tail deltas, tests): read the whole archive to a Go slice and + // point the FST + positions at it (no mmap). Bases use the mmap path in storage. + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + s := &Segment{Id: id} + if err := s.decodeSegment(data); err != nil { + return nil, err + } + return s, nil +} + +// sliceMembers returns the FST / docmap / ranking / positions members as SLICES +// into data (zero-copy) by walking the tar and tracking the reader offset, so the +// same code serves an in-memory blob or an mmap'd file. +func sliceMembers(data []byte) (docmap, fst, ranking, blocks, positions []byte, err error) { + br := bytes.NewReader(data) + tr := tar.NewReader(br) + for { + h, e := tr.Next() + if e == io.EOF { + break + } + if e != nil { + return nil, nil, nil, nil, nil, e + } + off := int64(len(data)) - int64(br.Len()) // tar positions br at the content + if off < 0 || off+h.Size > int64(len(data)) { + return nil, nil, nil, nil, nil, moerr.NewInternalErrorNoCtx("fulltext2: member out of range") + } + seg := data[off : off+h.Size] + switch h.Name { + case memberDocmap: + docmap = seg + case memberTermDict: + fst = seg + case memberPostings: + ranking = seg + case memberBlocks: + blocks = seg + case memberPositions: + positions = seg + } + } + return docmap, fst, ranking, blocks, positions, nil +} + +// decodeSegment builds the loaded segment over data (an in-memory blob or an mmap): +// the ranking DIRECTORY (per-block skip/max meta) expands RESIDENT, while the FST, +// the docID/tf blocks, and the compressed positions stay as views into data (decoded +// on demand). Callers keep data alive (mmapData for a base, GC for a tail). +func (s *Segment) decodeSegment(data []byte) error { + docmap, fst, ranking, blocks, positions, err := sliceMembers(data) + if err != nil { + return err + } + if err := s.decodeDocmap(docmap); err != nil { + return err + } + if err := s.decodePostings(ranking, blocks, positions); err != nil { + return err + } + dict, err := loadTermDict(fst) + if err != nil { + return err + } + s.dict = dict + s.ownedBytes = int64(len(data)) + // s.N is set by decodeDocmap (loaded segments have pks==nil, so len(s.pks) is 0). + return nil +} + +func writeMember(tw *tar.Writer, name string, data []byte) error { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(data)), Typeflag: tar.TypeReg}); err != nil { + return err + } + _, err := tw.Write(data) + return err +} + +// leBuf appends little-endian scalars without the per-call heap allocation +// binary.Write incurs (its `data any` boxes every scalar) — the bytes are +// identical to binary.Write(LittleEndian, ...). Mirrors bm25's leBuf. +type leBuf struct { + b bytes.Buffer + tmp [binary.MaxVarintLen64]byte +} + +func (w *leBuf) u32(v uint32) { binary.LittleEndian.PutUint32(w.tmp[:4], v); w.b.Write(w.tmp[:4]) } +func (w *leBuf) u64(v uint64) { binary.LittleEndian.PutUint64(w.tmp[:8], v); w.b.Write(w.tmp[:8]) } +func (w *leBuf) i32(v int32) { w.u32(uint32(v)) } +func (w *leBuf) i64(v int64) { w.u64(uint64(v)) } + +// uvarint appends v as a LEB128 varint (1–10 bytes). Used for the delta-encoded +// posting streams (docID gaps, position gaps) where values are small. +func (w *leBuf) uvarint(v uint64) { n := binary.PutUvarint(w.tmp[:], v); w.b.Write(w.tmp[:n]) } + +// ---- docmap: pkType + N + ord->pk + ord->docLen [+ INCLUDE section] ---- + +const ( + // docmapIncludeTag marks an optional INCLUDE-values section appended after docLen. + // Its presence (a byte remaining after docLen) is how a decoder tells an INCLUDE index + // from a plain one — fulltext2 is new, so there is no old format to stay compatible with; + // a non-INCLUDE docmap is byte-identical to before. The tag guards against a future + // trailing section being misread as INCLUDE values. + docmapIncludeTag byte = 0x49 // 'I' + includeNotNull byte = 0 // per-value flag + includeNull byte = 1 + // maxIncludeCols bounds the decoded column count so a corrupt header can't allocate an + // enormous offset table. Well above the DDL-time practical limit. + maxIncludeCols = 64 +) + +func (s *Segment) encodeDocmap() ([]byte, error) { + var w leBuf + w.i32(s.PkType) + w.i64(int64(len(s.pks))) + for _, pk := range s.pks { + if err := w.encodePkLen(s.PkType, pk); err != nil { + return nil, err + } + } + for _, dl := range s.docLen { + w.i32(dl) + } + // INCLUDE section (only when the index has INCLUDE columns): tag, nCols, per-col type, + // then a dense stride-addressed FIXED region (integer cols: per-doc [NULL bitmask][values]) + // followed by an offset-addressed VARLENA region (varchar/char cols: per-doc per-col + // [nullFlag][u32 len][content]). Fixed cols carry no per-doc offset (stride-computed), so a + // numeric-only INCLUDE index needs no resident offset table. + if nIncl := len(s.includeTypes); nIncl > 0 { + if len(s.includeVals) != len(s.pks) { + return nil, moerr.NewInternalErrorNoCtxf( + "fulltext2: includeVals len %d != docs %d", len(s.includeVals), len(s.pks)) + } + w.b.WriteByte(docmapIncludeTag) + w.u32(uint32(nIncl)) + for _, t := range s.includeTypes { + w.i32(t) + } + lay := computeIncludeLayout(s.includeTypes) + valRegion := lay.fixedStride - lay.maskBytes + // FIXED region. + for ord := range s.pks { + row := s.includeVals[ord] + mask := make([]byte, lay.maskBytes) + vals := make([]byte, valRegion) + for c, t := range s.includeTypes { + fi := lay.fixedIdx[c] + if fi < 0 { + continue + } + var v any + if c < len(row) { + v = row[c] + } + if v == nil { + mask[fi/8] |= 1 << (uint(fi) & 7) + continue // value bytes stay zero + } + vb, err := encodePk(t, v) + if err != nil { + return nil, err + } + if len(vb) != lay.fixedWidth[c] { + return nil, moerr.NewInternalErrorNoCtxf("fulltext2: include fixed col %d width %d != %d", c, len(vb), lay.fixedWidth[c]) + } + copy(vals[lay.fixedValOff[c]:], vb) + } + w.b.Write(mask) + w.b.Write(vals) + } + // VARLENA region. + if lay.nVarlena > 0 { + for ord := range s.pks { + row := s.includeVals[ord] + for c, t := range s.includeTypes { + if lay.varlenaIdx[c] < 0 { + continue + } + var v any + if c < len(row) { + v = row[c] + } + if v == nil { + w.b.WriteByte(includeNull) + continue + } + w.b.WriteByte(includeNotNull) + if err := w.encodePkLen(t, v); err != nil { + return nil, err + } + } + } + } + } + return w.b.Bytes(), nil +} + +func (s *Segment) decodeDocmap(data []byte) error { + if len(data) < 12 { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap too short") + } + s.PkType = int32(binary.LittleEndian.Uint32(data[0:4])) + n := int64(binary.LittleEndian.Uint64(data[4:12])) + if n < 0 { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap negative N") + } + s.N = n + // Build a pk offset table into the (view) docmap bytes rather than materializing N + // boxed `any` pks — pk(ord) decodes on demand from pkRaw. pkOffsets[i] is the byte + // offset of pk i's u32 length prefix within data; pkRaw retains the view so the pk + // bytes stay mmap-backed (base) / GC-alive (tail). This is the O(docs) resident-heap + // reduction: 4 B/doc (offset) instead of ~24 B/doc (interface + boxed value). + offs := make([]int32, n) + pos := 12 + for i := int64(0); i < n; i++ { + if pos+4 > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated pk length") + } + if pos > math.MaxInt32 { + // pkOffsets is []int32; a >2GB docmap (e.g. huge blob pks) would wrap the offset + // negative and panic in pk(ord). Fail loudly instead. Bounded by max_index_capacity + // docs, so this only trips with very large per-pk payloads. + return moerr.NewInternalErrorNoCtx("fulltext2: docmap pk section exceeds 2GB") + } + offs[i] = int32(pos) + l := int(binary.LittleEndian.Uint32(data[pos:])) + // For a fixed-width pk type the stored length MUST equal the type width; a + // corrupt smaller length would pass the pos bound below but then make pk(ord) + // read past its slice (LittleEndian.Uint64 on <8 bytes panics). Validate here at + // load so corruption fails cleanly instead of crashing a query goroutine. + if w, fixed := fixedPkByteWidth(s.PkType); fixed && l != w { + return moerr.NewInternalErrorNoCtxf("fulltext2: docmap pk %d width %d != expected %d", i, l, w) + } + pos += 4 + l + if pos < 0 || pos > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated pk data") + } + } + // Validate pkType ONCE (decodePk rejects unsupported types) so pk(ord) can trust it; + // per-pk width was already checked in the loop above for fixed-width types. + if n > 0 { + off := int(offs[0]) + l := int(binary.LittleEndian.Uint32(data[off:])) + if _, err := decodePk(s.PkType, data[off+4:off+4+l]); err != nil { + return err + } + } + s.pks = nil + s.pkOffsets = offs + s.pkRaw = data // pkOffsets are absolute offsets into data + + // docLen: N contiguous little-endian int32 after the pk section. + if pos+int(n)*4 > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated docLen") + } + dl := make([]int32, n) + for i := int64(0); i < n; i++ { + dl[i] = int32(binary.LittleEndian.Uint32(data[pos+int(i)*4:])) + } + s.docLen = dl + pos += int(n) * 4 + + // Optional INCLUDE section. Absent ⇒ nothing after docLen (plain index). + if pos < len(data) { + if err := s.decodeIncludeSection(data, pos); err != nil { + return err + } + } + return nil +} + +// decodeIncludeSection parses the trailing INCLUDE-values section starting at pos: a tag, the +// column count + types, then the dense FIXED region (skipped by stride — no offsets) and the +// VARLENA region (for which it builds an (ord*nVarlena+varlenaIdx)->offset table into `data`, +// a view; mmap-backed for a base). Fixed values are read directly by stride at query time; no +// value is materialized at load, and a numeric-only INCLUDE index builds NO offset table. +func (s *Segment) decodeIncludeSection(data []byte, pos int) error { + if data[pos] != docmapIncludeTag { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap unknown trailing section") + } + pos++ + if pos+4 > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include header") + } + nIncl := int(binary.LittleEndian.Uint32(data[pos:])) + pos += 4 + if nIncl <= 0 || nIncl > maxIncludeCols { + return moerr.NewInternalErrorNoCtxf("fulltext2: docmap bad include col count %d", nIncl) + } + itypes := make([]int32, nIncl) + for c := 0; c < nIncl; c++ { + if pos+4 > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include types") + } + itypes[c] = int32(binary.LittleEndian.Uint32(data[pos:])) + pos += 4 + } + lay := computeIncludeLayout(itypes) + + // FIXED region: N rows × fixedStride bytes, read directly by stride at query time. + fixedStart := pos + fixedSize := int(s.N) * lay.fixedStride + if fixedSize < 0 || fixedStart+fixedSize > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include fixed region") + } + pos += fixedSize + + // VARLENA region: build the offset table (only varlena cols cost resident offsets). + var varOffs []int32 + if lay.nVarlena > 0 { + varOffs = make([]int32, int(s.N)*lay.nVarlena) + for ord := int64(0); ord < s.N; ord++ { + for vi := 0; vi < lay.nVarlena; vi++ { + if pos >= len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include varlena values") + } + if pos > math.MaxInt32 { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap include section exceeds 2GB") + } + varOffs[int(ord)*lay.nVarlena+vi] = int32(pos) + flag := data[pos] + pos++ + if flag == includeNull { + continue + } + if flag != includeNotNull { + return moerr.NewInternalErrorNoCtxf("fulltext2: docmap bad include null-flag %d", flag) + } + if pos+4 > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include value length") + } + l := int(binary.LittleEndian.Uint32(data[pos:])) + pos += 4 + l + if pos < 0 || pos > len(data) { + return moerr.NewInternalErrorNoCtx("fulltext2: docmap truncated include value data") + } + } + } + } + s.includeTypes = itypes + s.includeRaw = data + s.includeLay = lay + s.includeFixedStart = fixedStart + s.includeVarOffsets = varOffs + return nil +} + +// ---- termdict (FST) + postings ---- + +// encodeTermsAndPostings builds the FST (term -> ordinal) and THREE posting sections +// in one pass over the sorted terms (ordinal i in the FST is posting list i): +// +// - RANKING directory (kept RESIDENT at load): version, nterms, then per term +// df(varint), nblk(varint), the term's absolute base offsets into the blocks/positions +// sections + term-level maxTf/minDocLen, and per BlockSize-doc block +// {lastDocGap(varint, from the previous block's last ord), maxTf(1 byte), +// minDocLn(varint), blkByteLen(varint), posByteLen(varint)}. This is the Block-Max +// skip metadata + a directory into the blocks section; it is all WAND needs to prune, +// and it is O(df/BlockSize) — ~1/128 the postings. Each entry is SELF-CONTAINED and the +// FST value is its BYTE OFFSET, so LookupLoaded decodes one term's directory on demand +// (resident directory heap is O(the current query), not O(vocabulary)). +// - BLOCKS (mmap, block-decoded on demand): per term, per block, the docID GAPS +// (varint, from the previous block's last ord → tiny deltas) then the block's raw +// tf bytes. The WAND cursor decodes only the blocks its walk lands on; docIDs +// never fully expand into RAM at load (the OOM-scaling fix). +// - POSITIONS (mmap, phrase-only): per term (same order) per doc pc(varint) + +// position GAPS(varint). Never touched by ranking. +// +// Delta+varint shrinks docIDs/positions ~4×; blocking docIDs (vs one flat per-term +// stream) is what lets the loader keep them on the mmap and random-access one block. +// +// postingsFormatV1 is the single, current on-disk format for BOTH base index segments and +// CDC tail segments (they share this serialization). fulltext2 is experimental and +// fresh-deploy-only, so there is no migration and the version is NOT advanced as the engine +// evolves — the version byte is only a stale-blob guard. New per-doc data (e.g. INCLUDE +// column values) is added in the docmap member (presence-tagged), not by bumping this. +const postingsFormatV1 byte = 1 + +func (s *Segment) encodeTermsAndPostings() (fst, ranking, blocks, positions []byte, err error) { + n := len(s.sortedTerms) + values := make([]uint64, n) + var w, bw, pw leBuf // w=ranking directory, bw=blocks, pw=positions + w.b.WriteByte(postingsFormatV1) + w.i64(int64(n)) + for i, term := range s.sortedTerms { + tp := s.terms[term] + if tp.blockLastDoc == nil { // ensure the block-max directory (build normally sets it) + deriveTermStats(tp, s.docLen) + } + df := len(tp.docIDs) + nblk := (df + BlockSize - 1) / BlockSize + // Self-contained entry: the FST maps term -> this byte offset, and the entry + // carries its own blocks/positions base offsets + term-level max/min so it can be + // decoded in isolation (no walk of prior terms). + values[i] = uint64(w.b.Len()) + w.uvarint(uint64(df)) + w.uvarint(uint64(nblk)) + w.uvarint(uint64(bw.b.Len())) // blockDataBase: absolute offset into the blocks section + w.uvarint(uint64(pw.b.Len())) // posRawBase: absolute offset into the positions section + w.b.WriteByte(tp.maxTf) // term-level max tf (was recomputed at load in V5) + w.uvarint(uint64(tp.minDocLen)) + var prevLast int64 + for b := 0; b < nblk; b++ { + lo := b * BlockSize + hi := lo + BlockSize + if hi > df { + hi = df + } + // block bytes: docID gaps (from prevLast) then raw tfs. + blkStart := bw.b.Len() + prev := prevLast + for j := lo; j < hi; j++ { + bw.uvarint(uint64(tp.docIDs[j] - prev)) + prev = tp.docIDs[j] + } + bw.b.Write(tp.tfs[lo:hi]) + // this block's positions (parallel to its docs) — grouped per block so the + // directory can record each block's byte length, making positions block- + // SEEKABLE for the phrase cursor (posRaw bytes are unchanged vs V4: still + // per-doc pc + ascending gaps in doc order, so materializePositions is + // unaffected). V4 wrote them once per term after all blocks. + posStart := pw.b.Len() + if tp.positions != nil { // position-free segments omit the positional payload + for j := lo; j < hi; j++ { + pos := tp.positions[j] + pw.uvarint(uint64(len(pos))) + var pp int32 + for _, p := range pos { // ascending positions → non-negative gaps + pw.uvarint(uint64(p - pp)) + pp = p + } + } + } + // directory entry. + w.uvarint(uint64(tp.blockLastDoc[b] - prevLast)) // lastDocGap + w.b.WriteByte(tp.blockMaxTf[b]) + w.uvarint(uint64(tp.blockMinDocLn[b])) + w.uvarint(uint64(bw.b.Len() - blkStart)) // this block's docID/tf byte length + w.uvarint(uint64(pw.b.Len() - posStart)) // this block's positions byte length (V5) + prevLast = tp.blockLastDoc[b] + } + } + if fst, err = buildTermDictFST(s.sortedTerms, values); err != nil { + return nil, nil, nil, nil, err + } + return fst, w.b.Bytes(), bw.b.Bytes(), pw.b.Bytes(), nil +} + +// decodePostings BINDS the loaded segment to its (mmap'd) ranking/blocks/positions +// sections WITHOUT expanding any term. V6 entries are self-contained and reachable by +// byte offset (the FST value), so terms decode lazily in LookupLoaded — the resident +// directory heap is O(the current query), not O(vocabulary). +func (s *Segment) decodePostings(ranking, blocks, positions []byte) error { + if len(ranking) == 0 { + s.ranking, s.blocks, s.positions = nil, nil, nil + return nil + } + if len(ranking) < 1+8 { + return moerr.NewInternalErrorNoCtx("fulltext2: ranking blob too short") + } + if ranking[0] != postingsFormatV1 { + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("fulltext2: unsupported postings format %d", ranking[0])) + } + s.ranking, s.blocks, s.positions = ranking, blocks, positions + return nil +} + +// decodeTermEntry lazily decodes ONE term's self-contained directory entry at byte +// offset `off` in s.ranking (the FST value) into a transient termPostings whose +// blockData/posRaw are views into s.blocks/s.positions. Callers hold the result for the +// query's lifetime (WAND/phrase cursors, evalClause), so the entry decodes at most a few +// times per query and never persists — resident directory heap is O(query), not O(vocab). +// Returns (nil,false) on a corrupt/out-of-bounds entry (defense-in-depth on already-CRC'd +// data). +func (s *Segment) decodeTermEntry(off int) (*termPostings, bool) { + r := s.ranking + if off < 0 || off >= len(r) { + return nil, false + } + p := off + uv := func() (uint64, bool) { + v, n := binary.Uvarint(r[p:]) + if n <= 0 { + return 0, false + } + p += n + return v, true + } + dfu, ok := uv() + if !ok { + return nil, false + } + nblku, ok := uv() + if !ok { + return nil, false + } + baseB, ok := uv() // blockDataBase: absolute offset into s.blocks + if !ok { + return nil, false + } + baseP, ok := uv() // posRawBase: absolute offset into s.positions + if !ok { + return nil, false + } + if p >= len(r) { + return nil, false + } + termMaxTf := r[p] + p++ + minDLu, ok := uv() + if !ok { + return nil, false + } + // A df/nblk larger than the blocks section is impossible (each posting is >= 1 byte). + if dfu > uint64(len(s.blocks)) || nblku > uint64(len(s.blocks)) { + return nil, false + } + df, nblk := int(dfu), int(nblku) + tp := &termPostings{ndoc: df, maxTf: termMaxTf, minDocLen: int32(minDLu)} + if nblk == 0 { + return tp, true + } + tp.blockLastDoc = make([]int64, nblk) + tp.blockMaxTf = make([]uint8, nblk) + tp.blockMinDocLn = make([]int32, nblk) + tp.blockOff = make([]int64, nblk+1) // byte offsets RELATIVE to this term's blockData + tp.blockPosOff = make([]int64, nblk+1) // byte offsets RELATIVE to this term's posRaw + var prevLast, cb, cpos int64 + for b := 0; b < nblk; b++ { + gap, ok := uv() + if !ok { + return nil, false + } + prevLast += int64(gap) + tp.blockLastDoc[b] = prevLast + if p >= len(r) { + return nil, false + } + tp.blockMaxTf[b] = r[p] + p++ + mdl, ok := uv() + if !ok { + return nil, false + } + tp.blockMinDocLn[b] = int32(mdl) + blkLen, ok := uv() + if !ok || blkLen > uint64(len(s.blocks)) { + return nil, false + } + tp.blockOff[b] = cb + cb += int64(blkLen) + posLen, ok := uv() + if !ok || posLen > uint64(len(s.positions)) { + return nil, false + } + tp.blockPosOff[b] = cpos + cpos += int64(posLen) + } + tp.blockOff[nblk] = cb + tp.blockPosOff[nblk] = cpos + // blockData/posRaw are views into the mmap sections at this term's stored bases. + if int64(baseB)+cb > int64(len(s.blocks)) || int64(baseP)+cpos > int64(len(s.positions)) { + return nil, false + } + tp.blockData = s.blocks[baseB : int64(baseB)+cb] + tp.posRaw = s.positions[baseP : int64(baseP)+cpos] + return tp, true +} + +// deriveTermStats fills tp's raw, scorer-agnostic score-UB fields from its +// postings + docLen: the term-level maxTf/minDocLen AND the per-block Block-Max +// skip metadata (blockLastDoc/blockMaxTf/blockMinDocLn), one entry per +// ceil(df/BlockSize). Both are idf/avgdl-free so one segment serves BM25 and +// TF-IDF; the active scorer derives its impact bound at query time. Computed in a +// single pass (the term-level max/min is the max/min over the block stats), and +// called from BOTH the builder and Deserialize — the direct analogue of bm25's +// WandModel.finalizeScoring. Assumes df >= 1 (a term with no postings is never +// stored). +func deriveTermStats(tp *termPostings, docLen []int32) { + df := len(tp.docIDs) + tp.ndoc = df + if df == 0 { + return + } + nblk := (df + BlockSize - 1) / BlockSize + tp.blockLastDoc = make([]int64, nblk) + tp.blockMaxTf = make([]uint8, nblk) + tp.blockMinDocLn = make([]int32, nblk) + var termMaxTf uint8 + termMinDL := int32(math.MaxInt32) + for b := 0; b < nblk; b++ { + lo := b * BlockSize + hi := lo + BlockSize + if hi > df { + hi = df + } + var maxTf uint8 + minDL := int32(math.MaxInt32) + for i := lo; i < hi; i++ { + if tp.tfs[i] > maxTf { + maxTf = tp.tfs[i] + } + if dl := docLen[tp.docIDs[i]]; dl < minDL { + minDL = dl + } + } + tp.blockLastDoc[b] = tp.docIDs[hi-1] // ascending → last is the block's max ord + tp.blockMaxTf[b] = maxTf + tp.blockMinDocLn[b] = minDL + if maxTf > termMaxTf { + termMaxTf = maxTf + } + if minDL < termMinDL { + termMinDL = minDL + } + } + tp.maxTf = termMaxTf + tp.minDocLen = termMinDL +} + +// LookupLoaded resolves a term to its posting list on a LOADED segment (via the +// FST ordinal). Returns (nil, false) if the segment is not loaded or the term is +// absent. Query-side counterpart of the build-side Lookup. +func (s *Segment) LookupLoaded(term string) (*termPostings, bool) { + if s.dict == nil { + return nil, false + } + off, ok, err := s.dict.get(term) // V6: the FST value is the entry's byte offset in ranking + if err != nil || !ok { + return nil, false + } + return s.decodeTermEntry(int(off)) +} diff --git a/pkg/fulltext2/serialize_test.go b/pkg/fulltext2/serialize_test.go new file mode 100644 index 0000000000000..be265eeb15153 --- /dev/null +++ b/pkg/fulltext2/serialize_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// buildSegment assembles a build-side segment directly (as the build sink will). +func buildSegment(pkType int32, pks []any, docLen []int32, terms map[string]*termPostings) *Segment { + s := NewSegment("seg0", pkType) + s.pks = pks + s.docLen = docLen + s.N = int64(len(pks)) + s.setTerms(terms) + return s +} + +func roundtrip(t *testing.T, s *Segment) *Segment { + t.Helper() + data, err := s.Serialize() + require.NoError(t, err) + // deterministic output (FST + leBuf + tar all deterministic) + data2, err := s.Serialize() + require.NoError(t, err) + require.Equal(t, Checksum(data), Checksum(data2), "Serialize must be deterministic") + + loaded, err := Deserialize(s.Id, bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + return loaded +} + +// assertPostingsEqual compares a loaded term's posting list to the original. The +// loaded side keeps docIDs/tfs block-compressed on the mmap and positions +// compressed, so compare via materialize* rather than the (nil) build-side fields. +func assertPostingsEqual(t *testing.T, want, got *termPostings) { + t.Helper() + require.Equal(t, want.docIDs, got.materializeDocIDs()) + require.Equal(t, want.tfs, got.materializeTfs()) + gotPos := got.materializePositions() + require.Equal(t, len(want.positions), len(gotPos)) + for i := range want.positions { + require.Equalf(t, want.positions[i], gotPos[i], "positions[%d]", i) + } +} + +// allPks materializes a loaded segment's pks via the on-demand accessor (loaded +// segments hold no resident []any — pk(ord) decodes from the mmap-backed docmap). +func allPks(s *Segment) []any { + out := make([]any, s.numDocs()) + for i := range out { + out[i] = s.pk(int64(i)) + } + return out +} + +func TestSegmentRoundtripInt64PK(t *testing.T) { + terms := map[string]*termPostings{ + "apple": { + docIDs: []int64{0, 2}, + tfs: []uint8{3, 1}, + positions: [][]int32{{1, 5, 9}, {4}}, + }, + "banana": { + docIDs: []int64{1}, + tfs: []uint8{2}, + positions: [][]int32{{0, 7}}, + }, + "中文": { + docIDs: []int64{0, 1, 2}, + tfs: []uint8{1, 1, 1}, + positions: [][]int32{{2}, {3}, {6}}, + }, + } + orig := buildSegment(int32(types.T_int64), []any{int64(100), int64(200), int64(300)}, + []int32{5, 7, 3}, terms) + + loaded := roundtrip(t, orig) + + require.Equal(t, int32(types.T_int64), loaded.PkType) + require.Equal(t, int64(3), loaded.N) + require.Equal(t, []any{int64(100), int64(200), int64(300)}, allPks(loaded)) + require.Equal(t, []int32{5, 7, 3}, loaded.docLen) + + for term, want := range terms { + got, ok := loaded.LookupLoaded(term) + require.True(t, ok, term) + assertPostingsEqual(t, want, got) + } + // a missing term + _, ok := loaded.LookupLoaded("cherry") + require.False(t, ok) + // prefix still works over the loaded FST + it, ok, err := loaded.dict.prefixIter("app") + require.NoError(t, err) + require.True(t, ok) + term, _ := it.Current() + require.Equal(t, "apple", string(term)) + _ = it.Close() +} + +func TestSegmentRoundtripVarcharPK(t *testing.T) { + terms := map[string]*termPostings{ + "x": {docIDs: []int64{0, 1}, tfs: []uint8{1, 4}, positions: [][]int32{{0}, {1, 2, 3, 8}}}, + } + orig := buildSegment(int32(types.T_varchar), + []any{[]byte("alpha"), []byte("beta")}, []int32{2, 9}, terms) + + loaded := roundtrip(t, orig) + require.Equal(t, int32(types.T_varchar), loaded.PkType) + require.Equal(t, []any{[]byte("alpha"), []byte("beta")}, allPks(loaded)) + require.Equal(t, []int32{2, 9}, loaded.docLen) + got, ok := loaded.LookupLoaded("x") + require.True(t, ok) + assertPostingsEqual(t, terms["x"], got) +} + +func TestSegmentRoundtripDatetimePK(t *testing.T) { + terms := map[string]*termPostings{ + "t": {docIDs: []int64{0}, tfs: []uint8{1}, positions: [][]int32{{0}}}, + } + orig := buildSegment(int32(types.T_datetime), + []any{types.Datetime(1234567890)}, []int32{1}, terms) + + loaded := roundtrip(t, orig) + require.Equal(t, []any{types.Datetime(1234567890)}, allPks(loaded)) +} + +// Narrow integer PKs (TINYINT/SMALLINT, signed + unsigned) must round-trip: the +// build path (vector.GetAny) delivers them natively and no DDL guard rejects them, +// so the pk codec has to handle T_int8/T_int16/T_uint8/T_uint16 or CREATE FULLTEXT2 +// crashes with "unsupported pk type". Regression cover for the codec gap. +func TestSegmentRoundtripNarrowIntPK(t *testing.T) { + terms := map[string]*termPostings{ + "n": {docIDs: []int64{0, 1}, tfs: []uint8{1, 1}, positions: [][]int32{{0}, {0}}}, + } + cases := []struct { + typ types.T + pks []any + }{ + {types.T_int8, []any{int8(-128), int8(127)}}, + {types.T_uint8, []any{uint8(0), uint8(255)}}, + {types.T_int16, []any{int16(-32768), int16(32767)}}, + {types.T_uint16, []any{uint16(0), uint16(65535)}}, + } + for _, c := range cases { + t.Run(c.typ.String(), func(t *testing.T) { + orig := buildSegment(int32(c.typ), c.pks, []int32{1, 1}, terms) + loaded := roundtrip(t, orig) + require.Equal(t, int32(c.typ), loaded.PkType) + require.Equal(t, c.pks, allPks(loaded)) + got, ok := loaded.LookupLoaded("n") + require.True(t, ok) + assertPostingsEqual(t, terms["n"], got) + }) + } +} + +func TestSegmentRoundtripEmpty(t *testing.T) { + orig := buildSegment(int32(types.T_int64), []any{}, []int32{}, map[string]*termPostings{}) + loaded := roundtrip(t, orig) + require.Equal(t, int64(0), loaded.N) + require.Equal(t, 0, loaded.dict.len()) + _, ok := loaded.LookupLoaded("anything") + require.False(t, ok) +} diff --git a/pkg/fulltext2/sink.go b/pkg/fulltext2/sink.go new file mode 100644 index 0000000000000..0bc6826e16f40 --- /dev/null +++ b/pkg/fulltext2/sink.go @@ -0,0 +1,106 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" +) + +// RowEvent is one source-table change fed to the build sink: an insert (Pk + +// Text) or a delete (Pk, Delete=true). An UPDATE is delivered as a delete of the +// old row followed by an insert of the new one — both for the same Pk. +type RowEvent struct { + Delete bool + Pk any + Text []byte +} + +// Sink accumulates a batch of row events and produces one segment (the inserted +// rows) plus a delete set. It is the engine-side of the maintenance path: the +// initial full build feeds every source row through one Sink to make a tag=0 +// base; each subsequent CDC batch feeds its changes through a Sink to make a +// tag=1 tail segment + the batch's deletes. The Index then merges base + tails + +// the accumulated deletes, and liveness (highest Recency) resolves updates. +// +// A pk that is BOTH deleted and (re)inserted in the same batch is an UPDATE — the +// insert at this batch's recency supersedes older copies via liveness, so no +// delete tombstone is emitted for it. Only a pk that is deleted and NOT +// reinserted becomes a tombstone. +type Sink struct { + pkType int32 + tok tokenizer.Tokenizer + inserts []Doc + insertKeys map[any]bool + deleteKeys map[any]bool +} + +// NewSink creates a sink for a batch. pkType is the source pk's types.T; tok is +// the index's tokenizer (used for both this build and later query tokenization). +func NewSink(pkType int32, tok tokenizer.Tokenizer) *Sink { + return &Sink{ + pkType: pkType, + tok: tok, + insertKeys: make(map[any]bool), + deleteKeys: make(map[any]bool), + } +} + +// Add feeds one row event into the batch. +func (s *Sink) Add(ev RowEvent) { + key := normalizeKey(ev.Pk) + if ev.Delete { + s.deleteKeys[key] = true + return + } + s.inserts = append(s.inserts, Doc{Pk: ev.Pk, Text: ev.Text}) + s.insertKeys[key] = true +} + +// Build finalizes the batch into a segment (the inserted rows, at the given +// recency) and the tombstone set (pk key → recency) for pks deleted but not +// reinserted in this batch. +func (s *Sink) Build(id string, recency int64) (*Segment, map[any]int64, error) { + seg, err := BuildSegmentFromDocs(id, s.pkType, s.inserts, s.tok) + if err != nil { + return nil, nil, err + } + seg.Recency = recency + + var deletes map[any]int64 + for key := range s.deleteKeys { + if s.insertKeys[key] { + continue // reinserted in the same batch → an UPDATE, not a tombstone + } + if deletes == nil { + deletes = make(map[any]int64) + } + deletes[key] = recency + } + return seg, deletes, nil +} + +// MergeDeletes folds several per-batch delete maps into one (latest recency +// wins), for feeding NewIndex. +func MergeDeletes(maps ...map[any]int64) map[any]int64 { + out := make(map[any]int64) + for _, m := range maps { + for key, rec := range m { + if cur, ok := out[key]; !ok || rec > cur { + out[key] = rec + } + } + } + return out +} diff --git a/pkg/fulltext2/sink_test.go b/pkg/fulltext2/sink_test.go new file mode 100644 index 0000000000000..34f547af98e45 --- /dev/null +++ b/pkg/fulltext2/sink_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// TestSinkInsertUpdateDelete: a base build then a CDC batch that updates pk 2, +// deletes pk 3, and inserts pk 4. The assembled Index reflects the converged +// state: old text of an updated row gone, its new text visible, deleted row +// absent, new row present. +func TestSinkInsertUpdateDelete(t *testing.T) { + tok := tokenizer.NewSimpleTokenizer() + + // Base build (recency 0): full source scan. + base := NewSink(int32(types.T_int64), tok) + base.Add(RowEvent{Pk: int64(1), Text: []byte("alpha")}) + base.Add(RowEvent{Pk: int64(2), Text: []byte("beta")}) + base.Add(RowEvent{Pk: int64(3), Text: []byte("gamma")}) + baseSeg, baseDel, err := base.Build("base", 0) + require.NoError(t, err) + require.Empty(t, baseDel) + + // CDC batch (recency 1): UPDATE pk2 (delete+insert), DELETE pk3, INSERT pk4. + tail := NewSink(int32(types.T_int64), tok) + tail.Add(RowEvent{Delete: true, Pk: int64(2)}) + tail.Add(RowEvent{Pk: int64(2), Text: []byte("omega")}) + tail.Add(RowEvent{Delete: true, Pk: int64(3)}) + tail.Add(RowEvent{Pk: int64(4), Text: []byte("delta")}) + tailSeg, tailDel, err := tail.Build("tail", 1) + require.NoError(t, err) + // pk2 was reinserted (UPDATE) → no tombstone; pk3 is a pure delete. + require.Equal(t, map[any]int64{normalizeKey(int64(3)): 1}, tailDel) + + idx := NewIndex([]*Segment{baseSeg, tailSeg}, MergeDeletes(baseDel, tailDel)) + + require.Equal(t, int64(3), idx.NumDocs()) // pk 1, 2(new), 4 + require.Equal(t, []any{int64(1)}, iquery(t, idx, "alpha")) + require.Empty(t, iquery(t, idx, "beta")) // pk2 old text superseded + require.Equal(t, []any{int64(2)}, iquery(t, idx, "omega")) // pk2 new text + require.Empty(t, iquery(t, idx, "gamma")) // pk3 deleted + require.Equal(t, []any{int64(4)}, iquery(t, idx, "delta")) +} + +// TestSinkThroughSerialize: sink → serialize → deserialize → Index still +// converges (the maintenance path persists segments). +func TestSinkThroughSerialize(t *testing.T) { + tok := tokenizer.NewSimpleTokenizer() + sink := NewSink(int32(types.T_int64), tok) + sink.Add(RowEvent{Pk: int64(10), Text: []byte("quick brown fox")}) + sink.Add(RowEvent{Pk: int64(11), Text: []byte("lazy dog")}) + seg, _, err := sink.Build("base", 0) + require.NoError(t, err) + + data, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("base", bytes.NewReader(data)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + idx := NewIndex([]*Segment{loaded}, nil) + require.Equal(t, []any{int64(10)}, iquery(t, idx, "quick brown fox")) + require.Equal(t, []any{int64(11)}, iquery(t, idx, "lazy dog")) +} diff --git a/pkg/fulltext2/storage.go b/pkg/fulltext2/storage.go new file mode 100644 index 0000000000000..0f8d6b4109f60 --- /dev/null +++ b/pkg/fulltext2/storage.go @@ -0,0 +1,1066 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/system" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// TableConfig locates a fulltext2 index's persistent segment store + metadata +// table; it is the JSON const arg passed to the fulltext2_create / fulltext2_search +// TVFs. Mirrors bm25.wand.TableConfig. +type TableConfig struct { + DbName string `json:"db"` + SrcTable string `json:"src"` + IndexTable string `json:"index"` // chunk store (FullText2Index_TblType_Storage) + MetadataTable string `json:"metadata"` // metadata (FullText2Index_TblType_Metadata) + PKey string `json:"pkey"` + Parser string `json:"parser,omitempty"` + // Capacity is max_index_capacity: the create build splits the tag=0 base into + // sub-indexes of at most Capacity docs each (0 => DefaultBuildCapacity). + Capacity int64 `json:"capacity,omitempty"` + // PostingCapacity is max_postings_capacity: a segment also seals once it holds + // this many postings (term occurrences), whichever comes first with Capacity. + // Bounds per-segment build memory regardless of doc size (0 => DefaultPostingCapacity). + PostingCapacity int64 `json:"posting_capacity,omitempty"` + // PositionFree builds segments without the positional payload (bag-of-words + // retrieval only; ~half the footprint, FST kept). Sourced from the persisted + // position_free algo_param so every build path (create, CDC tail, compact) agrees. + PositionFree bool `json:"position_free,omitempty"` + FromSource bool `json:"from_source,omitempty"` + // IncludeTypes is the INCLUDE columns' types.T in column order. The build SQL passes the + // INCLUDE source columns as the trailing fulltext2_create args (after the text columns), + // so the TVF knows how many trailing argVecs are INCLUDE values (vs text) and their types + // to store the actual per-doc value in the docmap. nil ⇒ no INCLUDE columns. + IncludeTypes []int32 `json:"include_types,omitempty"` + // IncludeColumns is the INCLUDE columns' NAMES in column order — carried in the SEARCH + // cfg so Fulltext2Search.Search can map a covering query's RequestedIncludeColumns (by + // name) to each result's positional Include values. nil ⇒ no INCLUDE columns. + IncludeColumns []string `json:"include_columns,omitempty"` +} + +// runSql / runStreamingSql indirect the sqlexec executor entry points so unit tests can +// mock the DB round-trips (metadata reads, base/tail chunk loads, budget gates, +// compaction). Production points them at the real executor; tests swap them for fakes +// that return synthetic result batches, optionally dispatching by SQL text. +var ( + runSql = sqlexec.RunSql + runStreamingSql = sqlexec.RunStreamingSql +) + +// SubIndexId is the index_id for the i-th tag=0 base sub-index of a build +// identified by uid (which must carry a per-build-unique component so concurrent +// builds never collide). Load enumerates ids from the metadata table. +func SubIndexId(uid string, i int) string { return fmt.Sprintf("%s:%d", uid, i) } + +// maxInsertTuples bounds the (index_id, chunk_id, data, tag) rows per INSERT (base-segment and +// batched CDC-tail persist). Its primary job is cutting RunSql round-trips — many tiny frames become +// a few multi-row INSERTs. It is kept MODERATE (500, well below hnsw's 2000 in +// pkg/vectorindex/hnsw/model.go) because each row's load_file data flows into the storage engine's +// ASYNC object writer (file/S3), which buffers-and-flushes and thus accumulates memory as rows are +// pushed; fulltext2 CDC persist is a BACKGROUND op, so a smaller batch hands that writer a bounded +// slice (<= maxInsertTuples*MaxChunkSize ≈ 32 MiB) per statement instead of a large burst. NOTE: this is only +// a secondary throttle — the dominant memory bound is the TOTAL data pushed per iteration (the whole +// tail delta), which the per-iteration work bound (not batch size) is what actually caps. +const maxInsertTuples = 500 + +// ToInsertSqls serializes the segment, spills it to a temp file under the LOCAL +// fileservice's __fulltext2 dir (load_file reads it back by path), and emits the +// SQL to persist it: one metadata row + the bytes split into <= MaxChunkSize +// (index_id, chunk_id, data, tag) rows read via load_file. tag=0 is a compacted +// base (sync CREATE/REINDEX build); tag=1 is a CDC delta. sqlproc resolves the LOCAL +// SSD spill dir (falls back to /tmp when none is attached). The returned cleanup MUST +// run after the SQLs execute (they read the temp file at execution). +func (s *Segment) ToInsertSqls(sqlproc *sqlexec.SqlProcess, cfg TableConfig, ts int64, tag int) (sqls []string, cleanup func(), err error) { + buf, err := s.Serialize() + if err != nil { + return nil, nil, err + } + checksum := vectorindex.CheckSumFromBuffer(buf) + filesize := int64(len(buf)) + + fp, err := createLocalSpillFile(sqlproc, "ft2build") + if err != nil { + return nil, nil, err + } + path := fp.Name() + cleanup = func() { fp.Close(); os.Remove(path) } + if _, err = fp.Write(buf); err != nil { + cleanup() + return nil, nil, err + } + if err = fp.Sync(); err != nil { + cleanup() + return nil, nil, err + } + + metaTbl := sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable) + sqls = append(sqls, fmt.Sprintf("INSERT INTO %s (%s, %s, %s, %s, %s, %s) VALUES (%s, %d, %s, %d, %d, %d)", + metaTbl, + catalog.FullText2Index_TblCol_Metadata_Index_Id, catalog.FullText2Index_TblCol_Metadata_Timestamp, + catalog.FullText2Index_TblCol_Metadata_Checksum, catalog.FullText2Index_TblCol_Metadata_Filesize, + catalog.FullText2Index_TblCol_Metadata_Recency, catalog.FullText2Index_TblCol_Metadata_Nrow, + sqlquote.String(s.Id), ts, sqlquote.String(checksum), filesize, s.Recency, s.N)) + sqls = append(sqls, fileChunkInsertSqls(cfg, s.Id, 0, path, 0, int(filesize), tag)...) + return sqls, cleanup, nil +} + +// fileChunkRange is one contiguous [offset, offset+length) byte range of a spilled file to persist +// as storage rows: offset 0 for a whole-file base segment, or a frame's offset within a PACKED CDC +// spool file (many frames share one file). +type fileChunkRange struct { + path string + offset int64 + length int +} + +// framesInsertSqls emits the load_file storage-row tuples for a SEQUENCE of file ranges, each range +// split into <= MaxChunkSize rows, batched at maxInsertTuples rows per INSERT statement +// ACROSS range boundaries. chunk_ids are assigned contiguously from startChunkId in range order; +// the next free chunk_id is returned. Batching across ranges is what keeps a burst of tiny CDC tail +// frames to O(totalChunks/maxInsertTuples) statements instead of one INSERT per frame. Mirrors +// bm25.wand.FileChunkInsertSqls (single-range) generalized to many ranges. +func framesInsertSqls(cfg TableConfig, id string, startChunkId int64, ranges []fileChunkRange, tag int) (sqls []string, nextChunkId int64) { + prefix := fmt.Sprintf("INSERT INTO %s (%s, %s, %s, %s) VALUES ", + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, catalog.FullText2Index_TblCol_Storage_Chunk_Id, + catalog.FullText2Index_TblCol_Storage_Data, catalog.FullText2Index_TblCol_Storage_Tag) + var vals []string + chunkID := startChunkId + for _, r := range ranges { + for off := 0; off < r.length; off += vectorindex.MaxChunkSize { + sz := vectorindex.MaxChunkSize + if off+sz > r.length { + sz = r.length - off + } + url := fmt.Sprintf("file://%s?offset=%d&size=%d", r.path, r.offset+int64(off), sz) + vals = append(vals, fmt.Sprintf("(%s, %d, load_file(cast(%s as datalink)), %d)", + sqlquote.String(id), chunkID, sqlquote.String(url), tag)) + chunkID++ + if len(vals) == maxInsertTuples { + sqls = append(sqls, prefix+strings.Join(vals, ", ")) + vals = vals[:0] + } + } + } + if len(vals) > 0 { + sqls = append(sqls, prefix+strings.Join(vals, ", ")) + } + return sqls, chunkID +} + +// fileChunkInsertSqls splits a [baseOffset, baseOffset+dataLen) BYTE RANGE of a spilled file into +// <= MaxChunkSize storage rows read via load_file (no hex/unhex). Single-range convenience over +// framesInsertSqls (used by the base-segment persist). +func fileChunkInsertSqls(cfg TableConfig, id string, startChunkId int64, path string, baseOffset int64, dataLen int, tag int) []string { + sqls, _ := framesInsertSqls(cfg, id, startChunkId, []fileChunkRange{{path: path, offset: baseOffset, length: dataLen}}, tag) + return sqls +} + +// TailFileInsertSqls persists ONE spilled tag=1 CDC frame (index_id=CdcTailId) that lives at +// [offset, offset+frameLen) within a packed spool file. Its chunk rows are contiguous from +// startChunkId, so recency (chunk_id) ordering across frames is unchanged by the packing. +func TailFileInsertSqls(cfg TableConfig, startChunkId int64, path string, offset int64, frameLen int) []string { + return fileChunkInsertSqls(cfg, vectorindex.CdcTailId, startChunkId, path, offset, frameLen, int(vectorindex.Tag_CdcEvents)) +} + +// TailFramesInsertSqls persists ALL of an iteration's spilled tag=1 CDC frames as FEW multi-row +// INSERTs: the per-chunk load_file tuples of every frame, batched at maxInsertTuples rows per +// statement ACROSS frame boundaries — so N tiny frames cost ~N/maxInsertTuples statements (and +// RunSql round-trips) instead of one INSERT per frame. chunk_ids are contiguous from startChunkId in frame +// order (recency identical to a per-frame persist); the next free chunk_id (end of the tail range) +// is returned. +func TailFramesInsertSqls(cfg TableConfig, startChunkId int64, frames []TailSegment) (sqls []string, nextChunkId int64) { + ranges := make([]fileChunkRange, len(frames)) + for i, f := range frames { + ranges[i] = fileChunkRange{path: f.Path, offset: f.Offset, length: f.FrameLen} + } + return framesInsertSqls(cfg, vectorindex.CdcTailId, startChunkId, ranges, int(vectorindex.Tag_CdcEvents)) +} + +// DeleteSqls removes one index id's chunks + metadata row (rebuild idempotency). +func DeleteSqls(cfg TableConfig, id string) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(id)), + fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable), + catalog.FullText2Index_TblCol_Metadata_Index_Id, sqlquote.String(id)), + } +} + +// DeleteAllBasesSqls removes every tag=0 base (all sub-index chunk rows + all +// metadata rows); the tag=1 CdcTail is untouched. Makes CREATE idempotent. +func DeleteAllBasesSqls(cfg TableConfig) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %d", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_ModelChunk)), + fmt.Sprintf("DELETE FROM %s WHERE TRUE", sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)), + } +} + +// DeleteTailSqls removes the entire tag=1 CdcTail (a REINDEX discards the delta log). +func DeleteTailSqls(cfg TableConfig) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %d", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)), + } +} + +// readMetadata fetches an index id's metadata (checksum + filesize + recency). +func readMetadata(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string) (checksum string, filesize, recency int64, found bool, err error) { + sql := fmt.Sprintf("SELECT %s, %s, %s FROM %s WHERE %s = %s", + catalog.FullText2Index_TblCol_Metadata_Checksum, catalog.FullText2Index_TblCol_Metadata_Filesize, + catalog.FullText2Index_TblCol_Metadata_Recency, + sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable), + catalog.FullText2Index_TblCol_Metadata_Index_Id, sqlquote.String(id)) + res, err := runSql(sqlproc, sql) + if err != nil { + return "", 0, 0, false, err + } + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + checksum = bat.Vecs[0].GetStringAt(0) + filesize = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[1], 0) + recency = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[2], 0) + found = true + break + } + res.Close() + return checksum, filesize, recency, found, nil +} + +// estBytesPerDocHeap approximates the Go-heap cost of loading ONE document's +// index-level metadata: the docmap slot (pk `any` + docLen) plus the per-doc +// liveness structures NewIndex builds from it (liveLoc/top map entries + a liveOrd +// bit). Base posting blocks are mmap'd (reclaimable OS page cache) and are excluded — +// they cannot OOM-kill the CN. It is deliberately generous so checkBaseLoadBudget +// fails a doomed load fast rather than letting it crash the node; a modest +// over-estimate only rejects a borderline load (recoverable via compaction / more +// memory), never silently OOMs. +const estBytesPerDocHeap = 256 + +// checkBaseLoadBudget fails fast if loading every tag=0 base's per-doc metadata into +// the Go heap would blow the CN memory budget — the base analogue of +// checkTailLoadBudget (which guards the tail). It gates on SUM(nrow) (doc count), NOT +// filesize: the bulk of a base (posting blocks) is mmap'd and reclaimable, so only the +// O(docs) docmap + NewIndex liveness maps land on the Go heap. Budget = MemoryTotal*0.8 +// - live Go heap (cgroup-aware). Turns a node-killing OOM into a clear, actionable +// error suggesting compaction. +func checkBaseLoadBudget(sqlproc *sqlexec.SqlProcess, cfg TableConfig) error { + // CAST AS SIGNED so the sum reads back as int64 regardless of how SUM types its + // result (GetFixedAtNoTypeCheck[int64] would misread a decimal vector). + sql := fmt.Sprintf("SELECT CAST(COALESCE(SUM(%s), 0) AS SIGNED) FROM %s", + catalog.FullText2Index_TblCol_Metadata_Nrow, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + res, err := runSql(sqlproc, sql) + if err != nil { + return err + } + defer res.Close() + var ndoc int64 + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + ndoc = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0) + } + need := ndoc * estBytesPerDocHeap + avail := int64(system.MemoryTotal())*8/10 - int64(system.MemoryGolang()) + if need > avail { + return moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf( + "fulltext2 index %s.%s needs ~%d MB of heap for %d docs' metadata but only ~%d MB is free "+ + "(MemoryTotal*0.8 - Go heap). If the index has many deleted rows, MERGE "+ + "(ALTER TABLE ... ALTER REINDEX ... MERGE) reclaims them and may bring it under budget; "+ + "otherwise the live index is too large for this CN — add memory", + cfg.DbName, cfg.IndexTable, need>>20, ndoc, avail>>20)) + } + return nil +} + +// LoadAllBases loads every tag=0 base sub-index listed in the metadata table. +// Returns nil when no base was built (empty-table create → CDC-only index). +// +// NOTE: the heap-budget guard (checkBaseLoadBudget) is deliberately NOT called here. +// This is shared by the query load path AND CompactSegments (MERGE). Guarding it here +// would make the guard self-blocking — MERGE is the very remedy the guard's error +// suggests, but MERGE must load the bases to reclaim dead docs. The query path calls +// the guard itself (Fulltext2Search.Load); MERGE stays exempt so it can always run. +func LoadAllBases(sqlproc *sqlexec.SqlProcess, cfg TableConfig) ([]*Segment, error) { + return loadAllBasesUncached(sqlproc, cfg, nil) +} + +func loadAllBasesUncached(sqlproc *sqlexec.SqlProcess, cfg TableConfig, trace *loadTrace) ([]*Segment, error) { + idSQL := fmt.Sprintf("SELECT %s FROM %s", + catalog.FullText2Index_TblCol_Metadata_Index_Id, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + res, err := runSql(sqlproc, idSQL) + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + if err != nil { + return nil, err + } + var ids []string + for _, bat := range res.Batches { + if bat == nil { + continue + } + for i := 0; i < bat.RowCount(); i++ { + ids = append(ids, bat.Vecs[0].GetStringAt(i)) + } + } + res.Close() + bases := make([]*Segment, 0, len(ids)) + for _, id := range ids { + m, lerr := loadFromStorage(sqlproc, cfg, id, trace) + if lerr != nil { + freeSegs(bases) + return nil, lerr + } + bases = append(bases, m) + } + return bases, nil +} + +func loadAllBases(sqlproc *sqlexec.SqlProcess, cfg TableConfig, trace *loadTrace, generation loadGeneration) ([]*Segment, error) { + indexKey := cfg.DbName + "." + cfg.IndexTable + committed := false + defer func() { + if !committed { + // A later base can fail after earlier segments have been acquired. + // Roll back only claims held by this load; another generation may have + // reused or committed the same immutable base in the meantime. + loadedBasePool.rollback(generation.owner) + } + }() + idSQL := fmt.Sprintf("SELECT %s FROM %s", + catalog.FullText2Index_TblCol_Metadata_Index_Id, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + res, err := runSql(sqlproc, idSQL) + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + if err != nil { + return nil, err + } + var ids []string + for _, bat := range res.Batches { + if bat == nil { + continue + } + for i := 0; i < bat.RowCount(); i++ { + ids = append(ids, bat.Vecs[0].GetStringAt(i)) + } + } + res.Close() + + bases := make([]*Segment, 0, len(ids)) + used := make(map[baseKey]struct{}, len(ids)) + for _, id := range ids { + var metaStart time.Time + if trace != nil { + metaStart = time.Now() + } + checksum, filesize, recency, found, lerr := readMetadata(sqlproc, cfg, id) + if trace != nil { + trace.addInternalSQL(time.Since(metaStart)) + } + if lerr != nil { + freeSegs(bases) + return nil, lerr + } + if !found { + freeSegs(bases) + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("fulltext2 index %s metadata not found", id)) + } + if filesize <= 0 { + freeSegs(bases) + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("fulltext2 index %s has empty filesize", id)) + } + key := baseKey{index: indexKey, id: id, checksum: checksum, filesize: filesize} + m, lerr := loadedBasePool.acquireOwned(sqlproc.GetContext(), key, func() (*Segment, error) { + return loadFromStorageMetadata(sqlproc, cfg, id, checksum, filesize, recency, trace) + }, recency, generation.owner) + if lerr != nil { + // Free the segments already mapped this call before bailing: each owns an + // mmap (+ a linked /tmp spill file on the fallback path), so returning without + // freeing leaks them, and a retrying caller (query reload / MERGE) accumulates + // both. Mirrors LoadTailSegments' freeSegs(bases) on its own error path. + freeSegs(bases) + return nil, lerr + } + bases = append(bases, m) + used[key] = struct{}{} + } + if !loadGenerationCurrent(generation) { + loadedBasePool.rollback(generation.owner) + } else { + published := loadedBasePool.commitOwnedIfCurrent(indexKey, used, generation.owner, func() bool { + return loadGenerationCurrent(generation) + }) + if !published { + loadedBasePool.rollback(generation.owner) + } + } + committed = true + return bases, nil +} + +// LoadFromStorage reads an index id's metadata + chunks, verifies the checksum, +// and deserializes it. Chunks stream by chunk_id offset into a temp file so the +// mpool never holds the whole index. +func LoadFromStorage(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string) (*Segment, error) { + return loadFromStorage(sqlproc, cfg, id, nil) +} + +func loadFromStorage(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string, trace *loadTrace) (*Segment, error) { + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + checksum, filesize, recency, found, err := readMetadata(sqlproc, cfg, id) + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + if err != nil { + return nil, err + } + if !found { + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("fulltext2 index %s metadata not found", id)) + } + if filesize <= 0 { + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("fulltext2 index %s has empty filesize", id)) + } + return loadFromStorageMetadata(sqlproc, cfg, id, checksum, filesize, recency, trace) +} + +func loadFromStorageMetadata(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id, checksum string, filesize, recency int64, trace *loadTrace) (*Segment, error) { + if trace != nil { + trace.addBaseBytes(filesize) + } + + // Materialize the segment on the fast LOCAL (SSD) fileservice so mmap page faults + // come off the 2 GB/s mount, not /tmp (128 MB/s on AWS). path=="" means an + // anonymous SSD file (unlinked; munmap frees the inode) — the same + // CreateAndRemoveFile the JOIN/mergeorder spill uses. + fp, path, err := createLocalTempFile(sqlproc, "ft2idx") + if err != nil { + return nil, err + } + cleanup := func() { + fp.Close() + if path != "" { + os.Remove(path) + } + } + if err = fp.Truncate(filesize); err != nil { + cleanup() + return nil, err + } + if err = streamChunksToFile(sqlproc, cfg, id, filesize, fp, trace); err != nil { + cleanup() + return nil, err + } + // mmap the file read-only (shared across queries; FST + positions are views into + // it, page-cache-backed). The fd is not needed once mapped. + var mmapStart time.Time + if trace != nil { + mmapStart = time.Now() + } + data, err := mmapReadOnly(fp) + if trace != nil { + trace.addMmap(time.Since(mmapStart)) + } + fp.Close() + if err != nil { + if path != "" { + os.Remove(path) + } + return nil, err + } + // Checksum the mapped bytes (the anonymous SSD file has no path to CheckSum). + var checksumStart time.Time + if trace != nil { + checksumStart = time.Now() + } + actualChecksum := vectorindex.CheckSumFromBuffer(data) + if trace != nil { + trace.addChecksum(time.Since(checksumStart)) + } + if actualChecksum != checksum { + _ = munmap(data) + if path != "" { + os.Remove(path) + } + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("fulltext2 index %s checksum mismatch", id)) + } + // The Segment OWNS the mapping (+ path for the /tmp fallback): Free() munmaps and, + // if linked, deletes it. + m := &Segment{Id: id, mmapData: data, mmapPath: path} + if err := m.decodeSegment(data); err != nil { + m.Free() + return nil, err + } + m.Recency = recency + return m, nil +} + +// ft2LocalDir is fulltext2's own subdir under the LOCAL fileservice (on the SSD +// data-dir) for segment spill / mmap files — sibling to the JOIN spill's "__spill". +// EVERY fulltext2 temp file (the build-side load_file spill, the CDC tail spill, and +// the query-side mmap materialization) lives here so it lands on the fast SSD mount, +// not /tmp (128 MB/s on AWS). +const ft2LocalDir = "__fulltext2" + +// localSpillDir returns the on-disk __fulltext2 scratch directory under the LOCAL +// (SSD) fileservice, creating it if absent. Returns "" when no LOCAL fileservice is +// attached (tests / one-shot tools) so callers fall back to the OS temp dir via +// os.CreateTemp("", …)/os.MkdirTemp("", …). Used by the build-side spill (load_file +// needs a real path) — the query mmap path uses the anonymous createLocalTempFile. +func localSpillDir(sqlproc *sqlexec.SqlProcess) string { + if sqlproc == nil || sqlproc.Proc == nil { + return "" + } + return LocalSpillDir(sqlproc.GetContext(), sqlproc.Proc.Base.FileService) +} + +// LocalSpillDir resolves the __fulltext2 dir under the LOCAL fileservice of rootFS +// (the CN's root FileService), creating it. Engine-agnostic so the ISCP CDC tail can +// resolve it from the engine's FileService without a sqlproc. Returns "" when rootFS +// has no LOCAL fileservice. +func LocalSpillDir(ctx context.Context, rootFS fileservice.FileService) string { + if rootFS == nil { + return "" + } + local, err := fileservice.Get[*fileservice.LocalFS](rootFS, defines.LocalFileServiceName) + if err != nil { + return "" + } + dir := filepath.Join(local.RootPath(), ft2LocalDir) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "" + } + return dir +} + +// createLocalSpillFile creates a NAMED temp file (prefix) under the LOCAL fileservice's +// __fulltext2 dir — its absolute on-disk path is what load_file reads back. Falls back +// to the OS temp dir when no LOCAL fileservice is attached (localSpillDir==""). +func createLocalSpillFile(sqlproc *sqlexec.SqlProcess, prefix string) (*os.File, error) { + return os.CreateTemp(localSpillDir(sqlproc), prefix) +} + +// createLocalTempFile returns a temp file for the segment's mmap. It prefers the +// LOCAL fileservice's __fulltext2 subdir (on the SSD data-dir) via the same +// CreateAndRemoveFile the JOIN spill uses — an ANONYMOUS file (unlinked; the fd + +// mapping keep the inode alive, Free just munmaps, no os.Remove). Falls back to +// os.CreateTemp (/tmp, linked → Free deletes by path) when no process/fileservice +// is attached (tests / one-shot tools). Returns (file, path, err); path=="" for the +// anonymous SSD file. name is the caller-chosen file-name prefix. +func createLocalTempFile(sqlproc *sqlexec.SqlProcess, name string) (*os.File, string, error) { + if sqlproc != nil && sqlproc.Proc != nil { + ctx := sqlproc.GetContext() + // LOCAL fileservice (SSD data-dir) -> ensure our __fulltext2 subdir -> a + // MutableFileService rooted there (mirrors process.GetSpillFileService). + if local, e := fileservice.Get[fileservice.MutableFileService]( + sqlproc.Proc.Base.FileService, defines.LocalFileServiceName); e == nil { + if e2 := local.EnsureDir(ctx, ft2LocalDir); e2 == nil { + if sub, ok := fileservice.SubPath(local, ft2LocalDir).(fileservice.MutableFileService); ok { + if f, e3 := sub.CreateAndRemoveFile(ctx, name+"_"+uuid.NewString()); e3 == nil { + return f, "", nil + } + } + } + } + } + f, err := os.CreateTemp("", name) + if err != nil { + return nil, "", err + } + return f, f.Name(), nil +} + +// checkTailLoadBudget fails fast when the CDC tail's stored bytes exceed the CN memory +// budget, returning a clear, actionable error instead of letting the tail load OOM-kill +// the CN (which would take down EVERY query on the node, not just this one). The tail +// chunks are read into the Go HEAP (append([]byte)) and each frame Deserialized into a +// Go-heap segment, so SUM(LENGTH(data)) — the ACTUAL stored bytes — approximates the load +// footprint. The mmap'd base is reclaimable OS page cache and is deliberately NOT counted +// (it cannot cause an OOM-kill). Budget = MemoryTotal*0.8 - live Go heap; MemoryTotal is +// cgroup-aware, and MemoryGolang already includes any tails currently resident, so this +// gates an INCREMENTAL load. The 0.8 headroom absorbs the (small) transient query-mpool +// usage the formula omits. +func checkTailLoadBudget(sqlproc *sqlexec.SqlProcess, cfg TableConfig) error { + return checkTailLoadBudgetAfter(sqlproc, cfg, -1, nil) +} + +func checkTailLoadBudgetAfter(sqlproc *sqlexec.SqlProcess, cfg TableConfig, after int64, trace *loadTrace) error { + where := fmt.Sprintf("%s = %s AND %s = %d", + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + if after >= 0 { + where += fmt.Sprintf(" AND %s > %d", catalog.FullText2Index_TblCol_Storage_Chunk_Id, after) + } + sql := fmt.Sprintf("SELECT COALESCE(SUM(LENGTH(%s)), 0) FROM %s WHERE %s", + catalog.FullText2Index_TblCol_Storage_Data, sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + where) + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + res, err := runSql(sqlproc, sql) + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + if err != nil { + return err + } + defer res.Close() + var need int64 + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + need = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0) + } + // LoadTailSegments holds SEVERAL full copies of the tail bytes at once — the raw + // per-chunk []byte copies, the reassembled per-frame buffers, and the Deserialized + // segments — so the real peak is a multiple of the stored bytes, not 1x. Scale the + // estimate up so the guard fails BEFORE that peak OOM-kills the CN rather than after + // the SUM passes. Conservative (may reject a borderline load that would just fit); a + // clear "compact/add memory" error beats a node-killing OOM. + const tailLoadPeakFactor = 3 + need *= tailLoadPeakFactor + avail := int64(system.MemoryTotal())*8/10 - int64(system.MemoryGolang()) + if need > avail { + return moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf( + "fulltext2 CDC tail for %s.%s needs ~%d MB to load but only ~%d MB is free "+ + "(MemoryTotal*0.8 - Go heap); compact it (ALTER ... REINDEX) or increase CN memory", + cfg.DbName, cfg.IndexTable, need>>20, avail>>20)) + } + return nil +} + +// LoadTailSegments loads the tag=1 CdcTail: the SELECTed chunk rows are ordered, +// reassembled into frames, and each insert frame decoded into a segment (its +// Recency = the frame's first chunk_id). Delete frames are folded into the pk +// tombstone map. Empty tail → (nil, nil, nil). +func LoadTailSegments(sqlproc *sqlexec.SqlProcess, cfg TableConfig) ([]*Segment, map[any]int64, error) { + return loadTailSegments(sqlproc, cfg, nil) +} + +func loadTailSegments(sqlproc *sqlexec.SqlProcess, cfg TableConfig, trace *loadTrace) ([]*Segment, map[any]int64, error) { + segs, deletes, _, err := loadTailSegmentsAfter(sqlproc, cfg, -1, trace) + return segs, deletes, err +} + +func loadTailSegmentsAfter(sqlproc *sqlexec.SqlProcess, cfg TableConfig, after int64, trace *loadTrace) ([]*Segment, map[any]int64, int64, error) { + // Fail fast if the tail can't fit in the memory budget, rather than letting it + // OOM-kill the CN as it decodes into the Go heap. + if err := checkTailLoadBudgetAfter(sqlproc, cfg, after, trace); err != nil { + return nil, nil, 0, err + } + where := fmt.Sprintf("%s = %s AND %s = %d", + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + if after >= 0 { + where += fmt.Sprintf(" AND %s > %d", catalog.FullText2Index_TblCol_Storage_Chunk_Id, after) + } + sql := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s", + catalog.FullText2Index_TblCol_Storage_Chunk_Id, catalog.FullText2Index_TblCol_Storage_Data, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + where) + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + res, err := runSql(sqlproc, sql) + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + if err != nil { + return nil, nil, 0, err + } + var chunks []TailChunk + maxChunk := after + for _, bat := range res.Batches { + if bat == nil { + continue + } + cids := vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0]) + for i := 0; i < bat.RowCount(); i++ { + data := bat.Vecs[1].GetRawBytesAt(i) + if trace != nil { + trace.addTailBytes(int64(len(data))) + } + if cids[i] > maxChunk { + maxChunk = cids[i] + } + chunks = append(chunks, TailChunk{ChunkId: cids[i], Data: append([]byte(nil), data...)}) + } + } + res.Close() + if len(chunks) == 0 { + return nil, nil, maxChunk, nil + } + + ordered, err := orderTailChunks(chunks) + if err != nil { + return nil, nil, 0, err + } + frames, err := reassembleFrames(ordered) + if err != nil { + return nil, nil, 0, err + } + + var segs []*Segment + deletes := make(map[any]int64) + for _, f := range frames { + records, _, nInserts, nDeletes, _, uerr := cuvscdc.UnframeCdcChunk(f.Data) + if uerr != nil { + freeSegs(segs) + return nil, nil, 0, uerr + } + switch { + case nInserts > 0: + seg, derr := Deserialize(fmt.Sprintf("tail-%d", f.ChunkId), bytes.NewReader(records)) + if derr != nil { + freeSegs(segs) + return nil, nil, 0, derr + } + seg.Recency = f.ChunkId + segs = append(segs, seg) + case nDeletes > 0: + recs, derr := DecodeDeleteLog(records) + if derr != nil { + freeSegs(segs) + return nil, nil, 0, derr + } + deletes = foldDeleteFrame(deletes, recs, f.ChunkId) + } + } + return segs, deletes, maxChunk, nil +} + +// streamChunksToFile streams a tag=0 index's chunk rows, writing each at +// chunk_id*MaxChunkSize into fp; the assembled bytes must fill filesize exactly. +func streamChunksToFile(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string, filesize int64, fp *os.File, trace *loadTrace) error { + sql := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s = %s", + catalog.FullText2Index_TblCol_Storage_Chunk_Id, catalog.FullText2Index_TblCol_Storage_Data, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(id)) + written, _, err := streamChunkRowsToFile(sqlproc, sql, 0, filesize, fp, trace) + if err != nil { + return err + } + if written != filesize { + return moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("fulltext2 index %s incomplete: wrote %d of %d bytes", id, written, filesize)) + } + return nil +} + +// streamChunkRowsToFile streams the (chunk_id, data) rows of sql and writes each +// at (chunk_id-baseChunk)*MaxChunkSize into fp, bounding the mpool to the stream +// buffer. Returns bytes written + chunk-row count. +func streamChunkRowsToFile(sqlproc *sqlexec.SqlProcess, sql string, baseChunk, bound int64, fp *os.File, trace *loadTrace) (written, nchunks int64, err error) { + streamCh := make(chan executor.Result, 2) + errorCh := make(chan error, 2) + ctx, cancel := context.WithCancelCause(sqlproc.GetTopContext()) + defer cancel(nil) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer func() { + close(streamCh) + wg.Done() + }() + var sqlStart time.Time + if trace != nil { + sqlStart = time.Now() + } + if _, e := runStreamingSql(ctx, sqlproc, sql, streamCh, errorCh); e != nil { + errorCh <- e + } + if trace != nil { + trace.addInternalSQL(time.Since(sqlStart)) + } + }() + + var loopErr error + closed := false + seen := make(map[int64]struct{}) // chunk_ids already written, to catch duplicates + for !closed { + select { + case res, ok := <-streamCh: + if !ok { + closed = true + break + } + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + cids := vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0]) + for i, cid := range cids { + data := bat.Vecs[1].GetRawBytesAt(i) + off := (cid - baseChunk) * int64(vectorindex.MaxChunkSize) + if off < 0 || off+int64(len(data)) > bound { + loopErr = moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("fulltext2 chunk_id %d out of range [base %d, bound %d]", cid, baseChunk, bound)) + break + } + // A duplicate (index_id, chunk_id) row would write its region twice and + // inflate `written`, letting the caller's written==filesize check pass while + // a DIFFERENT chunk's region is never filled (a hole the CRC only catches + // after a full stream+mmap). Reject the duplicate here, fail-fast. + if _, dup := seen[cid]; dup { + loopErr = moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("fulltext2 duplicate chunk_id %d", cid)) + break + } + seen[cid] = struct{}{} + var writeStart time.Time + if trace != nil { + writeStart = time.Now() + } + if _, e := fp.WriteAt(data, off); e != nil { + loopErr = e + break + } + if trace != nil { + trace.addTempWrite(time.Since(writeStart)) + } + written += int64(len(data)) + nchunks++ + } + if loopErr != nil { + break + } + } + res.Close() + if loopErr != nil { + closed = true + } + case e := <-errorCh: + loopErr = e + closed = true + case <-ctx.Done(): + loopErr = context.Cause(ctx) + closed = true + } + } + if loopErr != nil { + cancel(loopErr) + } + for res := range streamCh { + res.Close() + } + wg.Wait() + if loopErr == nil { + select { + case e := <-errorCh: + loopErr = e + default: + } + } + if loopErr != nil { + return 0, 0, loopErr + } + return written, nchunks, nil +} + +// NextTailChunkIdSql returns a SELECT for the next free tag=1 CdcTail chunk_id — +// the monotonic append position (= recency) the sinker frames at. It is +// GREATEST(MAX tail chunk_id, MAX tag=0 base recency) + 1, so an appended tail is +// always newer than every base under Index liveness, and the sequence never +// resets after a compaction folds the tail into a base. Mirrors bm25's +// NextTailChunkIdSql. +func NextTailChunkIdSql(cfg TableConfig) string { + tailMax := fmt.Sprintf("COALESCE((SELECT MAX(%s) FROM %s WHERE %s = %s AND %s = %d), 0)", + catalog.FullText2Index_TblCol_Storage_Chunk_Id, sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + baseMax := fmt.Sprintf("COALESCE((SELECT MAX(%s) FROM %s), 0)", + catalog.FullText2Index_TblCol_Metadata_Recency, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + return fmt.Sprintf("SELECT GREATEST(%s, %s) + 1", tailMax, baseMax) +} + +// NextTailChunkId runs NextTailChunkIdSql — the next append position for the CDC +// sinker. +func NextTailChunkId(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + res, err := runSql(sqlproc, NextTailChunkIdSql(cfg)) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + return 0, nil +} + +// StaleGenSqls returns the two queries whose results form the cache-freshness generation +// used by Fulltext2Search.IsStale: MAX(metadata.timestamp) (bumped by a REBUILD/MERGE that +// writes a new base model row) and MAX(storage.chunk_id) of the tag=1 CdcTail (bumped by a +// CDC append). A change in EITHER means the loaded index is stale. Two reads because +// timestamp and tag live in different tables (metadata has no tag; storage has no +// timestamp). The tail read is scoped to (CdcTailId, tag=1) — the exact CDC delta — so an +// unrelated base sub-index's higher chunk_id can't mask a fresh append. +func StaleGenSqls(cfg TableConfig) (tsSQL, tailSQL string) { + tsSQL = fmt.Sprintf("SELECT COALESCE(MAX(%s), 0) FROM %s", + catalog.FullText2Index_TblCol_Metadata_Timestamp, + sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + tailSQL = fmt.Sprintf("SELECT COALESCE(MAX(%s), -1) FROM %s WHERE %s = %s AND %s = %d", + catalog.FullText2Index_TblCol_Storage_Chunk_Id, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + return +} + +func resultScalarInt64(res executor.Result) int64 { + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0) + } + return 0 +} + +// LoadGeneration reads the current (timestamp, tailChunk) generation using the caller's +// LIVE sqlproc/txn — called at load time so the captured generation reflects exactly the +// txn snapshot the cached index was built from. +func LoadGeneration(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (ts int64, tail int64, err error) { + defer func() { + if r := recover(); r != nil { + ts, tail, err = 0, 0, moerr.NewInternalErrorNoCtx(fmt.Sprintf("LoadGeneration recovered: %v", r)) + } + }() + tsSQL, tailSQL := StaleGenSqls(cfg) + res, err := runSql(sqlproc, tsSQL) + if err != nil { + return 0, 0, err + } + ts = resultScalarInt64(res) + res.Close() + res, err = runSql(sqlproc, tailSQL) + if err != nil { + return 0, 0, err + } + tail = resultScalarInt64(res) + res.Close() + return ts, tail, nil +} + +// QueryGeneration reads the current (timestamp, tailChunk) generation in the BACKGROUND +// (housekeeping goroutine, no live sqlproc) via an executor-managed auto-commit txn, keyed +// by the CN UUID + tenant captured at load. Used by IsStale. +func QueryGeneration(ctx context.Context, cnUUID string, accountID uint32, cfg TableConfig) (ts int64, tail int64, err error) { + defer func() { + if r := recover(); r != nil { + ts, tail, err = 0, 0, moerr.NewInternalErrorNoCtx(fmt.Sprintf("QueryGeneration recovered: %v", r)) + } + }() + tsSQL, tailSQL := StaleGenSqls(cfg) + res, err := sqlexec.RunSqlAutoCommit(ctx, cnUUID, accountID, cfg.DbName, tsSQL) + if err != nil { + return 0, 0, err + } + ts = resultScalarInt64(res) + res.Close() + res, err = sqlexec.RunSqlAutoCommit(ctx, cnUUID, accountID, cfg.DbName, tailSQL) + if err != nil { + return 0, 0, err + } + tail = resultScalarInt64(res) + res.Close() + return ts, tail, nil +} + +// CountTailChunks returns the number of tag=1 CdcTail chunk rows — the idxcron +// tail-growth gate (fold once the delta is large enough). +func CountTailChunks(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + sql := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = %s AND %s = %d", + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.FullText2Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.FullText2Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + return scanInt64(sqlproc, sql) +} + +// SumBaseNrow sums metadata.nrow over the tag=0 bases — the base doc count for the +// dead-doc estimate (compared to the live source row count). +func SumBaseNrow(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + sql := fmt.Sprintf("SELECT COALESCE(SUM(%s), 0) FROM %s", + catalog.FullText2Index_TblCol_Metadata_Nrow, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + return scanInt64(sqlproc, sql) +} + +func scanInt64(sqlproc *sqlexec.SqlProcess, sql string) (int64, error) { + res, err := runSql(sqlproc, sql) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat != nil && bat.RowCount() > 0 { + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + } + return 0, nil +} + +// FrameChunkCount is how many MaxChunkSize storage rows a frame of frameLen bytes +// occupies (>= 1) — the streaming sinker advances chunk_id past each spilled +// segment by this without holding the framed bytes. Mirrors bm25's FrameChunkCount. +func FrameChunkCount(frameLen int) int64 { + n := int64((frameLen + vectorindex.MaxChunkSize - 1) / vectorindex.MaxChunkSize) + if n < 1 { + n = 1 + } + return n +} diff --git a/pkg/fulltext2/storage_mock_test.go b/pkg/fulltext2/storage_mock_test.go new file mode 100644 index 0000000000000..22d0684b7a0f2 --- /dev/null +++ b/pkg/fulltext2/storage_mock_test.go @@ -0,0 +1,429 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/stretchr/testify/require" +) + +// swapRunSql / swapRunStreamingSql install a mock and return a restore func. The engine +// indirects sqlexec.RunSql / RunStreamingSql through these package vars so DB round-trips +// are mockable. +func swapRunSql(t *testing.T, fn func(*sqlexec.SqlProcess, string) (executor.Result, error)) { + prev := runSql + runSql = fn + t.Cleanup(func() { runSql = prev }) +} + +func swapRunStreamingSql(t *testing.T, fn func(context.Context, *sqlexec.SqlProcess, string, chan executor.Result, chan error) (executor.Result, error)) { + prev := runStreamingSql + runStreamingSql = fn + t.Cleanup(func() { runStreamingSql = prev }) +} + +func int64Batch(mp *mpool.MPool, v int64) *batch.Batch { + b := batch.NewWithSize(1) + b.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + _ = vector.AppendFixed[int64](b.Vecs[0], v, false, mp) + b.SetRowCount(1) + return b +} + +func metaBatch(mp *mpool.MPool, checksum string, filesize, recency int64) *batch.Batch { + b := batch.NewWithSize(3) + b.Vecs[0] = vector.NewVec(types.T_varchar.ToType()) + b.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + b.Vecs[2] = vector.NewVec(types.T_int64.ToType()) + _ = vector.AppendBytes(b.Vecs[0], []byte(checksum), false, mp) + _ = vector.AppendFixed[int64](b.Vecs[1], filesize, false, mp) + _ = vector.AppendFixed[int64](b.Vecs[2], recency, false, mp) + b.SetRowCount(1) + return b +} + +// chunkBatch splits buf into (chunk_id, data) rows of <= MaxChunkSize bytes, the shape +// the base-chunk streaming reader consumes. +func chunkBatch(mp *mpool.MPool, buf []byte) *batch.Batch { + nchunks := (len(buf) + vectorindex.MaxChunkSize - 1) / vectorindex.MaxChunkSize + if nchunks == 0 { + nchunks = 0 + } + b := batch.NewWithSize(2) + b.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + b.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + for i := 0; i < nchunks; i++ { + off := i * vectorindex.MaxChunkSize + end := off + vectorindex.MaxChunkSize + if end > len(buf) { + end = len(buf) + } + _ = vector.AppendFixed[int64](b.Vecs[0], int64(i), false, mp) + _ = vector.AppendBytes(b.Vecs[1], buf[off:end], false, mp) + } + b.SetRowCount(nchunks) + return b +} + +func mockSqlProc(t *testing.T) (*sqlexec.SqlProcess, *mpool.MPool) { + proc := testutil.NewProc(t) + return sqlexec.NewSqlProcess(proc), proc.Mp() +} + +func TestReadMetadata(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + // found row. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{metaBatch(mp, "chk", 42, 7)}}, nil + }) + checksum, filesize, recency, found, err := readMetadata(sp, cfg, "id0") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "chk", checksum) + require.Equal(t, int64(42), filesize) + require.Equal(t, int64(7), recency) + + // no rows → found=false. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: nil}, nil + }) + _, _, _, found, err = readMetadata(sp, cfg, "id0") + require.NoError(t, err) + require.False(t, found) + + // runSql error is surfaced. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{}, moerr.NewInternalErrorNoCtx("boom") + }) + _, _, _, _, err = readMetadata(sp, cfg, "id0") + require.Error(t, err) +} + +func TestScanHelpers(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 13)}}, nil + }) + + n, err := CountTailChunks(sp, cfg) + require.NoError(t, err) + require.Equal(t, int64(13), n) + + n, err = SumBaseNrow(sp, cfg) + require.NoError(t, err) + require.Equal(t, int64(13), n) + + n, err = NextTailChunkId(sp, cfg) + require.NoError(t, err) + require.Equal(t, int64(13), n) + + // error path. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{}, moerr.NewInternalErrorNoCtx("boom") + }) + _, err = SumBaseNrow(sp, cfg) + require.Error(t, err) +} + +func TestLoadBudgetGates(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + // small doc/byte counts fit comfortably → nil. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 100)}}, nil + }) + require.NoError(t, checkBaseLoadBudget(sp, cfg)) + require.NoError(t, checkTailLoadBudget(sp, cfg)) + + // an enormous count exceeds the CN budget → actionable error (no int64 overflow). + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, int64(1)<<40)}}, nil + }) + require.Error(t, checkBaseLoadBudget(sp, cfg)) + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, int64(1)<<50)}}, nil + }) + require.Error(t, checkTailLoadBudget(sp, cfg)) +} + +func TestLoadAllBasesEmpty(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + // enumerate returns no ids → no bases (never touches LoadFromStorage). + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: nil}, nil + }) + trace := &loadTrace{start: time.Now()} + bases, err := loadAllBasesUncached(sp, cfg, trace) + require.NoError(t, err) + require.Empty(t, bases) +} + +func TestLoadTailSegmentsEmpty(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + // dispatch by SQL: the budget gate is a SUM(LENGTH(...)) 1-col scan; the tail SELECT + // (chunk_id, data) returns no rows → empty tail. + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + if strings.Contains(sql, "LENGTH(") { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + } + return executor.Result{Mp: mp, Batches: nil}, nil + }) + trace := &loadTrace{start: time.Now()} + segs, deletes, _, err := loadTailSegmentsAfter(sp, cfg, 7, trace) + require.NoError(t, err) + require.Empty(t, segs) + require.Empty(t, deletes) + require.GreaterOrEqual(t, trace.phase.internalSQL, time.Duration(0)) +} + +// TestLoadFromStorageRoundTrip mocks the metadata read + base-chunk stream with a REAL +// serialized segment, exercising the full decode path (spill → mmap → checksum → decode). +func TestLoadFromStorageRoundTrip(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + b := NewBuilder("seg0", int32(types.T_int64)) + feed(t, b, int64(1), "hello", "world") + feed(t, b, int64(2), "hello", "matrix") + seg, err := b.Finish() + require.NoError(t, err) + seg.Id = "seg0" + seg.Recency = 5 + buf, err := seg.Serialize() + require.NoError(t, err) + checksum := vectorindex.CheckSumFromBuffer(buf) + filesize := int64(len(buf)) + + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{metaBatch(mp, checksum, filesize, 5)}}, nil + }) + swapRunStreamingSql(t, func(_ context.Context, _ *sqlexec.SqlProcess, _ string, sc chan executor.Result, _ chan error) (executor.Result, error) { + sc <- executor.Result{Mp: mp, Batches: []*batch.Batch{chunkBatch(mp, buf)}} + return executor.Result{}, nil + }) + + trace := &loadTrace{start: time.Now()} + loaded, err := loadFromStorage(sp, cfg, "seg0", trace) + require.NoError(t, err) + require.Equal(t, "seg0", loaded.Id) + require.Equal(t, int64(5), loaded.Recency) + require.Equal(t, seg.N, loaded.N) + require.Equal(t, filesize, trace.event.BaseBytes) + require.Positive(t, trace.phase.tempWrite) + loaded.Free() + + // a checksum mismatch (corrupt stream) is detected and rejected. + swapRunStreamingSql(t, func(_ context.Context, _ *sqlexec.SqlProcess, _ string, sc chan executor.Result, _ chan error) (executor.Result, error) { + bad := append([]byte(nil), buf...) + bad[0] ^= 0xFF + sc <- executor.Result{Mp: mp, Batches: []*batch.Batch{chunkBatch(mp, bad)}} + return executor.Result{}, nil + }) + _, err = LoadFromStorage(sp, cfg, "seg0") + require.ErrorContains(t, err, "checksum mismatch") + + // missing metadata → clear error. + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: nil}, nil + }) + _, err = LoadFromStorage(sp, cfg, "seg0") + require.ErrorContains(t, err, "metadata not found") +} + +// tailChunkBatch renders forged (chunk_id, data) rows as the tail-data result batch. +func tailChunkBatch(mp *mpool.MPool, chunks []TailChunk) *batch.Batch { + b := batch.NewWithSize(2) + b.Vecs[0] = vector.NewVec(types.T_int64.ToType()) + b.Vecs[1] = vector.NewVec(types.T_varchar.ToType()) + for _, c := range chunks { + _ = vector.AppendFixed[int64](b.Vecs[0], c.ChunkId, false, mp) + _ = vector.AppendBytes(b.Vecs[1], c.Data, false, mp) + } + b.SetRowCount(len(chunks)) + return b +} + +// TestCompactSegmentsFoldsTail drives the full MERGE: no bases + a forged tag=1 tail +// insert frame ⇒ the tail is folded into a fresh base. runSql dispatches by SQL text +// (the user's "different runSql for different SQLs") and DELETE/INSERT writes succeed. +func TestCompactSegmentsFoldsTail(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + // forge a tail insert frame carrying two docs. + tb := NewBuilder("tail", int32(types.T_int64)) + feed(t, tb, int64(1), "hello", "world") + feed(t, tb, int64(2), "hello", "matrix") + tseg, err := tb.Finish() + require.NoError(t, err) + framed, err := FrameSegment(tseg) + require.NoError(t, err) + chunks := splitFrameChunks(1, framed) + + var deleteAllRan, insertRan bool + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + switch { + case strings.Contains(sql, "GREATEST"): // NextTailChunkId + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 100)}}, nil + case strings.Contains(sql, "LENGTH("): // checkTailLoadBudget + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 1)}}, nil + case strings.Contains(sql, vectorindex.CdcTailId) && strings.Contains(sql, "SELECT"): // tail data + return executor.Result{Mp: mp, Batches: []*batch.Batch{tailChunkBatch(mp, chunks)}}, nil + case strings.HasPrefix(strings.TrimSpace(sql), "SELECT"): // LoadAllBases enumerate → no bases + return executor.Result{Mp: mp, Batches: nil}, nil + default: // DELETE / INSERT writes succeed + if strings.HasPrefix(sql, "DELETE") && strings.Contains(sql, "TRUE") { + deleteAllRan = true + } + if strings.HasPrefix(sql, "INSERT") { + insertRan = true + } + return executor.Result{Mp: mp}, nil + } + }) + + trace := &loadTrace{start: time.Now()} + tracedSegs, tracedDeletes, maxChunk, err := loadTailSegmentsAfter(sp, cfg, -1, trace) + require.NoError(t, err) + require.Len(t, tracedSegs, 1) + require.Empty(t, tracedDeletes) + require.Equal(t, chunks[len(chunks)-1].ChunkId, maxChunk) + require.Positive(t, trace.event.TailBytes) + freeSegs(tracedSegs) + + nlive, err := CompactSegments(sp, cfg, 0, 0) + require.NoError(t, err) + require.Equal(t, 2, nlive) // both docs are live → folded into the fresh base + require.True(t, deleteAllRan, "MERGE must clear prior bases first") + require.True(t, insertRan, "MERGE must persist the rebuilt base") +} + +// twoIdBatch renders a 2-row index_id enumerate result. +func twoIdBatch(mp *mpool.MPool, a, b string) *batch.Batch { + bat := batch.NewWithSize(1) + bat.Vecs[0] = vector.NewVec(types.T_varchar.ToType()) + _ = vector.AppendBytes(bat.Vecs[0], []byte(a), false, mp) + _ = vector.AppendBytes(bat.Vecs[0], []byte(b), false, mp) + bat.SetRowCount(2) + return bat +} + +// loadOneBase round-trips a real serialized segment through LoadFromStorage (mocked +// metadata + chunk stream), returning the mapped segment. +func loadOneBase(t *testing.T, sp *sqlexec.SqlProcess, mp *mpool.MPool, cfg TableConfig, id string) *Segment { + t.Helper() + b := NewBuilder(id, int32(types.T_int64)) + feed(t, b, int64(1), "hello") + seg, err := b.Finish() + require.NoError(t, err) + seg.Id = id + buf, err := seg.Serialize() + require.NoError(t, err) + swapRunSql(t, func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{Mp: mp, Batches: []*batch.Batch{metaBatch(mp, vectorindex.CheckSumFromBuffer(buf), int64(len(buf)), 0)}}, nil + }) + swapRunStreamingSql(t, func(_ context.Context, _ *sqlexec.SqlProcess, _ string, sc chan executor.Result, _ chan error) (executor.Result, error) { + sc <- executor.Result{Mp: mp, Batches: []*batch.Batch{chunkBatch(mp, buf)}} + return executor.Result{}, nil + }) + m, err := LoadFromStorage(sp, cfg, id) + require.NoError(t, err) + return m +} + +// TestFreeSegsReleasesMmap pins that freeSegs munmaps a loaded segment (nils mmapData), +// the primitive LoadAllBases now uses to avoid leaking on a partial load. +func TestFreeSegsReleasesMmap(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + m := loadOneBase(t, sp, mp, cfg, "s0") + require.NotNil(t, m.mmapData) // mapped + freeSegs([]*Segment{m, nil}) // nil entry must be a no-op + require.Nil(t, m.mmapData) // munmapped +} + +// TestLoadAllBasesFreesOnPartialFailure: enumerate returns two ids; base "s0" maps, then +// base "s1" fails (metadata missing). LoadAllBases must free the already-mapped s0 before +// returning the error rather than leaking its mmap (+ spill file) — the #4 fix. +func TestLoadAllBasesFreesOnPartialFailure(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + b := NewBuilder("s0", int32(types.T_int64)) + feed(t, b, int64(1), "hello") + seg, err := b.Finish() + require.NoError(t, err) + buf, err := seg.Serialize() + require.NoError(t, err) + + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + switch { + case strings.Contains(sql, "'s1'"): // readMetadata for s1 → missing → LoadFromStorage errors + return executor.Result{Mp: mp, Batches: nil}, nil + case strings.Contains(sql, "checksum"): // readMetadata for s0 + return executor.Result{Mp: mp, Batches: []*batch.Batch{metaBatch(mp, vectorindex.CheckSumFromBuffer(buf), int64(len(buf)), 0)}}, nil + default: // enumerate index_id → [s0, s1] + return executor.Result{Mp: mp, Batches: []*batch.Batch{twoIdBatch(mp, "s0", "s1")}}, nil + } + }) + swapRunStreamingSql(t, func(_ context.Context, _ *sqlexec.SqlProcess, _ string, sc chan executor.Result, _ chan error) (executor.Result, error) { + sc <- executor.Result{Mp: mp, Batches: []*batch.Batch{chunkBatch(mp, buf)}} + return executor.Result{}, nil + }) + + trace := &loadTrace{start: time.Now()} + bases, err := loadAllBasesUncached(sp, cfg, trace) + require.Error(t, err, "s1 fails to load") + require.Nil(t, bases, "no partial slice is returned (s0 was freed)") +} + +// TestCompactSegmentsNoDelta covers the early-out: empty bases + empty tail ⇒ nothing to +// compact. +func TestCompactSegmentsNoDelta(t *testing.T) { + sp, mp := mockSqlProc(t) + cfg := testStorageCfg() + + swapRunSql(t, func(_ *sqlexec.SqlProcess, sql string) (executor.Result, error) { + if strings.Contains(sql, "LENGTH(") { + return executor.Result{Mp: mp, Batches: []*batch.Batch{int64Batch(mp, 0)}}, nil + } + return executor.Result{Mp: mp, Batches: nil}, nil // empty enumerate + empty tail + }) + nlive, err := CompactSegments(sp, cfg, 0, 0) + require.NoError(t, err) + require.Equal(t, 0, nlive) +} diff --git a/pkg/fulltext2/storage_test.go b/pkg/fulltext2/storage_test.go new file mode 100644 index 0000000000000..d4cba6b7c6d64 --- /dev/null +++ b/pkg/fulltext2/storage_test.go @@ -0,0 +1,189 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +func testStorageCfg() TableConfig { + return TableConfig{DbName: "db", SrcTable: "src", IndexTable: "__store", MetadataTable: "__meta"} +} + +func TestSubIndexId(t *testing.T) { + require.Equal(t, "build:7", SubIndexId("build", 7)) + require.Equal(t, "a:b:c:0", SubIndexId("a:b:c", 0)) +} + +func TestDeleteSqls(t *testing.T) { + cfg := testStorageCfg() + + del := DeleteSqls(cfg, "idx:1") + require.Len(t, del, 2) + require.Contains(t, del[0], "__store") + require.Contains(t, del[0], "idx:1") + require.Contains(t, del[1], "__meta") + + all := DeleteAllBasesSqls(cfg) + require.Len(t, all, 2) + require.Contains(t, all[0], "__store") + require.Contains(t, all[1], "__meta") + + tail := DeleteTailSqls(cfg) + require.Len(t, tail, 1) + require.Contains(t, tail[0], "__store") +} + +func TestFileChunkInsertSqls(t *testing.T) { + cfg := testStorageCfg() + + // a small whole-file (baseOffset 0) fits in ONE chunk row and ONE INSERT. + one := fileChunkInsertSqls(cfg, "idx:0", 0, "/tmp/spill", 0, 10, int(vectorindex.Tag_ModelChunk)) + require.Len(t, one, 1) + require.Contains(t, one[0], "load_file(") + require.Contains(t, one[0], "offset=0") + require.Contains(t, one[0], "size=10") + + // a file spanning > maxInsertTuples chunks splits into multiple INSERT statements: + // maxInsertTuples+1 chunks ⇒ one full INSERT + one trailing INSERT. + nchunks := maxInsertTuples + 1 + dataLen := nchunks * vectorindex.MaxChunkSize + many := fileChunkInsertSqls(cfg, "idx:0", 0, "/tmp/spill", 0, dataLen, int(vectorindex.Tag_ModelChunk)) + require.Len(t, many, 2) + // every chunk contributes one VALUES tuple. + require.Equal(t, nchunks, strings.Count(many[0], "load_file(")+strings.Count(many[1], "load_file(")) + + // a PACKED frame at a non-zero baseOffset: the load_file range is shifted by the offset, so a + // two-chunk frame reads [off, off+MaxChunkSize) then [off+MaxChunkSize, off+len). + off := int64(4096) + packed := fileChunkInsertSqls(cfg, "idx:0", 0, "/tmp/spool", off, vectorindex.MaxChunkSize+7, int(vectorindex.Tag_CdcEvents)) + require.Len(t, packed, 1) + require.Contains(t, packed[0], fmt.Sprintf("offset=%d", off)) + require.Contains(t, packed[0], fmt.Sprintf("offset=%d", off+int64(vectorindex.MaxChunkSize))) + require.Contains(t, packed[0], "size=7") + + // the tail helper delegates to fileChunkInsertSqls with CdcTailId + Tag_CdcEvents, threading the frame offset. + tail := TailFileInsertSqls(cfg, 5, "/tmp/spool", 200, vectorindex.MaxChunkSize+1) + require.Len(t, tail, 1) + require.Contains(t, tail[0], vectorindex.CdcTailId) + require.Contains(t, tail[0], "offset=200") +} + +// TestTailFramesInsertSqls pins the cross-frame batching: N tiny (1-chunk) frames must NOT become N +// INSERT statements — they batch at maxInsertTuples rows per statement ACROSS frame +// boundaries, with contiguous chunk_ids and the right next-chunk-id returned. This is what turns a +// burst of ~360k op-run frames into ~3.6k RunSql round-trips instead of 360k. +func TestTailFramesInsertSqls(t *testing.T) { + cfg := testStorageCfg() + + // Enough tiny 1-chunk frames to force exactly 3 statements regardless of maxInsertTuples: + // two full batches + a 50-row remainder. (Robust to the maxInsertTuples value.) + n := 2*maxInsertTuples + 50 + frames := make([]TailSegment, n) + for i := 0; i < n; i++ { + frames[i] = TailSegment{Path: "/tmp/spool", Offset: int64(i * 8), FrameLen: 8} + } + startChunk := int64(7) + sqls, next := TailFramesInsertSqls(cfg, startChunk, frames) + + // n rows batched at maxInsertTuples/statement ⇒ 3 statements, NOT n (one-per-frame). + require.Len(t, sqls, 3) + total := 0 + for _, s := range sqls { + total += strings.Count(s, "load_file(") + require.Contains(t, s, vectorindex.CdcTailId) + } + require.Equal(t, n, total, "every frame contributes exactly one chunk row") + // chunk_ids are contiguous from startChunk; next = startChunk + total chunks. + require.Equal(t, startChunk+int64(n), next) + require.Contains(t, sqls[0], fmt.Sprintf(", %d, load_file", startChunk)) // first chunk id + require.Contains(t, sqls[2], fmt.Sprintf(", %d, load_file", startChunk+int64(n)-1)) // last chunk id + require.Contains(t, sqls[2], fmt.Sprintf("offset=%d", (n-1)*8)) // last frame's offset in the last stmt + + // empty input ⇒ no statements, next == start. + sqls0, next0 := TailFramesInsertSqls(cfg, startChunk, nil) + require.Empty(t, sqls0) + require.Equal(t, startChunk, next0) +} + +func TestNextTailChunkIdSql(t *testing.T) { + sql := NextTailChunkIdSql(testStorageCfg()) + require.Contains(t, sql, "GREATEST") + require.Contains(t, sql, vectorindex.CdcTailId) + require.Contains(t, sql, "__store") + require.Contains(t, sql, "__meta") +} + +func TestFrameChunkCount(t *testing.T) { + require.Equal(t, int64(1), FrameChunkCount(0)) // empty ⇒ at least one + require.Equal(t, int64(1), FrameChunkCount(1)) // partial ⇒ one + require.Equal(t, int64(1), FrameChunkCount(vectorindex.MaxChunkSize)) // exact ⇒ one + require.Equal(t, int64(2), FrameChunkCount(vectorindex.MaxChunkSize+1)) // spillover ⇒ two + require.Equal(t, int64(3), FrameChunkCount(2*vectorindex.MaxChunkSize+1)) // ceil +} + +func TestToInsertSqls(t *testing.T) { + cfg := testStorageCfg() + + b := NewBuilder("uid:0", int32(types.T_int64)) + feed(t, b, int64(1), "hello", "world") + feed(t, b, int64(2), "hello", "matrix") + seg, err := b.Finish() + require.NoError(t, err) + seg.Id = SubIndexId("uid", 0) + seg.Recency = 3 + + // nil sqlproc ⇒ createLocalSpillFile falls back to an os temp file. + sqls, cleanup, err := seg.ToInsertSqls(nil, cfg, 12345, int(vectorindex.Tag_ModelChunk)) + require.NoError(t, err) + require.NotNil(t, cleanup) + defer cleanup() + + // first SQL is the metadata row; the rest persist the chunk bytes. + require.GreaterOrEqual(t, len(sqls), 2) + require.Contains(t, sqls[0], "__meta") + require.Contains(t, sqls[0], "uid:0") + require.Contains(t, sqls[1], "__store") + require.Contains(t, sqls[1], "load_file(") +} + +func TestCreateLocalSpillAndTempFile(t *testing.T) { + // nil sqlproc ⇒ both spill helpers fall back to the OS temp dir. + require.Equal(t, "", localSpillDir(nil)) + + fp, err := createLocalSpillFile(nil, "ft2test") + require.NoError(t, err) + name := fp.Name() + require.NoError(t, fp.Close()) + require.NoError(t, os.Remove(name)) + + f, path, err := createLocalTempFile(nil, "ft2test") + require.NoError(t, err) + require.NotEmpty(t, path) // linked file in the OS temp dir + require.NoError(t, f.Close()) + require.NoError(t, os.Remove(path)) +} + +func TestLocalSpillDirNilRootFS(t *testing.T) { + require.Equal(t, "", LocalSpillDir(context.Background(), nil)) +} diff --git a/pkg/fulltext2/stream.go b/pkg/fulltext2/stream.go new file mode 100644 index 0000000000000..a519211d447ab --- /dev/null +++ b/pkg/fulltext2/stream.go @@ -0,0 +1,357 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "math" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// streamBatch is the max rows a streamSink buffers before flushing to emit +// (matches bm25's streamBatch). +const streamBatch = 8192 + +// streamSink batches (pk, score) results into a TYPED, box-free vectorindex.ColumnBuffer +// and flushes to emit in bounded chunks, so a no-LIMIT query returns every matching doc +// WITHOUT materializing them all and WITHOUT boxing a pk per doc. On an emit error it +// records it and stops (the walk checks stopped and bails). +type streamSink struct { + emit func(out *vectorindex.SearchOutput) error + keys *vectorindex.ColumnBuffer // pooled; nil until the first push after a flush + pkType types.T + scores []float32 + // includeCols is COLUMN-MAJOR: one nullable ColumnBuffer per FULL index INCLUDE column + // (segment-include order), each covering the whole batch — the box-free analogue of the + // old per-row [][]any. nil until the first push after a flush; len == len(includeTypes) + // when wantInclude. Not pooled (per-batch, potentially varied types); the consumer copies + // each out via AppendColumnBuffer and drops it. + includeCols []*vectorindex.ColumnBuffer + includeTypes []int32 // FULL index INCLUDE column types (drives includeCols alloc) + fixedW int // fixed pk width; 0 ⇒ varlena + err error + stopped bool + wantInclude bool // covered fast path: also carry each doc's INCLUDE values through emit + scoreRange *ScoreRange +} + +// newStreamSink resolves the index's pk type (all segments share it). The batch buffer +// itself is fetched from the pool lazily on the first push. Callers create it only after +// the globalN==0 early-out, so segments is non-empty. wantInclude turns on the covered +// fast path: each pushed doc also carries its INCLUDE values (box-free) through emit. +func newStreamSink(idx *Index, wantInclude bool, filter *prefilter, emit func(out *vectorindex.SearchOutput) error) *streamSink { + pkType := idx.segments[0].PkType + w, fixed := fixedPkByteWidth(pkType) + if !fixed { + w = 0 + } + s := &streamSink{emit: emit, pkType: types.T(pkType), fixedW: w, wantInclude: wantInclude} + if filter != nil { + s.scoreRange = filter.scoreRange + } + if wantInclude { + s.includeTypes = idx.segments[0].includeTypes + } + return s +} + +// ensure lazily fetches a pooled batch buffer of the sink's pk type, and (when the covered +// path is on) allocates one typed, nullable ColumnBuffer per FULL INCLUDE column. +func (s *streamSink) ensure() { + if s.keys == nil { + s.keys = GetColumnBuffer(s.pkType) + } + if s.wantInclude && s.includeCols == nil { + s.includeCols = make([]*vectorindex.ColumnBuffer, len(s.includeTypes)) + for c, t := range s.includeTypes { + s.includeCols[c] = &vectorindex.ColumnBuffer{Type: types.T(t)} + } + } +} + +// appendIncludes appends doc (seg, ord)'s INCLUDE values to the per-column buffers box-free. +// On a malformed segment it records the error and stops the sink so no torn batch is emitted. +func (s *streamSink) appendIncludes(seg *Segment, ord int64) { + for c := range s.includeCols { + if err := seg.appendIncludeTo(s.includeCols[c], c, ord); err != nil { + s.err = err + s.stopped = true + return + } + } +} + +// pushPk appends the pk at (seg, ord) to the typed batch WITHOUT boxing, decoding it +// straight from the segment docmap into ColumnBuffer.Data. +func (s *streamSink) pushPk(seg *Segment, ord int64, score float32) { + if s.stopped || !s.scoreRange.contains(score) { + return + } + s.ensure() + seg.appendPkTo(s.keys, s.fixedW, ord) + s.scores = append(s.scores, score) + if s.wantInclude { + s.appendIncludes(seg, ord) + if s.stopped { + return + } + } + if s.keys.N >= streamBatch { + s.flush() + } +} + +// pushAny appends an already-boxed pk (the materialized-fallback path: NL phrase / full +// boolean) by re-encoding its concrete value into ColumnBuffer.Data — no NEW box. When the +// covered fast path is on, the INCLUDE values are pulled box-free from (seg, ord). +func (s *streamSink) pushAny(seg *Segment, ord int64, pk any, score float32) { + if s.stopped || !s.scoreRange.contains(score) { + return + } + s.ensure() + appendAnyPk(s.keys, s.fixedW, pk) + s.scores = append(s.scores, score) + if s.wantInclude { + s.appendIncludes(seg, ord) + if s.stopped { + return + } + } + if s.keys.N >= streamBatch { + s.flush() + } +} + +func (s *streamSink) flush() { + if s.stopped || s.keys == nil || s.keys.N == 0 { + return + } + // Hand the batch to the consumer as a *SearchOutput — the SAME box-free container + // SearchInto fills on the LIMIT path, so both share one consumer (Dists is float32, + // matching the T_float32 score column). This SearchOutput is producer-owned per batch; + // the consumer recycles out.Keys (pooled) after copying and drops Dists/Include. + out := &vectorindex.SearchOutput{Keys: s.keys, Dists: s.scores, Include: s.includeCols} + if e := s.emit(out); e != nil { + s.err = e + s.stopped = true + PutColumnBuffer(s.keys) // emit didn't hand it off; recycle here + s.keys = nil + s.includeCols = nil + return + } + // Ownership of this batch passes to the consumer, which PutColumnBuffers the pk buffer + // back once it has copied Data out (the include buffers are not pooled, just dropped); + // the sink lazily Gets fresh ones on the next push. + s.keys = nil + s.scores = nil + s.includeCols = nil +} + +// appendPkTo decodes the pk at ord from seg's docmap into k.Data with no boxing: a +// fixed-width pk contributes its width bytes; a varlena pk its [u32 len][content] entry. +func (s *Segment) appendPkTo(k *vectorindex.ColumnBuffer, fixedW int, ord int64) { + if s.pks != nil { // build-side segment: re-encode the already-boxed value + appendAnyPk(k, fixedW, s.pks[ord]) + return + } + off := int(s.pkOffsets[ord]) + l := int(binary.LittleEndian.Uint32(s.pkRaw[off:])) + if fixedW > 0 { + k.Data = append(k.Data, s.pkRaw[off+4:off+4+fixedW]...) // width bytes, no len prefix + } else { + k.Data = append(k.Data, s.pkRaw[off:off+4+l]...) // [u32 len][content] verbatim + } + k.N++ +} + +// streamWAND does a heap-free document-at-a-time OR merge over one segment: it visits +// every matching doc in ord order, scores it (over the SHOULD terms present, on the +// GLOBAL corpus stats), and pushes it to the sink — no top-K heap, no WAND pruning, +// no full-result materialization. allow (liveness ∧ WHERE prefilter) drops +// non-admissible docs. The no-LIMIT case wants every match; ranking is done by the +// upstream ORDER BY score node. Mirrors bm25's streamInto. +func (s *Segment) streamWAND(clauses []clause, algo ScoreAlgo, gs *globalStats, allow Membership, sink *streamSink) { + avgDocLen := gs.avgdl(s) + iters := s.buildWandIters(clauses, algo, gs, avgDocLen) + defer releaseWandIters(iters) // recycle the per-cursor block buffers + if len(iters) == 0 { + return + } + for !sink.stopped { + minDoc := int64(math.MaxInt64) + for _, it := range iters { + if d := it.doc(); d < minDoc { + minDoc = d + } + } + if minDoc == math.MaxInt64 { + break // all cursors exhausted + } + if allowed(allow, minDoc) { + var score float32 + for _, it := range iters { + if it.doc() == minDoc { + // tf-aware, same as searchWAND / evalClause (fulltext2 keeps real tf). + score += it.weight * s.scoreTerm(algo, float64(it.tf()), it.idf2, minDoc, avgDocLen) + } + } + sink.pushPk(s, minDoc, score) + } + for _, it := range iters { + if it.doc() == minDoc { + it.idx++ + it.refresh() // keep the cached doc() in sync with idx + } + } + } +} + +// StreamQuery answers a no-LIMIT MATCH by streaming every matching (pk, score) to emit +// in bounded batches instead of materializing the whole result set (the ranking is +// done by the upstream ORDER BY score node). Every shape streams ONE SEGMENT AT A TIME +// with peak Go heap O(one segment), never O(globalN): a pure disjunction of single terms +// (the ranked-retrieval shape) streams heap-free via streamWAND; an NL phrase via +// streamPhrase; and a full boolean with MUST/MUST-NOT/mixed-phrase via streamBoolean +// (each segment's dense evalBoolean, freed before the next). All share the same sink — a +// uniform emit interface, and no separate paginated copy in the caller. +func (idx *Index) StreamQuery(pattern []byte, boolean bool, parser string, algo ScoreAlgo, filter *prefilter, wantInclude bool, emit func(out *vectorindex.SearchOutput) error) error { + if idx.globalN == 0 { + return nil + } + sink := newStreamSink(idx, wantInclude, filter, emit) + p := normalizeParser(parser) + + // Resolve the query the same way SearchQuery does, but prefer the streaming + // disjunctive path when the resolved query is a pure OR of single terms. + var q BoolQuery + var haveQ bool + pat := string(pattern) + if boolean { + var err error + if q, err = buildBooleanQuery(pat, p); err != nil { + return err + } + haveQ = true + } + // NL mode (haveQ stays false) is an exact positional phrase, not a pure disjunction, + // so it falls through to the materializing SearchQuery below (same phrase path). + + if haveQ { + if terms, ok := disjunctiveTerms(q); ok { + return idx.streamDisjunction(terms, algo, filter, sink) + } + } else { + // NL mode = exact positional phrase. Stream it ONE SEGMENT AT A TIME (peak = + // one segment's phrase matches, freed before the next) instead of the old + // SearchQuery(globalN) fallback, whose bounded top-K sized to globalN + // materialized the whole live corpus for a low-selectivity phrase. + slots, err := phraseSlots(pat, p) + if err != nil { + return err + } + return idx.streamPhrase(slots, algo, filter, sink) + } + + // Boolean, non-disjunctive (MUST / MUST-NOT / mixed phrase): stream each live match + // through the sink one segment at a time via evalBoolean. Peak Go heap is O(one + // segment) (bounded by its capacity), NOT O(globalN) — so a low-selectivity boolean + // (e.g. +common on millions of rows) no longer materializes the whole live corpus as + // a top-K sized to globalN before emitting. Ranking is the upstream ORDER BY node's. + return idx.streamBoolean(q, algo, filter, sink) +} + +// streamPhrase emits every live phrase match (pk, score) through sink WITHOUT a global +// top-K heap or a globalN result slice — the streaming analogue of SearchPhrase for the +// no-LIMIT NL path. A phrase has ONE idf² (a positive constant, so it does not change +// rank but does set the absolute score to match the LIMIT path), which needs the total +// live df, so this makes two cheap passes: pass 1 counts the live phrase df; pass 2 +// scores + streams. Peak Go heap is O(one segment's matchPhrase hits) (freed before the +// next segment), NOT O(globalN). Mirrors SearchPhrase's scoring (partial × idf²) exactly. +func (idx *Index) streamPhrase(slots []phraseSlot, algo ScoreAlgo, filter *prefilter, sink *streamSink) error { + if idx.globalN == 0 || len(slots) == 0 { + return nil + } + // Pass 1: distinct live phrase df (filter-independent, matching SearchPhrase's df). + df := 0 + for si, seg := range idx.segments { + for _, h := range seg.matchPhrase(slots) { + if idx.isLive(si, h.ord) { + df++ + } + } + } + if df == 0 { + return nil + } + idf2 := float32(idfSquared(idx.globalN, df)) + + // Pass 2: score + stream each live, WHERE-admitted match in doc order per segment. + for si, seg := range idx.segments { + allow := mkAllow(seg, filter) + for _, h := range seg.matchPhrase(slots) { + if !idx.isLive(si, h.ord) { + continue + } + if allowed(allow, h.ord) { + partial := seg.scoreTerm(algo, float64(h.tf), 1.0, h.ord, idx.globalAvgDocLen) + sink.pushAny(seg, h.ord, seg.pk(h.ord), partial*idf2) + if sink.err != nil { + return sink.err + } + } + } + } + sink.flush() + return sink.err +} + +// StreamBagOfWords is the no-LIMIT IN BM25 MODE path: it tokenizes the pattern into a +// pure disjunction of its tokens (bm25-style bag-of-words) and streams the ranked hits +// heap-free via streamWAND — the position-free analogue of StreamQuery's disjunctive +// branch, but a CJK run is tokenized to OR terms instead of a positional phrase, so it +// works on a POSITION_FREE index. +func (idx *Index) StreamBagOfWords(pattern []byte, parser string, algo ScoreAlgo, filter *prefilter, wantInclude bool, emit func(out *vectorindex.SearchOutput) error) error { + if idx.globalN == 0 { + return nil + } + q, err := buildBagOfWordsQuery(string(pattern), normalizeParser(parser)) + if err != nil { + return err + } + terms, ok := disjunctiveTerms(q) + if !ok { + return nil // no resolvable tokens → nothing to stream + } + return idx.streamDisjunction(terms, algo, filter, newStreamSink(idx, wantInclude, filter, emit)) +} + +// streamDisjunction streams a pure OR of terms heap-free across all segments (global +// corpus stats + per-segment liveness), pushing every (pk, score) into sink and +// flushing at the end. Shared by StreamQuery's boolean disjunctive branch and +// StreamBagOfWords' IN BM25 MODE path — a change to the disjunctive walk lands once. +func (idx *Index) streamDisjunction(terms []clause, algo ScoreAlgo, filter *prefilter, sink *streamSink) error { + gs := idx.newGlobalStats() + for si, seg := range idx.segments { + allow := andAllow(mkAllow(seg, filter), &livenessMembership{idx: idx, si: si}) + seg.streamWAND(terms, algo, gs, allow, sink) + if sink.err != nil { + return sink.err + } + } + sink.flush() + return sink.err +} diff --git a/pkg/fulltext2/stream_phrase_test.go b/pkg/fulltext2/stream_phrase_test.go new file mode 100644 index 0000000000000..dd5942f030b84 --- /dev/null +++ b/pkg/fulltext2/stream_phrase_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// TestStreamPhraseParity pins that the no-LIMIT streaming phrase path (streamPhrase) +// returns EXACTLY the same pk set and scores as the materializing SearchPhrase (the +// LIMIT path) — same idf²-scaled score, only without the globalN heap. +func TestStreamPhraseParity(t *testing.T) { + bb := NewBuilder("p", int32(types.T_int64)) + feed(t, bb, int64(1), "alpha", "beta") // phrase match + feed(t, bb, int64(2), "alpha", "beta") // phrase match + feed(t, bb, int64(3), "x", "alpha", "beta") // phrase match (offset) + feed(t, bb, int64(4), "beta", "alpha") // NOT adjacent as "alpha beta" + feed(t, bb, int64(5), "alpha", "gamma") // no beta + seg := loadedSeg(t, bb) + idx := NewIndex([]*Segment{seg}, nil) + + slots, err := phraseSlots("alpha beta", ParserDefault) + require.NoError(t, err) + + // materialized reference (LIMIT path, k above corpus so it returns every match). + want := idx.SearchPhrase(slots, BM25, 1<<20, nil) + wantScore := make(map[int64]float32, len(want)) + for _, r := range want { + wantScore[r.Pk.(int64)] = r.Score + } + require.ElementsMatch(t, []int64{1, 2, 3}, keysOf(wantScore)) + + // streamed (no-LIMIT) — decode the box-free int64 ColumnBuffer batches. + got := make(map[int64]float32) + emit := func(o *vectorindex.SearchOutput) error { + for i := 0; i < o.Keys.N; i++ { + pk := int64(binary.LittleEndian.Uint64(o.Keys.Data[i*8:])) + got[pk] = o.Dists[i] + } + PutColumnBuffer(o.Keys) + return nil + } + require.NoError(t, idx.StreamQuery([]byte("alpha beta"), false, ParserDefault, BM25, nil, false, emit)) + + // same pk set, same idf²-scaled scores. + require.Equal(t, len(wantScore), len(got)) + for pk, ws := range wantScore { + gs, ok := got[pk] + require.Truef(t, ok, "streamed result missing pk %d", pk) + require.InEpsilonf(t, ws, gs, 1e-6, "score mismatch for pk %d", pk) + } +} + +func keysOf(m map[int64]float32) []int64 { + out := make([]int64, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func streamPhraseIDs(t *testing.T, idx *Index, pattern string) map[int64]float32 { + t.Helper() + got := make(map[int64]float32) + emit := func(o *vectorindex.SearchOutput) error { + for i := 0; i < o.Keys.N; i++ { + got[int64(binary.LittleEndian.Uint64(o.Keys.Data[i*8:]))] = o.Dists[i] + } + PutColumnBuffer(o.Keys) + return nil + } + require.NoError(t, idx.StreamQuery([]byte(pattern), false, ParserDefault, BM25, nil, false, emit)) + return got +} + +// TestFillBlockPositionsReuseMultiBlock stresses the Phase-1 pooled-buffer reuse across +// MULTIPLE blocks (>BlockSize docs) with varying per-doc position counts: the phrase +// term repeats a different number of times per doc, so each reused out[i] slot changes +// length block to block. A stale/aliased backing array would surface as a wrong match +// set. Runs twice so a pooled buffer is re-Got from the pool between queries. +func TestFillBlockPositionsReuseMultiBlock(t *testing.T) { + const nDocs = 3 * BlockSize // spans several position blocks + bb := NewBuilder("mb", int32(types.T_int64)) + wantMatch := map[int64]bool{} + for d := 0; d < nDocs; d++ { + pk := int64(d) + reps := 1 + d%5 // varying position-list length per doc + var words []string + for r := 0; r < reps; r++ { + words = append(words, "alpha", "beta") + } + if d%3 == 0 { // a third have "beta alpha" appended → still contains the phrase "alpha beta" + words = append(words, "beta", "alpha") + } + if d%7 == 0 { // some docs break the phrase entirely (no adjacent alpha->beta) + words = []string{"beta", "alpha", "gamma"} + wantMatch[pk] = false + } else { + wantMatch[pk] = true + } + feed(t, bb, pk, words...) + } + seg := loadedSeg(t, bb) + idx := NewIndex([]*Segment{seg}, nil) + + slots, err := phraseSlots("alpha beta", ParserDefault) + require.NoError(t, err) + + run := func() { + res := idx.SearchPhrase(slots, BM25, 1<<20, nil) + got := map[int64]bool{} + for _, r := range res { + got[r.Pk.(int64)] = true + } + for pk, want := range wantMatch { + require.Equalf(t, want, got[pk], "pk %d match", pk) + } + } + run() // first pass populates the pool + run() // second pass re-Gets a reused buffer — must be identical +} + +// TestStreamPhraseParityMultiSegment: streamPhrase across a base + CDC-tail (with a +// delete making one base copy dead) must match SearchPhrase — same live set, same +// idf²-scaled scores — proving the multi-segment / liveness path streams correctly. +func TestStreamPhraseParityMultiSegment(t *testing.T) { + base := NewBuilder("base", int32(types.T_int64)) + feed(t, base, int64(1), "alpha", "beta") + feed(t, base, int64(2), "alpha", "beta") + feed(t, base, int64(3), "beta", "alpha") // no phrase + baseSeg := loadedSeg(t, base) + baseSeg.Recency = 0 + + tail := NewBuilder("tail", int32(types.T_int64)) + feed(t, tail, int64(4), "alpha", "beta") + tailSeg := loadedSeg(t, tail) + tailSeg.Recency = 10 + + // delete pk 2 (its base phrase-match copy must drop out of both paths). + deletes := foldDeleteFrame(nil, []DeleteRecord{{Pk: int64(2)}}, 20) + idx := NewIndex([]*Segment{baseSeg, tailSeg}, deletes) + + slots, err := phraseSlots("alpha beta", ParserDefault) + require.NoError(t, err) + want := idx.SearchPhrase(slots, BM25, 1<<20, nil) + wantScore := make(map[int64]float32, len(want)) + for _, r := range want { + wantScore[r.Pk.(int64)] = r.Score + } + require.ElementsMatch(t, []int64{1, 4}, keysOf(wantScore)) // 2 deleted, 3 no-phrase + + got := streamPhraseIDs(t, idx, "alpha beta") + require.Equal(t, len(wantScore), len(got)) + for pk, ws := range wantScore { + require.InEpsilonf(t, ws, got[pk], 1e-6, "score mismatch pk %d", pk) + } +} diff --git a/pkg/fulltext2/stream_test.go b/pkg/fulltext2/stream_test.go new file mode 100644 index 0000000000000..8bfa9b0d96ff9 --- /dev/null +++ b/pkg/fulltext2/stream_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "encoding/binary" + "errors" + "testing" + + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// int64ColumnBuffer decodes an int64-pk ColumnBuffer batch (Fixed, width 8) back to []int64. +func int64ColumnBuffer(k *vectorindex.ColumnBuffer) []int64 { + out := make([]int64, k.N) + for i := 0; i < k.N; i++ { + out[i] = int64(binary.LittleEndian.Uint64(k.Data[i*8:])) + } + return out +} + +// TestStreamQuery pins the no-LIMIT streaming path: StreamQuery must emit EXACTLY the +// same (pk, score) set the materialized SearchQuery(k=all) produces — for both the +// heap-free disjunctive path (streamWAND) and the materialize-then-emit fallback (NL +// phrase) — and it must stop and propagate an emit error. Uses a base + CDC-tail split +// so streaming exercises the per-segment liveness + global stats. +func TestStreamQuery(t *testing.T) { + split := NewIndex([]*Segment{ + buildFtSeg(t, "a", 0, 0, 12), + buildFtSeg(t, "b", 1, 12, 24), + }, nil) + + collect := func(pattern string, boolean bool) map[int64]float64 { + m := map[int64]float64{} + err := split.StreamQuery([]byte(pattern), boolean, ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { + require.LessOrEqual(t, o.Keys.N, streamBatch) + for i, k := range int64ColumnBuffer(o.Keys) { + _, dup := m[k] + require.Falsef(t, dup, "pk %d emitted twice", k) + m[k] = float64(o.Dists[i]) + } + return nil + }) + require.NoError(t, err) + return m + } + sameAsMaterialized := func(pattern string, boolean bool) { + want, err := split.SearchQuery([]byte(pattern), boolean, ParserDefault, BM25, 1000, nil) + require.NoError(t, err) + got := collect(pattern, boolean) + require.Equalf(t, len(want), len(got), "%q: same match count", pattern) + for _, r := range want { + g, ok := got[r.Pk.(int64)] + require.Truef(t, ok, "%q: pk %v streamed", pattern, r.Pk) + require.InDeltaf(t, r.Score, g, 1e-5, "%q: pk %v score", pattern, r.Pk) + } + } + + sameAsMaterialized("alpha beta gamma", true) // disjunctive → streamWAND + sameAsMaterialized("beta delta", true) // disjunctive → streamWAND + sameAsMaterialized("alpha", false) // NL single-word → materialize fallback + + // emit-error path: the callback error propagates and stops the walk. + sentinel := errors.New("consumer aborted") + err := split.StreamQuery([]byte("alpha beta"), true, ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { return sentinel }) + require.ErrorIs(t, err, sentinel) +} diff --git a/pkg/fulltext2/stream_vec_test.go b/pkg/fulltext2/stream_vec_test.go new file mode 100644 index 0000000000000..29e6c8afb9a95 --- /dev/null +++ b/pkg/fulltext2/stream_vec_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/stretchr/testify/require" +) + +// TestColumnBufferRoundTrip pins the box-free typed streaming codec: a pk encoded into a +// ColumnBuffer (producer, appendAnyPk) must decode back to the identical value in a vector +// of that type (consumer, AppendColumnBuffer), across the fixed-width and varlena paths. +func TestColumnBufferRoundTrip(t *testing.T) { + mp := mpool.MustNewZero() + + build := func(pkType types.T, vals []any) *vectorindex.ColumnBuffer { + w, fixed := fixedPkByteWidth(int32(pkType)) + if !fixed { + w = 0 + } + k := &vectorindex.ColumnBuffer{Type: pkType} + for _, v := range vals { + appendAnyPk(k, w, v) + } + require.Equal(t, len(vals), k.N) + return k + } + toVec := func(pkType types.T, vals []any) *vector.Vector { + vec := vector.NewVec(pkType.ToType()) + require.NoError(t, vectorindex.AppendColumnBuffer(build(pkType, vals), vec, mp)) + require.Equal(t, len(vals), vec.Length()) + return vec + } + + // fixed-width paths + require.Equal(t, []int64{1, -5, 1 << 40}, + vector.MustFixedColWithTypeCheck[int64](toVec(types.T_int64, []any{int64(1), int64(-5), int64(1 << 40)}))) + require.Equal(t, []uint64{0, 1, 1<<64 - 1}, + vector.MustFixedColWithTypeCheck[uint64](toVec(types.T_uint64, []any{uint64(0), uint64(1), uint64(1<<64 - 1)}))) + require.Equal(t, []uint64{7, 255}, + vector.MustFixedColWithTypeCheck[uint64](toVec(types.T_bit, []any{uint64(7), uint64(255)}))) + require.Equal(t, []int32{-3, 100000}, + vector.MustFixedColWithTypeCheck[int32](toVec(types.T_int32, []any{int32(-3), int32(100000)}))) + require.Equal(t, []int8{-1, 2}, + vector.MustFixedColWithTypeCheck[int8](toVec(types.T_int8, []any{int8(-1), int8(2)}))) + require.Equal(t, []types.Datetime{types.Datetime(123456789), types.Datetime(-42)}, + vector.MustFixedColWithTypeCheck[types.Datetime](toVec(types.T_datetime, []any{types.Datetime(123456789), types.Datetime(-42)}))) + require.Equal(t, []types.Decimal128{{B0_63: 5, B64_127: 9}}, + vector.MustFixedColWithTypeCheck[types.Decimal128](toVec(types.T_decimal128, []any{types.Decimal128{B0_63: 5, B64_127: 9}}))) + + // varlena paths + vvar := toVec(types.T_varchar, []any{[]byte("abc"), []byte(""), []byte("a longer varchar pk")}) + require.Equal(t, "abc", vvar.GetStringAt(0)) + require.Equal(t, "", vvar.GetStringAt(1)) + require.Equal(t, "a longer varchar pk", vvar.GetStringAt(2)) + + u1, _ := types.ParseUuid("00000000-0000-0000-0000-000000000001") + u2, _ := types.ParseUuid("12345678-1234-1234-1234-1234567890ab") + require.Equal(t, []types.Uuid{u1, u2}, + vector.MustFixedColWithTypeCheck[types.Uuid](toVec(types.T_uuid, []any{u1, u2}))) + + // remaining AppendColumnBuffer arms: decode without error at the right count, so every + // switch case is exercised (a missing case would panic or mis-count). + more := []struct { + typ types.T + vals []any + }{ + {types.T_uint32, []any{uint32(1), uint32(4000000000)}}, + {types.T_int16, []any{int16(-30000), int16(30000)}}, + {types.T_uint16, []any{uint16(60000)}}, + {types.T_uint8, []any{uint8(200)}}, + {types.T_date, []any{types.Date(19000)}}, + {types.T_time, []any{types.Time(123456)}}, + {types.T_timestamp, []any{types.Timestamp(1700000000)}}, + {types.T_decimal64, []any{types.Decimal64(42)}}, + {types.T_char, []any{[]byte("c")}}, + {types.T_text, []any{[]byte("longer text pk value")}}, + {types.T_blob, []any{[]byte{0, 1, 2, 255}}}, + {types.T_json, []any{[]byte(`{"k":1}`)}}, + } + for _, c := range more { + require.Equalf(t, len(c.vals), toVec(c.typ, c.vals).Length(), "type %v", c.typ) + } +} + +// TestStreamLoadedSegmentAllPkTypes covers the LOADED-segment streaming path (appendPkTo +// reading the docmap, not the build-side re-encode) plus StreamBagOfWords, across a +// fixed-width, a varlena, and a uuid pk (the encodePkLen default branch). It serializes → +// deserializes each segment, streams every doc, and checks the emitted pks via the +// pooled ColumnBuffer → vector path (AppendColumnBuffer + PutColumnBuffer). +func TestStreamLoadedSegmentAllPkTypes(t *testing.T) { + mp := mpool.MustNewZero() + uu, _ := types.ParseUuid("12345678-1234-1234-1234-1234567890ab") + + stream := func(pkType types.T, pks []any) *vector.Vector { + b := NewBuilder("ld", int32(pkType)) + for _, pk := range pks { + feed(t, b, pk, "alpha") + } + seg, err := b.Finish() + require.NoError(t, err) + blob, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("ld", bytes.NewReader(blob)) + require.NoError(t, err) + t.Cleanup(func() { _ = loaded.dict.Close() }) + + idx := NewIndex([]*Segment{loaded}, nil) + out := vector.NewVec(pkType.ToType()) + err = idx.StreamBagOfWords([]byte("alpha"), ParserDefault, BM25, nil, false, + func(o *vectorindex.SearchOutput) error { + e := vectorindex.AppendColumnBuffer(o.Keys, out, mp) + PutColumnBuffer(o.Keys) // recycle, as the real consumer does + return e + }) + require.NoError(t, err) + require.Equal(t, len(pks), out.Length()) + return out + } + + // int64 pk → fixed-width loaded path + require.ElementsMatch(t, []int64{10, 20, 30}, + vector.MustFixedColWithTypeCheck[int64](stream(types.T_int64, []any{int64(10), int64(20), int64(30)}))) + + // varchar pk → varlena loaded path + vv := stream(types.T_varchar, []any{[]byte("k1"), []byte("k2")}) + got := []string{vv.GetStringAt(0), vv.GetStringAt(1)} + require.ElementsMatch(t, []string{"k1", "k2"}, got) + + // uuid pk → encodePkLen default branch + varlena loaded path + require.ElementsMatch(t, []types.Uuid{uu}, + vector.MustFixedColWithTypeCheck[types.Uuid](stream(types.T_uuid, []any{uu}))) +} + +// TestStreamBagOfWordsEdges covers the two early-out branches of StreamBagOfWords: an +// empty index (globalN==0) and a pattern that resolves to no tokens. +func TestStreamBagOfWordsEdges(t *testing.T) { + noEmit := func(*vectorindex.SearchOutput) error { + t.Fatal("emit should not be called") + return nil + } + // empty index → globalN == 0 → nil, no emit. + require.NoError(t, NewIndex(nil, nil).StreamBagOfWords([]byte("alpha"), ParserDefault, BM25, nil, false, noEmit)) + + // non-empty index but a pattern that tokenizes to nothing → no resolvable tokens → nil. + b := NewBuilder("e", int32(types.T_int64)) + feed(t, b, int64(1), "alpha") + seg, err := b.Finish() + require.NoError(t, err) + require.NoError(t, NewIndex([]*Segment{seg}, nil).StreamBagOfWords([]byte(" "), ParserDefault, BM25, nil, false, noEmit)) +} diff --git a/pkg/fulltext2/tail_pool.go b/pkg/fulltext2/tail_pool.go new file mode 100644 index 0000000000000..c7090fc577354 --- /dev/null +++ b/pkg/fulltext2/tail_pool.go @@ -0,0 +1,458 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "sync" + "time" + + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +const ( + tailPoolMaxEntries = 256 + tailPoolMaxBytes = int64(1 << 30) + tailPoolIdleTTL = 15 * time.Minute +) + +type tailState struct { + baseGeneration int64 + maxChunk int64 + segments []*segmentLease + deletes map[any]int64 + bytes int64 + lastUsed time.Time +} + +type tailPool struct { + mu sync.Mutex + states map[string]*tailState + totalBytes int64 + maxEntries int + maxBytes int64 + idleTTL time.Duration +} + +var loadedTailPool = newTailPool() + +func newTailPool() *tailPool { + return &tailPool{ + states: make(map[string]*tailState), + maxEntries: tailPoolMaxEntries, + maxBytes: tailPoolMaxBytes, + idleTTL: tailPoolIdleTTL, + } +} + +func (p *tailPool) limits() (int, int64, time.Duration) { + maxEntries, maxBytes, idleTTL := p.maxEntries, p.maxBytes, p.idleTTL + if maxEntries <= 0 { + maxEntries = tailPoolMaxEntries + } + if maxBytes <= 0 { + maxBytes = tailPoolMaxBytes + } + if idleTTL <= 0 { + idleTTL = tailPoolIdleTTL + } + return maxEntries, maxBytes, idleTTL +} + +// estimateSegmentBytes is intentionally conservative. Tail segments are +// build-side Go objects, so their exact allocator footprint is runtime-specific; +// this estimate still gives the pool a deterministic byte bound instead of +// treating a state containing millions of postings as one small map entry. +func estimateSegmentBytes(s *Segment) int64 { + if s == nil { + return 0 + } + n := s.ownedBytes + if n == 0 { + n = int64(len(s.pks))*24 + int64(len(s.pkOffsets))*4 + int64(len(s.pkRaw)) + + int64(len(s.docLen))*4 + int64(len(s.includeTypes))*4 + int64(len(s.includeRaw)) + + int64(len(s.includeVarOffsets))*4 + } else { + // pkRaw, includeRaw, the FST, ranking, blocks, and positions are all + // views into ownedBytes and must not be counted again. + n += int64(len(s.pkOffsets))*4 + int64(len(s.docLen))*4 + + int64(len(s.includeTypes))*4 + int64(len(s.includeVarOffsets))*4 + } + for _, values := range s.includeVals { + n += int64(len(values)) * 16 + } + for word, postings := range s.terms { + n += int64(len(word)) + 64 + if postings == nil { + continue + } + n += int64(len(postings.docIDs))*8 + int64(len(postings.tfs))*1 + + int64(len(postings.positions))*24 + for _, positions := range postings.positions { + n += int64(len(positions)) * 4 + } + } + return n +} + +func estimateDeleteKeyBytes(key any) int64 { + const mapEntryOverhead = int64(32) + switch v := key.(type) { + case string: + return mapEntryOverhead + int64(len(v)) + case []byte: + return mapEntryOverhead + int64(len(v)) + default: + return mapEntryOverhead + } +} + +func estimateTailStateBytes(segs []*Segment, deletes map[any]int64) int64 { + n := int64(len(segs)) * 8 + for _, seg := range segs { + n += estimateSegmentBytes(seg) + } + for key := range deletes { + n += estimateDeleteKeyBytes(key) + } + return n +} + +func retireTailStates(states []*tailState) { + for _, state := range states { + for _, lease := range state.segments { + lease.retire() + } + } +} + +func (p *tailPool) evictLocked(now time.Time) (retire []*tailState) { + _, _, idleTTL := p.limits() + remove := func(index string, state *tailState) { + delete(p.states, index) + p.totalBytes -= state.bytes + if p.totalBytes < 0 { + p.totalBytes = 0 + } + retire = append(retire, state) + } + for index, state := range p.states { + if state.lastUsed.IsZero() { + continue + } + if now.Sub(state.lastUsed) >= idleTTL { + remove(index, state) + } + } + maxEntries, maxBytes, _ := p.limits() + for len(p.states) > maxEntries || p.totalBytes > maxBytes { + var oldestIndex string + var oldest *tailState + for index, state := range p.states { + if oldest == nil || state.lastUsed.Before(oldest.lastUsed) { + oldestIndex, oldest = index, state + } + } + if oldest == nil { + break + } + remove(oldestIndex, oldest) + } + return retire +} + +func (p *tailPool) evict(now time.Time) { + p.mu.Lock() + retire := p.evictLocked(now) + p.mu.Unlock() + retireTailStates(retire) +} + +func cloneDeletes(src map[any]int64) map[any]int64 { + if len(src) == 0 { + return nil + } + dst := make(map[any]int64, len(src)) + for k, v := range src { + dst[k] = v + } + return dst +} + +func mergeTailDeletes(base, delta map[any]int64) map[any]int64 { + merged := cloneDeletes(base) + if merged == nil && len(delta) > 0 { + merged = make(map[any]int64, len(delta)) + } + for k, v := range delta { + if cur, ok := merged[k]; !ok || v > cur { + merged[k] = v + } + } + return merged +} + +func (p *tailPool) state(index string) *tailState { + p.mu.Lock() + retire := p.evictLocked(time.Now()) + state := p.states[index] + if state != nil { + state.lastUsed = time.Now() + } + p.mu.Unlock() + retireTailStates(retire) + return state +} + +func (p *tailPool) acquireViews(index string, expected *tailState) ([]*Segment, map[any]int64, bool) { + p.mu.Lock() + retire := p.evictLocked(time.Now()) + if p.states[index] != expected { + p.mu.Unlock() + retireTailStates(retire) + return nil, nil, false + } + expected.lastUsed = time.Now() + segs, deletes := acquireTailViews(expected) + p.mu.Unlock() + retireTailStates(retire) + return segs, deletes, true +} + +func (p *tailPool) current(index string, expected *tailState) bool { + p.mu.Lock() + ok := p.states[index] == expected + if ok { + expected.lastUsed = time.Now() + } + p.mu.Unlock() + return ok +} + +func (p *tailPool) installDeltaAndAcquire(index string, expected, state *tailState) ([]*Segment, map[any]int64, bool) { + return p.installDeltaAndAcquireIfCurrent(index, expected, state, nil) +} + +func (p *tailPool) installDeltaAndAcquireIfCurrent(index string, expected, state *tailState, current func() bool) ([]*Segment, map[any]int64, bool) { + p.mu.Lock() + retire := p.evictLocked(time.Now()) + if current != nil && !current() { + p.mu.Unlock() + retireTailStates(retire) + return nil, nil, false + } + if p.states[index] != expected { + p.mu.Unlock() + retireTailStates(retire) + return nil, nil, false + } + state.lastUsed = time.Now() + p.totalBytes -= expected.bytes + p.states[index] = state + p.totalBytes += state.bytes + segs, deletes := acquireTailViews(state) + retire = append(retire, p.evictLocked(time.Now())...) + p.mu.Unlock() + retireTailStates(retire) + return segs, deletes, true +} + +func (p *tailPool) installAndAcquire(index string, state *tailState) ([]*Segment, map[any]int64) { + segs, deletes, _ := p.installAndAcquireIfCurrent(index, state, nil) + return segs, deletes +} + +func (p *tailPool) installAndAcquireIfCurrent(index string, state *tailState, current func() bool) ([]*Segment, map[any]int64, bool) { + p.mu.Lock() + retire := p.evictLocked(time.Now()) + if current != nil && !current() { + p.mu.Unlock() + retireTailStates(retire) + return nil, nil, false + } + old := p.states[index] + if old != nil { + p.totalBytes -= old.bytes + } + state.lastUsed = time.Now() + p.states[index] = state + p.totalBytes += state.bytes + segs, deletes := acquireTailViews(state) + retire = append(retire, p.evictLocked(time.Now())...) + p.mu.Unlock() + if old == nil || old == state { + retireTailStates(retire) + return segs, deletes, true + } + // Appending a delta transfers old segment leases into the new state. A full + // reset (MERGE/REBUILD or tail regression) has no shared leases to transfer. + if len(state.segments) > 0 && len(old.segments) > 0 && state.segments[0] == old.segments[0] { + retireTailStates(retire) + return segs, deletes, true + } + retire = append(retire, old) + retireTailStates(retire) + return segs, deletes, true +} + +func (p *tailPool) clear(index string) { + p.mu.Lock() + old := p.states[index] + delete(p.states, index) + if old != nil { + p.totalBytes -= old.bytes + if p.totalBytes < 0 { + p.totalBytes = 0 + } + } + p.mu.Unlock() + if old != nil { + retireTailStates([]*tailState{old}) + } +} + +func (p *tailPool) clearAll() { + p.mu.Lock() + states := make([]*tailState, 0, len(p.states)) + for index, state := range p.states { + delete(p.states, index) + states = append(states, state) + } + p.totalBytes = 0 + p.mu.Unlock() + retireTailStates(states) +} + +func acquireTailViews(state *tailState) ([]*Segment, map[any]int64) { + if state == nil { + return nil, nil + } + segs := make([]*Segment, 0, len(state.segments)) + for _, lease := range state.segments { + if view := lease.acquire(0); view != nil { + segs = append(segs, view) + } + } + return segs, cloneDeletes(state.deletes) +} + +func newTailState(baseGeneration, maxChunk int64, segs []*Segment, deletes map[any]int64) *tailState { + leases := make([]*segmentLease, 0, len(segs)) + for _, seg := range segs { + leases = append(leases, newSegmentLease(seg)) + } + state := &tailState{ + baseGeneration: baseGeneration, + maxChunk: maxChunk, + segments: leases, + deletes: cloneDeletes(deletes), + lastUsed: time.Now(), + } + state.bytes = estimateTailStateBytes(segs, state.deletes) + return state +} + +// loadTailWithReuse reuses the prior tail generation when the durable tail +// maximum is unchanged, or reads only chunks after the prior maximum when it +// advanced. A base generation change or tail regression forces a full tail +// reload and retires the old tail leases after active readers release them. +func loadTailWithReuse(sqlproc *sqlexec.SqlProcess, cfg TableConfig, baseGeneration, tailMax int64, trace *loadTrace) ([]*Segment, map[any]int64, int64, error) { + return loadTailWithReuseForGeneration(sqlproc, cfg, baseGeneration, tailMax, trace, loadGeneration{}) +} + +func loadTailWithReuseForGeneration(sqlproc *sqlexec.SqlProcess, cfg TableConfig, baseGeneration, tailMax int64, trace *loadTrace, generation loadGeneration) ([]*Segment, map[any]int64, int64, error) { + index := cfg.DbName + "." + cfg.IndexTable + current := func() bool { + return generation.attempt == 0 || loadGenerationCurrent(generation) + } + old := loadedTailPool.state(index) + if old != nil && old.baseGeneration == baseGeneration && tailMax == old.maxChunk { + segs, deletes, ok := loadedTailPool.acquireViews(index, old) + if ok { + return segs, deletes, old.maxChunk, nil + } + old = loadedTailPool.state(index) + } + + after := int64(-1) + if old != nil && old.baseGeneration == baseGeneration && tailMax > old.maxChunk { + if loadedTailPool.current(index, old) { + after = old.maxChunk + } else { + old = nil + } + } + newSegs, newDeletes, appliedMax, err := loadTailSegmentsAfter(sqlproc, cfg, after, trace) + if err != nil { + if after < 0 { + return nil, nil, 0, err + } + // A delta can begin in the middle of a multi-chunk frame when the + // previous snapshot ended before the frame became visible. Retry the + // complete tail so the frame header/prefix is available; persistent + // corruption still fails on this second pass. + newSegs, newDeletes, appliedMax, err = loadTailSegmentsAfter(sqlproc, cfg, -1, trace) + if err != nil { + return nil, nil, 0, err + } + old = nil + after = -1 + } + if appliedMax < tailMax { + // Keep the durable generation observed by the same load snapshot even + // when that snapshot has no tail rows to materialize. + appliedMax = tailMax + } + state := newTailState(baseGeneration, appliedMax, newSegs, newDeletes) + if after >= 0 && old != nil { + // Transfer old leases into the new state and append only the new frames. + state.segments = append(append([]*segmentLease(nil), old.segments...), state.segments...) + state.deletes = mergeTailDeletes(old.deletes, newDeletes) + state.bytes = estimateTailStateBytes(nil, state.deletes) + for _, lease := range state.segments { + lease.mu.Lock() + template := lease.template + lease.mu.Unlock() + state.bytes += estimateSegmentBytes(template) + } + } + if after >= 0 && old != nil { + segs, deletes, ok := loadedTailPool.installDeltaAndAcquireIfCurrent(index, old, state, current) + if ok { + return segs, deletes, appliedMax, nil + } + // Another invalidation replaced the state while the delta was being + // read. Do not publish a delta-only state without its predecessor. + for _, lease := range state.segments[len(old.segments):] { + lease.retire() + } + if !current() { + return nil, nil, 0, errLoadGenerationSuperseded + } + newSegs, newDeletes, appliedMax, err = loadTailSegmentsAfter(sqlproc, cfg, -1, trace) + if err != nil { + return nil, nil, 0, err + } + if appliedMax < tailMax { + appliedMax = tailMax + } + state = newTailState(baseGeneration, appliedMax, newSegs, newDeletes) + } + segs, deletes, ok := loadedTailPool.installAndAcquireIfCurrent(index, state, current) + if !ok { + retireTailStates([]*tailState{state}) + return nil, nil, 0, errLoadGenerationSuperseded + } + return segs, deletes, appliedMax, nil +} diff --git a/pkg/fulltext2/tail_pool_test.go b/pkg/fulltext2/tail_pool_test.go new file mode 100644 index 0000000000000..329ab21a36ecc --- /dev/null +++ b/pkg/fulltext2/tail_pool_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func TestTailPoolTransfersOnlyCurrentState(t *testing.T) { + p := &tailPool{states: make(map[string]*tailState)} + first := newTailState(1, 10, []*Segment{NewSegment("tail-10", 0)}, map[any]int64{"old": 10}) + second := newTailState(1, 11, []*Segment{NewSegment("tail-11", 0)}, map[any]int64{"old": 10, "new": 11}) + index := "db.store" + p.mu.Lock() + p.states[index] = first + p.mu.Unlock() + + second.segments = append(append([]*segmentLease(nil), first.segments...), second.segments...) + views, deletes, ok := p.installDeltaAndAcquire(index, first, second) + require.True(t, ok) + require.Len(t, views, 2) + require.Equal(t, int64(10), deletes["old"]) + require.Equal(t, int64(11), deletes["new"]) + for _, view := range views { + view.Free() + } + + p.clear(index) + require.Nil(t, first.segments[0].template) + require.Nil(t, second.segments[1].template) +} + +func TestTailPoolRejectsStaleDeltaInstall(t *testing.T) { + p := &tailPool{states: make(map[string]*tailState)} + index := "db.store" + first := newTailState(1, 10, []*Segment{NewSegment("tail-10", 0)}, nil) + other := newTailState(1, 12, []*Segment{NewSegment("tail-12", 0)}, nil) + state := newTailState(1, 11, []*Segment{NewSegment("tail-11", 0)}, nil) + state.segments = append(append([]*segmentLease(nil), first.segments...), state.segments...) + p.mu.Lock() + p.states[index] = other + p.mu.Unlock() + + views, _, ok := p.installDeltaAndAcquire(index, first, state) + require.False(t, ok) + require.Nil(t, views) + for _, lease := range state.segments[1:] { + lease.retire() + } + p.clear(index) +} + +func TestMergeTailDeletesInitializesInsertOnlyDelta(t *testing.T) { + merged := mergeTailDeletes(nil, map[any]int64{"new": 11}) + require.Equal(t, map[any]int64{"new": int64(11)}, merged) +} + +func TestTailPoolBoundsIndexesAndEvictsIdle(t *testing.T) { + p := &tailPool{ + states: make(map[string]*tailState), + maxEntries: 1, + maxBytes: 1 << 20, + idleTTL: time.Hour, + } + for i, index := range []string{"db.one", "db.two"} { + state := newTailState(1, int64(i), []*Segment{NewSegment(index, 0)}, nil) + views, _ := p.installAndAcquire(index, state) + for _, view := range views { + view.Free() + } + } + require.LessOrEqual(t, len(p.states), 1) + + p.idleTTL = time.Nanosecond + p.evict(time.Now().Add(time.Second)) + require.Empty(t, p.states) + p.clearAll() +} + +func TestTailPoolByteBoundCountsDeserializedArchiveAndDeletePayload(t *testing.T) { + b := NewBuilder("tail", int32(types.T_int64)) + words := make([]string, 100) + for i := range words { + words[i] = "repeated" + } + for doc := int64(0); doc < 100; doc++ { + feed(t, b, doc, words...) + } + seg, err := b.Finish() + require.NoError(t, err) + archive, err := seg.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("tail", bytes.NewReader(archive)) + require.NoError(t, err) + t.Cleanup(func() { + _ = loaded.dict.Close() + loaded.Free() + }) + + require.Equal(t, int64(len(archive)), loaded.ownedBytes) + deleteKey := strings.Repeat("varchar-pk-", 128) + state := newTailState(1, 1, []*Segment{loaded}, map[any]int64{deleteKey: 1}) + require.GreaterOrEqual(t, state.bytes, int64(len(archive))) + require.GreaterOrEqual(t, state.bytes, int64(len(archive)+len(deleteKey)+32)) + state.segments[0].retire() +} diff --git a/pkg/fulltext2/tailbuild.go b/pkg/fulltext2/tailbuild.go new file mode 100644 index 0000000000000..68964cafa2309 --- /dev/null +++ b/pkg/fulltext2/tailbuild.go @@ -0,0 +1,279 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "fmt" + "os" + "path/filepath" +) + +// defaultTailCapacity floors the streaming segment cap when a non-positive +// capacity is passed (the sinker sources it from max_index_capacity, default 1M). +const defaultTailCapacity int64 = DefaultBuildCapacity + +// TailSegment locates one sealed CDC frame inside a PACKED spool file: Path is the spool file, +// Offset is the frame's start byte within it, and FrameLen its length. Many frames share one spool +// file (rolled at DefaultTailSpoolRollBytes), so a burst of tiny CDC frames costs a handful of files +// instead of one file per frame; persist reads exactly [Offset, Offset+FrameLen) via load_file. +type TailSegment struct { + Path string + Offset int64 + FrameLen int +} + +// TailBuilder streams CDC insert/upsert rows into capacity-capped positional segments and DELETEs +// into capacity-capped pk-only frames, spilling each sealed frame's bytes to a temp file the moment +// it fills — so the sinker's peak memory is ONE open segment's postings plus one open delete run, +// never the whole CDC stream. +// +// Frames are sealed, and returned by Finish, in CHRONOLOGICAL order (the order op-runs arrive), so +// chunk_id — assigned sequentially at persist and used as the segment recency — reflects operation +// order. This gives last-writer-wins across CDC blobs WITHOUT any unbounded per-session set: a +// later DELETE frame outranks an earlier INSERT segment of the same pk (delete wins), and a +// re-INSERT after a DELETE outranks it (re-insert wins). Switching op type seals the currently-open +// structure first (see AddBatch), which is what preserves that order; the producer flushes a new +// blob per op change, so this seals at blob granularity, not per event. Mirrors bm25's TailBuilder +// (positional Builder here). NOT safe for concurrent use; Cleanup() must be deferred to remove the +// temp files. +// +// Each sealed frame is APPENDED to a rolling packed spool file (rolled at rollCap), not written to +// its own file — so an alternating delete/insert stream (which seals one frame per op run) costs a +// bounded number of files, O(total frame bytes / rollCap), instead of one file per op run (which +// could reach hundreds of thousands over a 30-minute iteration and exhaust inodes). The frame +// ORDER, count, and per-frame chunk_id are unchanged — only the file storage is packed — so +// last-writer-wins recency is byte-for-byte identical to a file-per-frame layout. +type TailBuilder struct { + pkType int32 + capacity int64 // doc cap (max_index_capacity) + postingCap int64 // posting cap (max_postings_capacity); seal on whichever is hit first + tokenize func(string) []WordPos + dir string + seq int + delSeq int + cur *Builder // open INSERT segment (nil between insert runs) + delBatch []DeleteRecord // open DELETE run (sealed on an op switch to insert, at capacity, or at Finish) + frames []TailSegment // sealed frames (insert segs + delete frames) in chronological (recency) order + + // Packed spool: sealed frames are appended to curFile at curOff until it would exceed rollCap, + // then a fresh file is opened. curFile is closed on roll and in Finish/Cleanup so persist's + // load_file reads flushed bytes. fileSeq names the spool files. + rollCap int64 + curFile *os.File + curPath string + curOff int64 + fileSeq int + + opts []BuildOpt // carried into each per-segment Builder (e.g. WithPositionFree) + includeTypes []int32 // INCLUDE column schema, adopted from the first CDC batch +} + +// NewTailBuilder creates a streaming tail builder backed by a private temp dir under +// spillDir (the LOCAL fileservice's __fulltext2 dir; "" ⇒ the OS temp dir, for tests / +// no-fileservice callers). Pass WithPositionFree() so a position-free index's CDC tail +// stays position-free. capacity is max_index_capacity (docs), postingCap is +// max_postings_capacity (postings); each non-positive value falls back to its default. +func NewTailBuilder(pkType int32, capacity, postingCap int64, spillDir string, tokenize func(string) []WordPos, opts ...BuildOpt) (*TailBuilder, error) { + if capacity < 1 { + capacity = defaultTailCapacity + } + if postingCap < 1 { + postingCap = DefaultPostingCapacity + } + dir, err := os.MkdirTemp(spillDir, "ftv2tail") + if err != nil { + return nil, err + } + return &TailBuilder{pkType: pkType, capacity: capacity, postingCap: postingCap, tokenize: tokenize, dir: dir, rollCap: DefaultTailSpoolRollBytes, opts: opts}, nil +} + +// DefaultTailSpoolRollBytes caps one packed spool file; appending a frame that would exceed it +// starts a fresh file (a single frame larger than the cap gets its own file). Bounds a temp file's +// size while packing many small CDC frames into few files instead of one file per frame. +const DefaultTailSpoolRollBytes int64 = 128 << 20 // 128 MiB + +// writeFrame appends framed to the current rolling spool file — starting a new file when the +// current one would exceed rollCap — and returns the file path plus the frame's byte offset within +// it, so persist can address exactly [offset, offset+len) via load_file. A frame is never split +// across files, so load_file always sees a contiguous range. +func (t *TailBuilder) writeFrame(framed []byte) (string, int64, error) { + if t.curFile != nil && t.curOff > 0 && t.curOff+int64(len(framed)) > t.rollCap { + if err := t.curFile.Close(); err != nil { + return "", 0, err + } + t.curFile = nil + } + if t.curFile == nil { + path := filepath.Join(t.dir, fmt.Sprintf("spool-%d.frames", t.fileSeq)) + t.fileSeq++ + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return "", 0, err + } + t.curFile, t.curPath, t.curOff = f, path, 0 + } + off := t.curOff + if _, err := t.curFile.Write(framed); err != nil { + return "", 0, err + } + t.curOff += int64(len(framed)) + return t.curPath, off, nil +} + +// AddBatch streams one decoded CDC batch: insert/upsert rows are tokenized into +// the open segment (sealed + spilled once it reaches `capacity` docs), deletes are +// collected. Same tokenizer as the search side, so build/query tokens match. +func (t *TailBuilder) AddBatch(cdc *Cdc) error { + // The CDC blob is self-describing: adopt its INCLUDE schema once so every tail segment + // this builder seals carries the docmap include section (WithIncludeTypes). A copy of + // t.opts avoids mutating the caller's slice. + if t.includeTypes == nil && len(cdc.IncludeTypes) > 0 { + t.includeTypes = cdc.IncludeTypes + t.opts = append(append([]BuildOpt(nil), t.opts...), WithIncludeTypes(t.includeTypes)) + } + // Process events IN ORDER, sealing at each op-type SWITCH so frames are produced — and later + // returned by Finish — in chronological order. Recency (chunk_id) then follows operation order + // across CDC blobs, giving last-writer-wins: switching to an insert seals the open delete run + // first (so a re-insert outranks the prior delete of that pk); switching to a delete seals the + // open insert segment first (so the delete outranks the prior insert). The producer flushes a + // new blob per op change, so this seals at blob granularity, not per event, and both open + // structures stay bounded to one capacity. + for i := range cdc.Events { + e := &cdc.Events[i] + switch e.Op { + case cdcInsert, cdcUpsert: + words := t.tokenize(e.Text) + // An INSERT of empty / zero-token text has nothing to index and no prior version to + // shadow, so skip it (matches the base build, which drops empty docs). An UPSERT of + // empty / NULL text MUST still emit a doc so it shadows the old indexed version — + // SetDoc writes a live 0-term doc for that. + if len(words) == 0 && e.Op == cdcInsert { + continue + } + if len(t.delBatch) > 0 { // op switch delete->insert: seal the delete run first (lower recency) + if err := t.sealDeletes(); err != nil { + return err + } + } + if t.cur == nil { + t.cur = NewBuilder(fmt.Sprintf("cdctail-%d", t.seq), t.pkType, t.opts...) + } + // REPLACE, not append: a repeated upsert of one pk within this open segment supersedes + // its earlier terms AND INCLUDE values (Add would merge). + t.cur.SetDoc(e.Pk, words, e.Include) + if ReachedSegmentCap(t.cur, t.capacity, t.postingCap) { + if err := t.seal(); err != nil { + return err + } + } + case cdcDelete: + if t.cur != nil { // op switch insert->delete: seal the insert segment first (lower recency) + if err := t.seal(); err != nil { + return err + } + } + t.delBatch = append(t.delBatch, DeleteRecord{Pk: e.Pk}) + if int64(len(t.delBatch)) >= t.capacity { + if err := t.sealDeletes(); err != nil { + return err + } + } + } + } + return nil +} + +// seal finalizes the open segment, frames it, and spills the framed bytes to a +// temp file. No-op with no open segment; an all-empty segment is dropped. +func (t *TailBuilder) seal() error { + if t.cur == nil { + return nil + } + seg, err := t.cur.Finish() + t.cur = nil + if err != nil { + return err + } + if seg.N == 0 { + return nil + } + framed, err := FrameSegment(seg) + if err != nil { + return err + } + t.seq++ + path, off, err := t.writeFrame(framed) + if err != nil { + return err + } + t.frames = append(t.frames, TailSegment{Path: path, Offset: off, FrameLen: len(framed)}) + return nil +} + +// sealDeletes frames the OPEN delete run and spills it to a temp file, appending it to the +// chronological frame list. Called on an op switch to insert, when the run reaches capacity, and +// at Finish — so tombstones stay bounded to one capacity's worth in the heap (never accumulate +// for the whole session). No-op when the run is empty. +func (t *TailBuilder) sealDeletes() error { + if len(t.delBatch) == 0 { + return nil + } + framed, err := FrameDeletes(t.pkType, t.delBatch) + if err != nil { + return err + } + t.delBatch = nil // free the run; the next delete re-grows a fresh slice + t.delSeq++ + path, off, err := t.writeFrame(framed) + if err != nil { + return err + } + t.frames = append(t.frames, TailSegment{Path: path, Offset: off, FrameLen: len(framed)}) + return nil +} + +// Finish seals the final open insert segment and delete run (in that order — at most one is open, +// since an op switch already sealed the other), CLOSES the current spool file so its bytes are +// flushed for load_file, then returns ALL frames in chronological (recency) order. chunk_id is +// assigned sequentially at persist, so recency = operation order and the last writer wins across +// CDC blobs (see AddBatch / the type doc). Each frame is a [Offset, Offset+FrameLen) range in its +// packed spool file, which the caller persists via load_file. Cleanup must be called afterwards. +func (t *TailBuilder) Finish() ([]TailSegment, error) { + if err := t.seal(); err != nil { + return nil, err + } + if err := t.sealDeletes(); err != nil { + return nil, err + } + if t.curFile != nil { + if err := t.curFile.Close(); err != nil { + return nil, err + } + t.curFile = nil + } + return t.frames, nil +} + +// Cleanup closes any open spool file and removes the temp dir with all packed spool files. Idempotent. +func (t *TailBuilder) Cleanup() { + if t.curFile != nil { + t.curFile.Close() + t.curFile = nil + } + if t.dir != "" { + os.RemoveAll(t.dir) + t.dir = "" + } +} diff --git a/pkg/fulltext2/termdict.go b/pkg/fulltext2/termdict.go new file mode 100644 index 0000000000000..1291c2e1a2a52 --- /dev/null +++ b/pkg/fulltext2/termdict.go @@ -0,0 +1,164 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// termDict is the on-disk / loaded term dictionary of a segment: a vellum FST +// mapping each indexed term to a uint64 value (the term's posting-offset; df is +// read from the posting-list header). It is the compact, sorted realization of +// the backbone described in doc.go — the build side accumulates terms in a Go +// map (Segment.terms), then buildTermDictFST encodes the sorted keys into this +// FST for storage and query. +// +// vellum is byte-oriented, and UTF-8 byte order equals Unicode code-point order, +// so terms sorted with Go's ordinary string sort (Segment.sortedTerms) are in +// exactly the ascending byte order vellum's builder requires — Chinese / CJK +// terms need no special handling, and prefix iteration works on whole-token +// (character-boundary) prefixes. +type termDict struct { + fst *vellum.FST + data []byte // backing bytes; vellum.Load references (does not copy) this +} + +// buildTermDictFST encodes ascending (term → value) pairs into vellum FST bytes. +// terms must be in ascending byte order (Segment.sortedTerms) and unique, and +// len(values) must equal len(terms); values[i] is term[i]'s posting-offset. +func buildTermDictFST(terms []string, values []uint64) ([]byte, error) { + var buf bytes.Buffer + b, err := vellum.New(&buf, nil) + if err != nil { + return nil, err + } + for i, t := range terms { + if err = b.Insert([]byte(t), values[i]); err != nil { + return nil, err + } + } + if err = b.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// loadTermDict wraps FST bytes (as produced by buildTermDictFST) for query. The +// returned dict references data directly, so data must outlive the dict; Close +// releases the FST. +func loadTermDict(data []byte) (*termDict, error) { + fst, err := vellum.Load(data) + if err != nil { + return nil, err + } + return &termDict{fst: fst, data: data}, nil +} + +// get returns the value (posting-offset) for an exact term, or ok=false if the +// term is absent. +func (d *termDict) get(term string) (val uint64, ok bool, err error) { + return d.fst.Get([]byte(term)) +} + +// len returns the number of terms in the dictionary. +func (d *termDict) len() int { return d.fst.Len() } + +// prefixIter returns an ascending iterator over every term that begins with +// prefix (the `word*` enumeration). An empty prefix iterates the whole dict. The +// caller must Close the iterator. A nil iterator with ok=false means the prefix +// matched nothing. +func (d *termDict) prefixIter(prefix string) (it *vellum.FSTIterator, ok bool, err error) { + start := []byte(prefix) + it, err = d.fst.Iterator(start, prefixSuccessor(start)) + if err == vellum.ErrIteratorDone { + return nil, false, nil // no term with this prefix + } + if err != nil { + return nil, false, err + } + return it, true, nil +} + +// prefixSuccessor returns the smallest byte string strictly greater than every +// string having prefix as a prefix — i.e. the exclusive upper bound for a prefix +// range scan: copy the prefix, increment the last byte that is < 0xFF, and drop +// the trailing 0xFF bytes. Returns nil (an open upper bound → iterate to the end +// of the FST) when prefix is empty or all 0xFF, both of which have no finite +// successor. +func prefixSuccessor(prefix []byte) []byte { + end := make([]byte, len(prefix)) + copy(end, prefix) + for i := len(end) - 1; i >= 0; i-- { + if end[i] < 0xFF { + end[i]++ + return end[:i+1] + } + } + return nil +} + +// prefixTerms collects every term with the given prefix into an ascending slice +// (the `word*` expansion, materialized). Empty prefix returns all terms. +func (d *termDict) prefixTerms(prefix string) ([]string, error) { + it, ok, err := d.prefixIter(prefix) + if err != nil || !ok { + return nil, err + } + defer func() { _ = it.Close() }() + var out []string + for { + term, _ := it.Current() + out = append(out, string(term)) + if err := it.Next(); err == vellum.ErrIteratorDone { + break + } else if err != nil { + return nil, err + } + } + return out, nil +} + +// forEachTerm streams every term (ascending) through fn WITHOUT materializing the whole +// vocabulary as a []string the way prefixTerms("") does — so a large high-cardinality +// index's MERGE reconstruction (forEachPosting) doesn't spike O(vocabulary) of strings. +func (d *termDict) forEachTerm(fn func(term string) error) error { + it, ok, err := d.prefixIter("") + if err != nil || !ok { + return err + } + defer func() { _ = it.Close() }() + for { + term, _ := it.Current() + if err := fn(string(term)); err != nil { + return err + } + if err := it.Next(); err == vellum.ErrIteratorDone { + break + } else if err != nil { + return err + } + } + return nil +} + +// Close releases the FST. Safe on a nil dict. +func (d *termDict) Close() error { + if d == nil || d.fst == nil { + return nil + } + return d.fst.Close() +} diff --git a/pkg/fulltext2/termdict_jieba_test.go b/pkg/fulltext2/termdict_jieba_test.go new file mode 100644 index 0000000000000..6aac31cd3f529 --- /dev/null +++ b/pkg/fulltext2/termdict_jieba_test.go @@ -0,0 +1,131 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Large-scale term-dict test: build the vellum FST from the REAL gojieba +// dictionary (~349k Chinese/mixed words) and exercise exact lookup + `word*` +// prefix enumeration at scale, proving UTF-8 byte order == code-point order (so +// CJK prefix ranges are correct) on production vocabulary rather than a toy set. +package fulltext2 + +import ( + "bufio" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// readJiebaWords loads the unique words (first column) of jieba.dict.utf8, or +// skips the test if the dictionary is not available in this environment. +func readJiebaWords(t *testing.T) []string { + t.Helper() + dir := tokenizer.JiebaDictDir() + if dir == "" { + t.Skip("jieba dict dir not found") + } + f, err := os.Open(filepath.Join(dir, "jieba.dict.utf8")) + if err != nil { + t.Skipf("cannot open jieba dict: %v", err) + } + defer f.Close() + + seen := make(map[string]struct{}) + var terms []string + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) == 0 { + continue + } + w := fields[0] + if _, dup := seen[w]; dup { + continue + } + seen[w] = struct{}{} + terms = append(terms, w) + } + require.NoError(t, sc.Err()) + return terms +} + +// TestTermDictFromJiebaDict builds the FST from the whole jieba vocabulary and +// checks exact get() + prefixTerms() over it. +func TestTermDictFromJiebaDict(t *testing.T) { + terms := readJiebaWords(t) + require.Greater(t, len(terms), 100000, "real jieba dict has ~349k words") + + // The FST requires ascending byte order + unique keys (the Segment invariant). + sort.Strings(terms) + values := make([]uint64, len(terms)) + want := make(map[string]uint64, len(terms)) + for i, w := range terms { + values[i] = uint64(i) + want[w] = uint64(i) + } + + data, err := buildTermDictFST(terms, values) + require.NoError(t, err) + d, err := loadTermDict(data) + require.NoError(t, err) + defer d.Close() + + require.Equal(t, len(terms), d.len()) + + // Exact lookups for well-known dictionary words return their assigned value. + for _, w := range []string{"中文", "北京", "清华大学", "苹果", "香蕉", "天安门"} { + exp, in := want[w] + if !in { + continue // tolerate dict variations across environments + } + val, ok, gerr := d.get(w) + require.NoError(t, gerr) + require.True(t, ok, "%q must be present", w) + require.Equal(t, exp, val, "value mismatch for %q", w) + } + + // A word that cannot be in the dictionary is absent. + _, ok, err := d.get("绝对不存在的怪词ZzQq") + require.NoError(t, err) + require.False(t, ok) + + // Prefix enumeration: every 中文* term begins with 中文, is a real dict word, + // and the results are ascending — the CJK `word*` path over real vocabulary. + pref, err := d.prefixTerms("中文") + require.NoError(t, err) + require.NotEmpty(t, pref) + require.Contains(t, pref, "中文") + require.True(t, sort.StringsAreSorted(pref), "prefix results must be ascending") + seen := make(map[string]struct{}, len(terms)) + for _, w := range terms { + seen[w] = struct{}{} + } + for _, w := range pref { + require.True(t, strings.HasPrefix(w, "中文"), "%q lacks prefix 中文", w) + _, in := seen[w] + require.True(t, in, "%q not in source vocabulary", w) + } + + // The prefix scan found the same count as a brute-force filter (no misses). + brute := 0 + for _, w := range terms { + if strings.HasPrefix(w, "中文") { + brute++ + } + } + require.Equal(t, brute, len(pref)) +} diff --git a/pkg/fulltext2/termdict_test.go b/pkg/fulltext2/termdict_test.go new file mode 100644 index 0000000000000..371a1433642df --- /dev/null +++ b/pkg/fulltext2/termdict_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "sort" + "testing" + + "github.com/blevesearch/vellum" + "github.com/stretchr/testify/require" +) + +// buildDict sorts the given terms (as the segment does), assigns each a distinct +// value, and builds + loads an FST term dict. Returns the dict and the expected +// term→value map. Terms must be unique. +func buildDict(t *testing.T, terms ...string) (*termDict, map[string]uint64) { + t.Helper() + sorted := append([]string(nil), terms...) + sort.Strings(sorted) + vals := make([]uint64, len(sorted)) + exp := make(map[string]uint64, len(sorted)) + for i, term := range sorted { + vals[i] = uint64(i*100 + 7) // distinct, non-trivial + exp[term] = vals[i] + } + data, err := buildTermDictFST(sorted, vals) + require.NoError(t, err) + d, err := loadTermDict(data) + require.NoError(t, err) + t.Cleanup(func() { _ = d.Close() }) + return d, exp +} + +// collectPrefix drains prefixIter into a slice of terms (ascending). +func collectPrefix(t *testing.T, d *termDict, prefix string) []string { + t.Helper() + it, ok, err := d.prefixIter(prefix) + require.NoError(t, err) + if !ok { + return nil + } + defer func() { _ = it.Close() }() + var got []string + for { + term, _ := it.Current() + got = append(got, string(term)) + if err := it.Next(); err == vellum.ErrIteratorDone { + break + } else { + require.NoError(t, err) + } + } + return got +} + +func TestTermDictGet(t *testing.T) { + d, exp := buildDict(t, "apple", "apply", "banana", "band") + require.Equal(t, 4, d.len()) + for term, want := range exp { + got, ok, err := d.get(term) + require.NoError(t, err) + require.True(t, ok, term) + require.Equal(t, want, got, term) + } + // misses + for _, miss := range []string{"", "app", "avocado", "bandit", "z"} { + _, ok, err := d.get(miss) + require.NoError(t, err) + require.False(t, ok, miss) + } +} + +func TestTermDictPrefixASCII(t *testing.T) { + d, _ := buildDict(t, "app", "apple", "apply", "banana", "band", "bat") + + require.Equal(t, []string{"app", "apple", "apply"}, collectPrefix(t, d, "app")) + require.Equal(t, []string{"apple", "apply"}, collectPrefix(t, d, "appl")) + require.Equal(t, []string{"banana", "band"}, collectPrefix(t, d, "ban")) + require.Equal(t, []string{"bat"}, collectPrefix(t, d, "bat")) + require.Equal(t, []string{"banana", "band", "bat"}, collectPrefix(t, d, "b")) + // empty prefix → all terms, ascending + require.Equal(t, []string{"app", "apple", "apply", "banana", "band", "bat"}, collectPrefix(t, d, "")) + // no matches — before, between, after + require.Empty(t, collectPrefix(t, d, "aa")) + require.Empty(t, collectPrefix(t, d, "az")) + require.Empty(t, collectPrefix(t, d, "z")) +} + +// TestTermDictChinese is the load-bearing case: CJK terms need no special +// handling because vellum is byte-oriented and UTF-8 byte order == code-point +// order, so Go's string sort is already vellum's required insertion order, and +// prefix iteration works on whole-character prefixes. +func TestTermDictChinese(t *testing.T) { + // 中(U+4E2D) 国(U+56FD) 文(U+6587) — bigrams share a leading character. + d, exp := buildDict(t, "中", "中国", "中文", "国", "文", "国家") + require.Equal(t, 6, d.len()) + + // exact lookup of a multi-byte term + got, ok, err := d.get("中文") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, exp["中文"], got) + + // prefix over a shared leading character → the two bigrams + the unigram + require.Equal(t, []string{"中", "中国", "中文"}, collectPrefix(t, d, "中")) + require.Equal(t, []string{"国", "国家"}, collectPrefix(t, d, "国")) + require.Equal(t, []string{"文"}, collectPrefix(t, d, "文")) + // full ascending order matches byte/code-point order + require.Equal(t, []string{"中", "中国", "中文", "国", "国家", "文"}, collectPrefix(t, d, "")) + require.Empty(t, collectPrefix(t, d, "日")) +} + +func TestPrefixSuccessor(t *testing.T) { + require.Equal(t, []byte{'a', 'p', 'q'}, prefixSuccessor([]byte("app"))) + require.Equal(t, []byte{0x01}, prefixSuccessor([]byte{0x00})) + // trailing 0xFF bytes are dropped, the first < 0xFF is incremented + require.Equal(t, []byte{0x02}, prefixSuccessor([]byte{0x01, 0xFF, 0xFF})) + // empty prefix and all-0xFF have no finite successor → nil (open upper bound) + require.Nil(t, prefixSuccessor([]byte{})) + require.Nil(t, prefixSuccessor([]byte{0xFF, 0xFF})) +} diff --git a/pkg/fulltext2/topk_bench_test.go b/pkg/fulltext2/topk_bench_test.go new file mode 100644 index 0000000000000..062610606408f --- /dev/null +++ b/pkg/fulltext2/topk_bench_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math/rand" + "testing" +) + +func benchResults(n, buckets int) []Result { + rng := rand.New(rand.NewSource(7)) + rs := make([]Result, n) + for i := range rs { + rs[i] = Result{Pk: int64(rng.Intn(1 << 30)), Score: float32(rng.Intn(buckets))} + } + return rs +} + +// n=45688 k=10 is the observed 'the' whole-corpus stopword shape. The FastMaxHeap +// selection is O(n) with zero per-candidate allocation regardless of the score +// distribution — Distinct (varied BM25 scores) and Tied (all-identical) both stay flat, +// unlike the old bounded-min-heap that allocated a sortKey string per candidate. +func BenchmarkTopK_Distinct(b *testing.B) { + rs := benchResults(45688, 45688) + b.ResetTimer() + for i := 0; i < b.N; i++ { + topKResults(rs, 10) + } +} +func BenchmarkTopK_Tied(b *testing.B) { + rs := benchResults(45688, 1) + b.ResetTimer() + for i := 0; i < b.N; i++ { + topKResults(rs, 10) + } +} diff --git a/pkg/fulltext2/topk_test.go b/pkg/fulltext2/topk_test.go new file mode 100644 index 0000000000000..c14ac8fe4a280 --- /dev/null +++ b/pkg/fulltext2/topk_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math" + "math/rand" + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestTopKResultsValidTopK fuzzes topKResults over many sizes, k values, and score +// distributions (including heavy ties). The contract is deliberately NOT a byte-for-byte +// match of a full sort — equal scores are equally relevant and their order is +// unspecified. It IS: (1) length == min(k, n); (2) output ordered by score descending; +// (3) a *valid* top-k — no returned doc scores below any doc that was left out. That is +// exactly "correct top-k by score, arbitrary tie-break". +func TestTopKResultsValidTopK(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for _, scoreBuckets := range []int{1, 2, 5, 50, 100000} { + for _, n := range []int{0, 1, 2, 3, 7, 8, 9, 33, 200, 1000} { + for _, k := range []int{0, 1, 2, 8, 10, n - 1, n, n + 5} { + results := make([]Result, n) + for i := range results { + results[i] = Result{ + Pk: int64(rng.Intn(1 << 30)), + Score: float32(rng.Intn(scoreBuckets)), + } + } + got := topKResults(results, k) + + wantLen := k + if wantLen > n { + wantLen = n + } + if k <= 0 { + wantLen = 0 + } + require.Lenf(t, got, wantLen, "n=%d k=%d buckets=%d", n, k, scoreBuckets) + + // (2) ordered by score descending. + for i := 1; i < len(got); i++ { + require.GreaterOrEqualf(t, got[i-1].Score, got[i].Score, + "not score-desc at %d (n=%d k=%d buckets=%d)", i, n, k, scoreBuckets) + } + if len(got) == 0 { + continue + } + // (3) valid top-k: the minimum returned score is >= the k-th largest + // score overall, i.e. nothing better than a returned doc was dropped. + allScores := make([]float64, n) + for i := range results { + allScores[i] = float64(results[i].Score) + } + sort.Sort(sort.Reverse(sort.Float64Slice(allScores))) + kth := allScores[len(got)-1] // k-th largest overall (got has min(k,n) items) + minReturned := math.Inf(1) + for _, r := range got { + if float64(r.Score) < minReturned { + minReturned = float64(r.Score) + } + } + require.GreaterOrEqualf(t, minReturned, kth, + "dropped a higher-scoring doc (n=%d k=%d buckets=%d)", n, k, scoreBuckets) + } + } + } +} diff --git a/pkg/fulltext2/wand.go b/pkg/fulltext2/wand.go new file mode 100644 index 0000000000000..79cf63fe1ccee --- /dev/null +++ b/pkg/fulltext2/wand.go @@ -0,0 +1,398 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// WAND (Weak-AND, Broder et al. 2003) disjunctive top-k for a pure OR of single +// terms. Each term posting iterator carries a max-impact upper bound (its +// largest possible weighted contribution); the algorithm keeps a top-k min-heap +// whose smallest score is the threshold θ, and skips any document whose summed +// upper bounds cannot reach θ — so most low-scoring documents are never scored. +// +// The result is the SAME top-k the full scan produces — the same set of documents +// and the same score for each; WAND only avoids work. It layers Block-Max WAND +// (Ding & Suel 2011) on top of classic term-level WAND: besides each term's +// whole-list max-impact, the per-block bounds (blockMaxTf/blockMinDocLn over +// BlockSize-doc blocks, derived by deriveTermStats) give a tighter score UB for the +// block covering the pivot, so a whole block region whose summed block-UB can't +// reach θ is skipped without scoring any doc in it. Block-skip only removes docs +// whose score UB is ≤ θ — which the `score > θ` insertion rule would reject anyway. +// +// Tie caveat: the ORDER of documents that share an equal score is unspecified (like +// bm25, which documents "ties arbitrary"). WAND sums a doc's term contributions in +// cursor order and the full scan in clause-map order, so the two can differ in the +// last float ULP and sort exactly-tied docs differently; and at the k-th boundary a +// tie may include one arbitrary member over another. The top-k SET (above the +// boundary) and the score multiset are identical either way. + +// wandIter is a term's posting cursor for WAND. It holds the termPostings for the +// resident Block-Max bounds (blockLastDoc/blockMaxTf/blockMinDocLn), plus a small +// PER-CURSOR decoded-block cache: docIDs/tfs are NOT resident on a loaded segment +// (they live block-compressed in the shared, read-only mmap), so the cursor decodes +// one block at a time into bDocs/bTfs. The cache is per-cursor, so concurrent queries +// over the same shared segment never race. WAND's block-skip means most blocks are +// never decoded. +type wandIter struct { + tp *termPostings + idx int + curBlk int // block index currently decoded into bDocs/bTfs (-1 = none) + blen int // valid entries in bDocs/bTfs + cur int64 // cached doc() for the current idx (math.MaxInt64 when exhausted) + bDocs []int64 // decoded docIDs of curBlk (cap BlockSize); backed by buf + bTfs []uint8 // decoded tfs of curBlk (cap BlockSize); backed by buf + buf *wandBuf // pooled backing for bDocs/bTfs; returned by releaseWandIters + idf2 float64 + weight float32 + maxImpact float32 // term-level upper bound of this term's weighted contribution +} + +func (it *wandIter) atEnd() bool { return it.idx >= it.tp.df() } + +// ensure decodes the block containing the cursor's current idx into bDocs/bTfs, if +// not already cached. Cheap (a field compare) when the cursor stays within a block. +func (it *wandIter) ensure() { + b := it.idx / BlockSize + if b != it.curBlk { + it.blen = it.tp.fillBlock(b, it.bDocs, it.bTfs) + it.curBlk = b + } +} + +// doc returns the cached current doc ord (math.MaxInt64 when exhausted, so an +// exhausted cursor sorts last). It is read many times per pivot iteration +// (insertion sort, pivot scan, blockMax, chooseSkip, alignment) while the cursor +// moves at most once, so cur is recomputed only on move (refresh) — avoiding the +// atEnd() check, the idx/BlockSize division in ensure, and the modulo per read. +func (it *wandIter) doc() int64 { return it.cur } + +// refresh recomputes cur after idx changes (construction/advance/skipTo). +func (it *wandIter) refresh() { + if it.atEnd() { + it.cur = math.MaxInt64 + return + } + it.ensure() + it.cur = it.bDocs[it.idx%BlockSize] +} + +// tf returns the current posting's capped term frequency. +func (it *wandIter) tf() uint8 { + it.ensure() + return it.bTfs[it.idx%BlockSize] +} + +// skipTo advances the cursor to the first doc >= d: locate the block via the +// RESIDENT blockLastDoc (no decode), then binary-search within that one block. +func (it *wandIter) skipTo(d int64) { + b := it.blockIndexAt(d) + if b >= it.tp.nblk() { + it.idx = it.tp.df() // past the last posting → exhausted + it.cur = math.MaxInt64 + return + } + if b != it.curBlk { + it.blen = it.tp.fillBlock(b, it.bDocs, it.bTfs) + it.curBlk = b + } + // d <= blockLastDoc[b] = bDocs[blen-1], so the lower bound is within the block + // and never moves the cursor backward (d >= current doc). Inline lower-bound + // binary search rather than sort.Search: avoids the per-call closure and the + // generic sort.Search frame in this hot skip path. + lo, hi := 0, it.blen + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if it.bDocs[mid] < d { + lo = mid + 1 + } else { + hi = mid + } + } + it.idx = b*BlockSize + lo + it.cur = it.bDocs[lo] +} + +// blockIndexAt returns the index of the block (>= the current block) containing +// doc d — the first block whose last ord >= d. nblk() if d is past the cursor's +// last posting. Uses only the resident blockLastDoc (no block decode). +func (it *wandIter) blockIndexAt(d int64) int { + bl := it.tp.blockLastDoc + // blockLastDoc is ascending, so the first block with bl[b] >= d is a lower-bound + // binary search from the cursor's current block. The block-skip branch calls this + // per iters[0..pivot] per pivot step over a long posting list, where the old linear + // scan was O(nblk); binary search makes it O(log nblk) on the hot WAND loop. + lo := it.idx / BlockSize + hi := len(bl) + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if bl[mid] < d { + lo = mid + 1 + } else { + hi = mid + } + } + return lo +} + +// blockMax is the weighted Block-Max score upper bound for the block containing +// doc d, in the SAME scorer as termMaxImpact (so it is ≤ the term-level bound). 0 +// if d is past the list. blockMaxTf (max tf in block) and blockMinDocLn (min doc +// length in block) give the largest bm25Factor achievable in the block. +func (it *wandIter) blockMax(d int64, algo ScoreAlgo, avgDocLen float64) float32 { + b := it.blockIndexAt(d) + if b >= len(it.tp.blockLastDoc) { + return 0 + } + if algo == BM25 { + return it.weight * float32(it.idf2*bm25Factor(float64(it.tp.blockMaxTf[b]), it.tp.blockMinDocLn[b], avgDocLen)) + } + return it.weight * float32(float64(it.tp.blockMaxTf[b])*it.idf2) // TfIdf +} + +// blockEndAt is the last ord of the block containing doc d (the upper edge of the +// region for which blockMax(d) is a valid bound). math.MaxInt64 if past the list. +func (it *wandIter) blockEndAt(d int64) int64 { + b := it.blockIndexAt(d) + if b >= len(it.tp.blockLastDoc) { + return math.MaxInt64 + } + return it.tp.blockLastDoc[b] +} + +// chooseSkip picks a cursor in [0..pivot] whose curDoc < target (so it makes +// progress), preferring the largest term max-impact (skip the heaviest list). The +// block-skip / align callers guarantee at least one such cursor exists. +func chooseSkip(iters []*wandIter, pivot int, target int64) int { + best := -1 + var bestImpact float32 + for i := 0; i <= pivot; i++ { + if iters[i].doc() < target && (best < 0 || iters[i].maxImpact > bestImpact) { + best = i + bestImpact = iters[i].maxImpact + } + } + if best < 0 { + best = 0 // defensive; should not happen + } + return best +} + +// topKHeap is the bounded top-k-by-score heap shared by the WAND and full boolean +// evaluators: a SoA FastMaxHeap keyed by ord (int64) with distance = -score, so keeping +// the k SMALLEST distances keeps the k HIGHEST scores with zero per-candidate allocation +// and no interface dispatch. Its root distance is the current threshold θ (the k-th best +// score, negated). Equal scores are equally relevant — ties are unspecified. +type topKHeap = vectorindex.FastMaxHeap[float32, int64] + +// newTopKHeap allocates the bounded heap's backing buffers. k is capped to maxDocs (the +// segment's live doc count) because the heap can never hold more than that many results: +// FastMaxHeap needs pre-sized buffers, so an absurd pushed LIMIT (e.g. ORDER BY score +// LIMIT 5e8) would otherwise eagerly allocate 2×k slices (~GBs) and OOM the CN, even on a +// tiny table. Bounding to maxDocs is the FastMaxHeap analogue of bm25's lazily-grown topK. +func newTopKHeap(k int, maxDocs int64) *topKHeap { + if int64(k) > maxDocs { + k = int(maxDocs) + } + if k < 0 { + k = 0 + } + return vectorindex.NewFastMaxHeap[float32, int64](k, make([]int64, k), make([]float32, k)) +} + +// searchWAND runs WAND over the disjunctive single-term SHOULD clauses and +// returns the top-k, score desc (ties by ascending ord). allow (nil = allow all) is +// the WHERE-clause prefilter: a doc is admitted to the top-k only if its ord passes, +// so the LIMIT bounds the FILTERED set. Block-skip stays valid — blockSum is a score +// upper bound over every doc in the region regardless of the filter, so a skipped +// region contains no admissible doc that could beat θ. +// buildWandIters makes a posting cursor for each present SHOULD term, with its +// max-impact bound under the given (global or local) stats. Shared by the top-k +// searchWAND and the no-LIMIT streamWAND. +func (s *Segment) buildWandIters(clauses []clause, algo ScoreAlgo, gs *globalStats, avgDocLen float64) []*wandIter { + iters := make([]*wandIter, 0, len(clauses)) + for _, c := range clauses { + pl, ok := s.lookup(c.terms[0]) + if !ok || pl.df() == 0 { + continue + } + idf2 := gs.idfFor(s, c.terms[0], pl) + b := getWandBuf() + it := &wandIter{ + tp: pl, + curBlk: -1, + bDocs: b.docs, + bTfs: b.tfs, + buf: b, + idf2: idf2, + weight: c.weight, + maxImpact: c.weight * s.termMaxImpact(algo, idf2, pl, avgDocLen), + } + it.refresh() // prime cur for idx=0 + iters = append(iters, it) + } + return iters +} + +func (s *Segment) searchWAND(clauses []clause, algo ScoreAlgo, k int, allow Membership, gs *globalStats) []Result { + avgDocLen := gs.avgdl(s) + + iters := s.buildWandIters(clauses, algo, gs, avgDocLen) + defer releaseWandIters(iters) // recycle the per-cursor block buffers + if len(iters) == 0 { + return nil + } + + h := newTopKHeap(k, s.N) + theta := float32(math.Inf(-1)) // until the heap holds k, accept everything + + for { + // Order cursors by current doc ascending (exhausted ones, doc()==sentinel, + // sort last). Insertion sort, NOT sort.Slice: the iters are nearly sorted + // between iterations (only the skipped cursor moved), so this is O(len) + // here; more importantly sort.Slice boxes the slice into an interface{} + // (runtime.convTslice) and heap-allocates the less closure on every call — + // in this hot per-pivot loop that alloc churn (plus the reflect Swapper) + // dominated the entire query CPU. + for i := 1; i < len(iters); i++ { + it := iters[i] + di := it.doc() + j := i - 1 + for j >= 0 && iters[j].doc() > di { + iters[j+1] = iters[j] + j-- + } + iters[j+1] = it + } + if iters[0].atEnd() { + break // all exhausted + } + + // Term-level WAND pivot: first cursor at which the cumulative max-impact + // reaches θ. + acc, pivot := float32(0), -1 + for i, it := range iters { + if it.atEnd() { + break + } + acc += it.maxImpact + if acc >= theta { + pivot = i + break + } + } + if pivot < 0 { + break // no remaining document can reach the threshold + } + pivotDoc := iters[pivot].doc() + + // Extend the pivot over every cursor also sitting on pivotDoc, so the + // block-max sum + skip bounds account for ALL of pivotDoc's contributors (a + // cursor past the term-UB pivot can still be at pivotDoc). Required for the + // block-skip bound to be a valid upper bound. + for pivot+1 < len(iters) && iters[pivot+1].doc() == pivotDoc { + pivot++ + } + + // Block-Max refinement: the summed per-block upper bounds of iters[0..pivot] + // for the blocks covering pivotDoc bound every doc in [pivotDoc, minBlockEnd]. + // If that can't exceed θ, skip the whole region instead of scoring pivotDoc. + // Skipped docs have score ≤ blockSum ≤ θ, which `score > θ` rejects anyway → + // same top-k SET, less work. (bm25 uses the identical `blockSum <= theta`.) + blockSum := float32(0) + for i := 0; i <= pivot; i++ { + blockSum += iters[i].blockMax(pivotDoc, algo, avgDocLen) + } + if blockSum <= theta { + next := int64(math.MaxInt64) + for i := 0; i <= pivot; i++ { + if e := iters[i].blockEndAt(pivotDoc); e < next { + next = e + } + } + next++ // first doc beyond the limiting block + if pivot+1 < len(iters) { + if nd := iters[pivot+1].doc(); nd < next { + next = nd // never skip past an unaccounted contributor + } + } + if next <= pivotDoc { + next = pivotDoc + 1 // guarantee forward progress + } + iters[chooseSkip(iters, pivot, next)].skipTo(next) + continue + } + + if iters[0].doc() == pivotDoc { + // All cursors before the pivot are already on pivotDoc. Score + admit it + // only if it passes the WHERE prefilter; the cursor advance always runs so a + // filtered-out pivot never stalls the walk. + if allowed(allow, pivotDoc) { + var score float32 + for _, it := range iters { + if it.doc() == pivotDoc { + // tf-aware, matching the full-scan evalClause scorer so WAND returns the + // identical ranking (fulltext2 keeps real tf; see termMaxImpact). + score += it.weight * s.scoreTerm(algo, float64(it.tf()), it.idf2, pivotDoc, avgDocLen) + } + } + // FastMaxHeap.Push fills until full, then admits iff -score < root + // distance (i.e. score > θ) with a single sift — the same replace-root + // behaviour, done internally. θ is the k-th best score = negated root. + h.Push(pivotDoc, -score) + if h.Full() { + _, negRoot, _ := h.Peek() + theta = -negRoot + } + } + for _, it := range iters { + if it.doc() == pivotDoc { + it.idx++ + it.refresh() + } + } + } else { + // Not aligned: move a lagging cursor (before the pivot) up to pivotDoc. + iters[chooseSkip(iters, pivot, pivotDoc)].skipTo(pivotDoc) + } + } + + return heapToResults(s, h) +} + +// termMaxImpact is the largest weighted-free score a term can contribute to any +// document, from its raw maxTf (and minDocLen for BM25's saturating factor). +func (s *Segment) termMaxImpact(algo ScoreAlgo, idf2 float64, pl *termPostings, avgDocLen float64) float32 { + if algo == BM25 { + return float32(idf2 * bm25Factor(float64(pl.maxTf), pl.minDocLen, avgDocLen)) + } + return float32(float64(pl.maxTf) * idf2) // TfIdf +} + +// heapToResults drains the bounded top-k heap into results ordered by score descending +// (equal scores unspecified). FastMaxHeap.Pop yields the largest distance (= lowest +// score) first, so filling back-to-front lands the output in score-desc order. +func heapToResults(s *Segment, h *topKHeap) []Result { + n := h.Len() + results := make([]Result, n) + for i := n - 1; i >= 0; i-- { + ord, negScore, _ := h.Pop() + results[i] = Result{Pk: s.pk(ord), Score: -negScore, Include: s.decodeInclude(ord)} + } + return results +} diff --git a/pkg/fulltext2/wand_blockmax_test.go b/pkg/fulltext2/wand_blockmax_test.go new file mode 100644 index 0000000000000..b08c650fc56c6 --- /dev/null +++ b/pkg/fulltext2/wand_blockmax_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "fmt" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// blockMaxCorpus builds a MULTI-BLOCK segment (many BlockSize=128 blocks per +// frequent term) with a Zipfian-ish vocabulary and varied doc lengths, so the +// Block-Max skip path actually fires. Deterministic (fixed seed) — no test flake. +func blockMaxCorpus(t *testing.T, nDocs int) []TokenizedDoc { + t.Helper() + rng := rand.New(rand.NewSource(0x5eed)) + // 40 words; earlier words are far more frequent (Zipf), giving long postings + // that span many blocks with skewed per-block max-tf — the shape Block-Max + // prunes best. + vocab := make([]string, 40) + for i := range vocab { + vocab[i] = fmt.Sprintf("w%02d", i) + } + pick := func() string { + // Zipf-like: index ~ floor(40 * u^3) biases hard toward small indices. + u := rng.Float64() + idx := int(float64(len(vocab)) * u * u * u) + if idx >= len(vocab) { + idx = len(vocab) - 1 + } + return vocab[idx] + } + docs := make([]TokenizedDoc, nDocs) + for i := 0; i < nDocs; i++ { + n := 3 + rng.Intn(18) // doc length 3..20 + terms := make([]string, n) + for j := 0; j < n; j++ { + terms[j] = pick() + } + docs[i] = TokenizedDoc{Pk: int64(i), Terms: terms} + } + return docs +} + +// requireSameTopK asserts the Block-Max walk returns the SAME top-k as the full +// scan: identical count, identical score multiset (score-desc, within float tol), +// and — for every doc scoring strictly ABOVE the k-th (boundary) score — an +// identical pk set. The ORDER of exactly-tied docs is NOT asserted: WAND sums a +// doc's term contributions in cursor order and the full scan in clause-map order, +// so tied scores can differ in the last ULP and sort differently, and at the k-th +// boundary a tie is broken arbitrarily (as in bm25). Those are the only permitted +// differences; a dropped/extra above-boundary doc or a wrong score fails here. +func requireSameTopK(t *testing.T, got, want []Result, ctx string) { + t.Helper() + require.Equalf(t, len(want), len(got), "%s: result count", ctx) + if len(want) == 0 { + return + } + // Score multiset: both are score-desc, so compare pairwise within tolerance. + for i := range want { + require.InDeltaf(t, want[i].Score, got[i].Score, 1e-5, "%s: score at rank %d", ctx, i) + } + // Set of docs strictly above the boundary score must match exactly (only the + // boundary tie may differ in membership). + const tol = 1e-5 + boundary := want[len(want)-1].Score + above := func(rs []Result) map[any]struct{} { + m := make(map[any]struct{}) + for _, r := range rs { + if r.Score > boundary+tol { + m[r.Pk] = struct{}{} + } + } + return m + } + require.Equalf(t, above(want), above(got), "%s: above-boundary pk set", ctx) +} + +// TestBlockMaxWANDParity is the correctness gate for the Block-Max WAND walk: over +// a large multi-block corpus it asserts searchWAND (block-max top-k) returns the +// IDENTICAL ranking as searchBooleanFull (the exhaustive scan) for many k values, +// both scorers, both build-side and serialized→loaded segments, and both bare and +// boosted (> / <) disjunctions. A block-skip that ever drops a real top-k doc +// would diverge here. +func TestBlockMaxWANDParity(t *testing.T) { + const nDocs = 3000 + docs := blockMaxCorpus(t, nDocs) + + build, err := BuildSegmentFromTokenized("bm", int32(types.T_int64), docs) + require.NoError(t, err) + require.Greater(t, int(build.N), BlockSize*4, "corpus must span many blocks") + + // The loaded variant exercises deriveTermStats on the Deserialize path. + blob, err := build.Serialize() + require.NoError(t, err) + loaded, err := Deserialize("bm", bytes.NewReader(blob)) + require.NoError(t, err) + + // Sanity: a frequent term's postings really do span several blocks (else the + // skip path is untested). w00 is the most frequent under the Zipf pick. + if pl, ok := build.lookup("w00"); ok { + require.Greater(t, len(pl.blockLastDoc), 3, "frequent term should have >3 blocks") + require.Equal(t, len(pl.blockMaxTf), len(pl.blockLastDoc)) + } else { + t.Fatal("expected w00 present") + } + + // Disjunctive queries: mixes of frequent + rare terms, 2..5 terms, some boosted. + mk := func(weights ...struct { + w string + x float64 + }) BoolQuery { + var q BoolQuery + for _, e := range weights { + q.should = append(q.should, clause{kind: clauseTerm, terms: []string{e.w}, weight: float32(e.x)}) + } + return q + } + type wq = struct { + w string + x float64 + } + queries := []BoolQuery{ + mk(wq{"w00", 1}, wq{"w01", 1}), + mk(wq{"w00", 1}, wq{"w05", 1}, wq{"w20", 1}), + mk(wq{"w02", 1}, wq{"w03", 1}, wq{"w30", 1}, wq{"w39", 1}), + mk(wq{"w00", 1.1}, wq{"w10", 0.9}, wq{"w15", 1}), // boosted / demoted + mk(wq{"w01", 1}, wq{"w02", 1}, wq{"w04", 1}, wq{"w08", 1}, wq{"w16", 1}), + mk(wq{"w38", 1}, wq{"w39", 1}), // both rare + } + ks := []int{1, 5, 20, 100, 500, nDocs} + algos := []ScoreAlgo{BM25, TfIdf} + + for _, seg := range []*Segment{build, loaded} { + tag := "build" + if seg == loaded { + tag = "loaded" + } + for _, algo := range algos { + for qi, q := range queries { + terms, ok := disjunctiveTerms(q) + require.True(t, ok, "query %d must be WAND-eligible", qi) + for _, k := range ks { + want, err := seg.searchBooleanFull(q, algo, k, nil, nil) + require.NoError(t, err) + got := seg.searchWAND(terms, algo, k, nil, nil) + requireSameTopK(t, got, want, + fmt.Sprintf("%s algo=%v q=%d k=%d", tag, algo, qi, k)) + } + } + } + } +} diff --git a/pkg/fulltext2/wand_test.go b/pkg/fulltext2/wand_test.go new file mode 100644 index 0000000000000..e57ffc5c7dd51 --- /dev/null +++ b/pkg/fulltext2/wand_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fulltext2 + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/stretchr/testify/require" +) + +// syntheticCorpus builds a deterministic corpus with varied df/tf so WAND has +// something to prune: doc i includes vocab term j when i % (j+2) == 0, repeated +// (i%3)+1 times. +func syntheticCorpus(t *testing.T) *Segment { + t.Helper() + vocab := []string{"alpha", "beta", "gamma", "delta", "epsilon"} + var docs []Doc + for i := 1; i <= 60; i++ { + var words []string + for j, w := range vocab { + if i%(j+2) == 0 { + for r := 0; r <= i%3; r++ { + words = append(words, w) + } + } + } + if len(words) == 0 { + words = []string{"filler"} + } + docs = append(docs, Doc{int64(i), []byte(strings.Join(words, " "))}) + } + s, err := BuildSegmentFromDocs("syn", int32(types.T_int64), docs, tokenizer.NewSimpleTokenizer()) + require.NoError(t, err) + return s +} + +func requireSameRanking(t *testing.T, label string, wand, full []Result) { + t.Helper() + require.Equal(t, len(full), len(wand), "%s: count", label) + // WAND only avoids work; it must return the same top-k SCORES as the full scan. Both + // results are score-descending, so the score sequence must match rank-by-rank — this + // is the sufficient correctness invariant: a pruned or mis-scored doc changes a score + // here, while a pk difference can only occur among EQUAL scores, whose identity is + // unspecified (neither heap imposes a tiebreak, matching bm25). So we compare scores, + // not pk. (1e-5 also absorbs the float-ULP score noise between the two accumulation + // orders, which can flip a near-tie's doc identity.) + for i := range full { + require.InDelta(t, full[i].Score, wand[i].Score, 1e-5, "%s: rank %d score", label, i) + } +} + +// TestWANDParity: for every disjunctive query, WAND returns the identical ranking +// (same docs, same order, same scores) as the full scan — across k and both +// scorers, on two corpora. This is the WAND correctness contract: it only avoids +// work, it never changes results. +func TestWANDParity(t *testing.T) { + type tc struct { + name string + seg *Segment + query string + } + small := fulltextCorpus(t) + syn := syntheticCorpus(t) + cases := []tc{ + {"small/2", small, "quick fox"}, + {"small/3", small, "quick fox dog"}, + {"small/weighted", small, ">quick fox alpha beta