From 61c42a48c7ca3fadfcede382e8bc813770e160f9 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 18 Aug 2026 11:41:35 -0700 Subject: [PATCH 1/3] feat: register nemotron-rerank-vl-1b-v2 alongside the existing reranker Adds nvidia/llama-nemotron-rerank-vl-1b-v2 (NIM 2.x, Rust runtime) as a second, non-default model. The existing llama-3.2-nv-rerankqa-1b-v2 stays is-default: collections pin provider + model in their schema, so the default only affects newly created collections and existing traffic is unaffected. Batch size is held at 10 to match the existing model so load comparisons are like-for-like; NIM 2.x accepts up to 512 passages per call, which is worth measuring as a separate change. Note this file is only used when the embedding gateway is disabled (RerankingProviderConfigProducer) - Astra dev/prod get the model list from EGW over gRPC, so prod registration is an EGW config change. This entry covers local/OSS/HCD and local testing against the new NIM. --- src/main/resources/reranking-providers-config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index 67739fe0a2..58a164753a 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -14,4 +14,16 @@ stargate: is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: - max-batch-size: 10 \ No newline at end of file + max-batch-size: 10 + # NIM 2.x (Rust runtime) reranking model, deployed alongside the model above. + # Not the default: collections pin provider + model in their schema, so the default + # only affects newly created collections. + # TODO confirm the gateway path once the GPU plane deployment is routed: the proxy + # may dispatch on the `model` field in the body (same path) or expose a new path. + - name: nvidia/llama-nemotron-rerank-vl-1b-v2 + url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking + properties: + # Kept at 10 to match the existing model so load comparisons are like-for-like. + # NIM 2.x accepts up to 512 passages per call; raising this collapses the + # per-findAndRerank fan-out and should be measured as a separate change. + max-batch-size: 10 From 34071b5ab6aad0cfa1a8fadc4f3e254e1c95449d Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 18 Aug 2026 11:42:49 -0700 Subject: [PATCH 2/3] Update reranking-providers-config.yaml --- src/main/resources/reranking-providers-config.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index 58a164753a..6efd32e3a1 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -16,14 +16,7 @@ stargate: properties: max-batch-size: 10 # NIM 2.x (Rust runtime) reranking model, deployed alongside the model above. - # Not the default: collections pin provider + model in their schema, so the default - # only affects newly created collections. - # TODO confirm the gateway path once the GPU plane deployment is routed: the proxy - # may dispatch on the `model` field in the body (same path) or expose a new path. - name: nvidia/llama-nemotron-rerank-vl-1b-v2 url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: - # Kept at 10 to match the existing model so load comparisons are like-for-like. - # NIM 2.x accepts up to 512 passages per call; raising this collapses the - # per-findAndRerank fan-out and should be measured as a separate change. max-batch-size: 10 From 601c9fa53259500d7b82da9d3d8bcbfa44bca726 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 18 Aug 2026 11:46:37 -0700 Subject: [PATCH 3/3] Update reranking-providers-config.yaml --- src/main/resources/reranking-providers-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index 6efd32e3a1..52d2af6216 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -11,12 +11,12 @@ stargate: enabled: true models: - name: nvidia/llama-3.2-nv-rerankqa-1b-v2 - is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: max-batch-size: 10 # NIM 2.x (Rust runtime) reranking model, deployed alongside the model above. - name: nvidia/llama-nemotron-rerank-vl-1b-v2 + is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: max-batch-size: 10