From a967c441635871bc50b38eced9662a0a0f607fbb Mon Sep 17 00:00:00 2001 From: loutPhilipps Date: Tue, 4 Aug 2026 16:18:17 +0200 Subject: [PATCH] search: add query_ast, counts, and warmup byte span attributes --- quickwit/Cargo.lock | 1 + quickwit/quickwit-search/Cargo.toml | 1 + quickwit/quickwit-search/src/leaf.rs | 38 ++++++++++++++++++++++++++-- quickwit/quickwit-search/src/root.rs | 16 +++++++++--- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 448e8b3db57..6adde624777 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -9306,6 +9306,7 @@ dependencies = [ "tokio-util", "tower 0.5.3", "tracing", + "tracing-opentelemetry", "ttl_cache", "ulid", "utoipa", diff --git a/quickwit/quickwit-search/Cargo.toml b/quickwit/quickwit-search/Cargo.toml index 781a49a779f..cbbe2b269d5 100644 --- a/quickwit/quickwit-search/Cargo.toml +++ b/quickwit/quickwit-search/Cargo.toml @@ -33,6 +33,7 @@ tokio = { workspace = true } tokio-util = { workspace = true } tower = { workspace = true, features = ["timeout"] } tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } ttl_cache = { workspace = true } ulid = { workspace = true } utoipa = { workspace = true } diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs index 894c09289e2..c1a0b3a7070 100644 --- a/quickwit/quickwit-search/src/leaf.rs +++ b/quickwit/quickwit-search/src/leaf.rs @@ -300,7 +300,6 @@ async fn run_cancellable( /// /// Returns whether the query is provably empty in this split (i.e. `on_absent` fired and /// warmup was short-circuited). -#[instrument(skip_all)] pub(crate) async fn warmup( searcher: &Searcher, warmup_info: &WarmupInfo, @@ -746,6 +745,12 @@ async fn leaf_search_single_split( warmup_info.simplify(); let warmup_start = Instant::now(); + // Baseline the counters so the span attributes cover warmup only. This matters for + // `download_counters`: opening the index fetches the footer and hotcache through the same + // counted storage on a cold footer cache. The byte-range cache is usually still empty at + // this point, since index-open reads are served by the hotcache above it. + let cache_bytes_before_warmup = byte_range_cache.get_num_bytes(); + let downloaded_bytes_before_warmup = download_counters.snapshot().0; leaf_search_state_guard.set_state(SplitSearchState::WarmUp); // Negative cache: a split is provably empty for this query if any required term has // previously been proven absent here. Absence is an immutable, query- and @@ -779,7 +784,36 @@ async fn leaf_search_single_split( HitSet::empty(), ); }; - warmup(&searcher, &warmup_info, &record_absence).await? + // `downloaded_mb` is what warmup actually fetched from blob storage; `total_mb` adds the + // part served from the fast-field cache, since the byte-range cache sits above it and + // records every miss it has to resolve below. Both are deltas since `warmup_start`, so + // index-open reads are not attributed here, and are recorded after warmup, before the + // search adds to either. + const BYTES_PER_MB: f64 = 1_000_000.0; + let warmup_span = info_span!( + "warmup", + downloaded_mb = tracing::field::Empty, + total_mb = tracing::field::Empty + ); + let provably_empty = warmup(&searcher, &warmup_info, &record_absence) + .instrument(warmup_span.clone()) + .await?; + warmup_span.record( + "downloaded_mb", + download_counters + .snapshot() + .0 + .saturating_sub(downloaded_bytes_before_warmup) as f64 + / BYTES_PER_MB, + ); + warmup_span.record( + "total_mb", + byte_range_cache + .get_num_bytes() + .saturating_sub(cache_bytes_before_warmup) as f64 + / BYTES_PER_MB, + ); + provably_empty }; let warmup_end = Instant::now(); let warmup_duration: Duration = warmup_end.duration_since(warmup_start); diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs index 50033fb5ff9..896a763f508 100644 --- a/quickwit/quickwit-search/src/root.rs +++ b/quickwit/quickwit-search/src/root.rs @@ -47,7 +47,8 @@ use tantivy::aggregation::agg_result::AggregationResults; use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; use tantivy::collector::Collector; use tantivy::schema::{Field, FieldEntry, FieldType, Schema}; -use tracing::{debug, error, info, info_span, instrument}; +use tracing::{Span, debug, error, info, info_span, instrument}; +use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::cluster_client::ClusterClient; use crate::collector::{QuickwitAggregations, make_merge_collector}; @@ -1312,8 +1313,6 @@ pub async fn root_search( let num_docs: usize = split_metadatas.iter().map(|split| split.num_docs).sum(); let num_splits = split_metadatas.len(); - // It would have been nice to add those in the context of the trace span, - // but with our current logging setting, it makes logs too verbose. info!( query_ast = search_request.query_ast.as_str(), agg = search_request.aggregation_request(), @@ -1324,6 +1323,17 @@ pub async fn root_search( "root_search" ); + // set attributes directly on the trace so it doesn't make logs too verbose + Span::current().set_attribute("query_ast", search_request.query_ast.clone()); + Span::current().set_attribute("num_docs", num_docs as i64); + Span::current().set_attribute("num_splits", num_splits as i64); + if let Some(start_timestamp) = search_request.start_timestamp { + Span::current().set_attribute("start_timestamp", start_timestamp); + } + if let Some(end_timestamp) = search_request.end_timestamp { + Span::current().set_attribute("end_timestamp", end_timestamp); + } + if let Some(max_total_split_searches) = searcher_context.searcher_config.max_splits_per_search && max_total_split_searches < num_splits {