Skip to content

DuckDB 2.0 (v2.0-cyanoptera head) #10513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joseph-isaacs wants to merge 2 commits into
base: develop
Choose a base branch
from
Draft

DuckDB 2.0 (v2.0-cyanoptera head) #10513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb261db
DuckDB 2.0 support, pinned to v2.0-cyanoptera head (8588363b)
claude Oct 15, 2026
3078838
Regenerate DuckDB SQL logic test expectations for DuckDB 2.0 head
claude Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion scripts/duckdb-r2-prepare.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,15 @@ else
src_dir="duckdb-${REF_DIR}"
extra=""
if [ "$PLATFORM_OS" = "osx" ]; then
# httpfs is out-of-tree with vcpkg dependencies, and vcpkg can't build
# universal binaries. The S3 tests that need it only run on Linux.
extra="OSX_BUILD_UNIVERSAL=1"
extensions="parquet;tpch;tpcds;icu"
else
make -C "$src_dir" setup-vcpkg
VCPKG_TOOLCHAIN_PATH="$PWD/${src_dir}/vcpkg/scripts/buildsystems/vcpkg.cmake"
export VCPKG_TOOLCHAIN_PATH
extensions="parquet;tpch;tpcds;icu;httpfs"
fi

make -C "$src_dir" \
Expand All @@ -42,7 +50,8 @@ else
BUILD_SHELL=false \
BUILD_UNITTESTS=false \
ENABLE_UNITTEST_CPP_TESTS=false \
BUILD_EXTENSIONS="parquet;tpch;tpcds" \
BUILD_EXTENSIONS="$extensions" \
LINK_CORE_EXTENSIONS=1 \
$extra

lib_dir="${src_dir}/build/release/src"
Expand Down
94 changes: 36 additions & 58 deletions vortex-duckdb/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,18 @@ const DUCKDB_RELEASES_URL: &str = "https://ci-builds.vortex.dev";

const DUCKDB_SOURCE_RELEASE_URL: &str = "https://github.com/duckdb/duckdb/archive/refs/tags";
const DUCKDB_SOURCE_COMMIT_URL: &str = "https://github.com/duckdb/duckdb/archive";
const DEFAULT_DUCKDB_VERSION: &str = "1.5.5";
const DEFAULT_DUCKDB_VERSION: &str = "8588363bf15e090fcbbfe2a68ec98b10c6866cb1";

const BUILD_ARTIFACTS: [&str; 3] = ["libduckdb.dylib", "libduckdb.so", "libduckdb_static.a"];
const BUILD_MARKER: &str = ".vx-build-complete";
const DUCKDB_CACHE_DIR: &str = "vortex-duckdb-cache";
const EXTRACT_MARKER: &str = ".vx-extract-complete";

const SOURCE_FILES: [&str; 12] = [
const SOURCE_FILES: [&str; 9] = [
"cpp/vortex_duckdb.cpp",
"cpp/copy_function.cpp",
"cpp/expr.cpp",
"cpp/optimizer.cpp",
"cpp/scalar_fn_pushdown.cpp",
"cpp/spatial_overrides.cpp",
"cpp/cast_pushdown.cpp",
"cpp/aggregate_fn_pushdown.cpp",
"cpp/table_filter.cpp",
"cpp/multi_file_reader.cpp",
Expand Down Expand Up @@ -354,48 +351,6 @@ fn extract(archive: &Path, dest: &Path) {
zip::ZipArchive::new(file).unwrap().extract(dest).unwrap();
}

fn git_apply(repo_dir: &Path, patch: &Path, args: &[&str]) -> bool {
let output = Command::new("git")
.current_dir(repo_dir)
.args(["apply", "-p1"])
.args(args)
.arg(patch)
.output();
match output {
Ok(out) => out.status.success(),
Err(e) => {
println!("cargo:error=git is required to patch DuckDB sources: {e}");
exit(1);
}
}
}

fn apply_source_patches(crate_dir: &Path, repo_dir: &Path) {
let mut patches: Vec<PathBuf> = fs::read_dir(crate_dir.join("patches"))
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|ext| ext == "diff"))
.collect();
patches.sort();

for patch in patches {
// A successful reverse dry-run means the patch is already applied.
if git_apply(repo_dir, &patch, &["--check", "--reverse"]) {
continue;
}
if !git_apply(repo_dir, &patch, &[]) {
println!(
"cargo:error=Failed to apply {} to {}; delete that directory to re-extract \
DuckDB sources",
patch.display(),
repo_dir.display()
);
exit(1);
}
println!("cargo:info=Applied {}", patch.display());
}
}

/// Download DuckDB library archive from R2 and extract it.
/// Return false if archive is not available or download failed
fn download_prebuilt(version: &DuckDBVersion, library_dir: &Path, target: &str) -> bool {
Expand Down Expand Up @@ -447,12 +402,23 @@ fn build_duckdb(version: &DuckDBVersion, duckdb_repo_dir: &Path) {
("1", "0")
};

// If we're building from a commit we need to build benchmark
// extensions statically, otherwise DuckDB tries to load them from an http
// endpoint with version 0.0.1 (all non-tagged builds) which doesn't exist.
let static_extensions = match version {
DuckDBVersion::Release(_) => "parquet",
DuckDBVersion::Commit(_) => "parquet;tpch;tpcds",
// If we're building from a commit we need to build some extensions
// statically, otherwise DuckDB tries to load them from an http endpoint
// with version 0.0.1 (all non-tagged builds) which doesn't exist.
let (static_extensions, link_extensions) = match version {
DuckDBVersion::Release(_) => ("parquet", "0"),
// tpch/tpcds/parquet needed for benchmarks
// icu needed for timestamptz tests
// httpfs needed for s3 write test, but it has vcpkg dependencies
DuckDBVersion::Commit(_) if env::var_os("VCPKG_TOOLCHAIN_PATH").is_some() => {
("parquet;tpch;tpcds;icu;httpfs", "1")
}
DuckDBVersion::Commit(_) => {
println!(
"cargo:warning=VCPKG_TOOLCHAIN_PATH is not set, building DuckDB without httpfs"
);
("parquet;tpch;tpcds;icu", "1")
}
};

let envs = [
Expand All @@ -463,6 +429,7 @@ fn build_duckdb(version: &DuckDBVersion, duckdb_repo_dir: &Path) {
("BUILD_UNITTESTS", "false"),
("ENABLE_UNITTEST_CPP_TESTS", "false"),
("BUILD_EXTENSIONS", static_extensions),
("LINK_CORE_EXTENSIONS", link_extensions),
];

let output = Command::new("make")
Expand Down Expand Up @@ -590,7 +557,21 @@ fn compile_cpp(duckdb_include_dir: &Path) {
.flag_if_supported("-fno-gnu-unique")
// We don't want compiler warnings inside duckdb headers, pass as flags
.flag("-isystem")
.flag(duckdb_include_dir)
.flag(duckdb_include_dir);

if let Some(root) = duckdb_include_dir.parent().and_then(Path::parent) {
let third_party = root.join("third_party");
if let Ok(entries) = fs::read_dir(&third_party) {
for entry in entries.flatten() {
let include = entry.path().join("include");
if include.is_dir() {
build.flag("-isystem").flag(include);
}
}
}
}

build
.include("include")
.include("cpp/include")
.files(SOURCE_FILES)
Expand Down Expand Up @@ -658,7 +639,6 @@ fn vortex_version(crate_dir: &Path) {

fn main() {
println!("cargo:rerun-if-changed=cpp/include");
println!("cargo:rerun-if-changed=patches");
println!("cargo:rerun-if-env-changed=VX_DUCKDB_DEBUG");
println!("cargo:rerun-if-env-changed=VX_DUCKDB_SAN");
println!("cargo:rerun-if-env-changed=DEBUG");
Expand All @@ -672,6 +652,7 @@ fn main() {
// duckdb if they are present
println!("cargo:rerun-if-env-changed=DUCKDB_SOURCE_DIR");
println!("cargo:rerun-if-env-changed=DUCKDB_VERSION");
println!("cargo:rerun-if-env-changed=VCPKG_TOOLCHAIN_PATH");

// sanity check that either none or both are provided
assert_eq!(
Expand All @@ -690,7 +671,6 @@ fn main() {
"cargo:info=Using DuckDB source from DUCKDB_SOURCE_DIR={}",
source_dir.display()
);
apply_source_patches(&crate_dir, &source_dir);
bindgen_c2rust(&crate_dir, &duckdb_include_dir);
cbindgen_rust2c(&crate_dir);
compile_cpp(&duckdb_include_dir);
Expand Down Expand Up @@ -742,8 +722,6 @@ fn main() {
fs::write(&extract_marker, version.to_string()).unwrap();
}

apply_source_patches(&crate_dir, &inner_dir);

drop(fs::remove_file(&duckdb_dir));
drop(fs::remove_dir_all(&duckdb_dir));
symlink(&source_dir, &duckdb_dir).unwrap();
Expand Down
5 changes: 5 additions & 0 deletions vortex-duckdb/cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ endif()
include_directories(include ../include)
include_directories(SYSTEM ${DUCKDB_INCLUDE})

file(GLOB DUCKDB_THIRD_PARTY_INCLUDES "${DUCKDB_DIR}/third_party/*/include")
foreach(THIRD_PARTY_INCLUDE ${DUCKDB_THIRD_PARTY_INCLUDES})
include_directories(SYSTEM ${THIRD_PARTY_INCLUDE})
endforeach()

# Auto-discover C++ source files
file(GLOB CPP_SOURCES "*.cpp")
add_library(vortex ${CPP_SOURCES})
42 changes: 18 additions & 24 deletions vortex-duckdb/cpp/aggregate_fn_pushdown.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
#include "duckdb/planner/expression/bound_columnref_expression.hpp"
#include "duckdb/planner/operator/logical_aggregate.hpp"
#include "optimizer.hpp"
#include "table_function.hpp"

using enum LogicalOperatorType;
Expand Down Expand Up @@ -37,8 +36,6 @@ static bool IsUngrouped(const LogicalAggregate &agg) {
!agg.expressions.empty();
}

constexpr inline idx_t COUNT_STAR_PROJ_IDX = std::numeric_limits<TableColumnStorageIndex>::max();

LogicalOperatorPtr TryReplaceAggregate(ClientContext &context,
LogicalOperatorPtr op,
Analyses &analyses,
Expand All @@ -53,7 +50,7 @@ LogicalOperatorPtr TryReplaceAggregate(ClientContext &context,
return op;
}

vector<std::pair<TableColumnScanIndex, const Expression &>> input;
vector<std::pair<idx_t, const Expression &>> input;
const idx_t aggregates_len = agg.expressions.size();
input.reserve(aggregates_len);

Expand All @@ -62,21 +59,22 @@ LogicalOperatorPtr TryReplaceAggregate(ClientContext &context,
return op;
}
const auto &bound_aggr = expr->Cast<BoundAggregateExpression>();
if (bound_aggr.IsDistinct() || bound_aggr.filter != nullptr || bound_aggr.order_bys != nullptr) {
if (bound_aggr.IsDistinct() || bound_aggr.GetFilter() != nullptr ||
bound_aggr.GetOrderBys() != nullptr) {
return op;
}

if (bound_aggr.function.name == "count_star") {
if (bound_aggr.Function().GetName() == "count_star") {
input.emplace_back(COUNT_STAR_PROJ_IDX, *expr);
continue;
}

if (bound_aggr.children.size() != 1 ||
bound_aggr.children[0]->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
if (bound_aggr.GetChildren().size() != 1 ||
bound_aggr.GetChildren()[0]->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
return op;
}
const auto &bound_col = bound_aggr.children[0]->Cast<BoundColumnRefExpression>();
const auto binding = Resolve(bound_col.binding, analyses, projections);
const auto &bound_col = bound_aggr.GetChildren()[0]->Cast<BoundColumnRefExpression>();
const auto binding = Resolve(bound_col.Binding(), analyses, projections);
if (!binding || &binding->analysis.get != get) {
return op;
}
Expand All @@ -87,28 +85,24 @@ LogicalOperatorPtr TryReplaceAggregate(ClientContext &context,
return op;
}

vector<string> names(aggregates_len); // need a copy because we reference original names
// GET now returns one column per aggregate. Expand existing columns
auto &column_ids = get->GetMutableColumnIds();
get->types.resize(aggregates_len);
get->returned_types.resize(aggregates_len);
column_ids.resize(aggregates_len);

vector<Identifier> names(aggregates_len); // need a copy because we reference original names

const vector<ColumnIndex> original_column_ids = get->GetColumnIds();
for (idx_t i = 0; i < aggregates_len; i++) {
const auto &[column_index, expr] = input[i];
if (column_index == COUNT_STAR_PROJ_IDX) {
names[i] = "count_star()";
} else {
const TableColumnStorageIndex storage_index = original_column_ids[column_index].GetPrimaryIndex();
const idx_t storage_index = get->GetColumnIds()[column_index].GetPrimaryIndex();
names[i] = get->names[storage_index];
}
}

// GET now returns one column per aggregate.
auto &column_ids = get->GetMutableColumnIds();
get->types.resize(aggregates_len);
get->returned_types.resize(aggregates_len);
column_ids.resize(aggregates_len);
for (idx_t i = 0; i < aggregates_len; i++) {
const auto &[_, expr] = input[i];
get->types[i] = expr.return_type;
get->returned_types[i] = expr.return_type;
get->types[i] = expr.GetReturnType();
get->returned_types[i] = expr.GetReturnType();
column_ids[i] = ColumnIndex {i};
}
get->names = std::move(names);
Expand Down
Loading
Loading