Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/Java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,7 @@ jobs:
- name: Build
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat"
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat"
mkdir build
cd build
mkdir release
Expand All @@ -749,7 +749,7 @@ jobs:
- name: List Symbols
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat"
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat"
dumpbin.exe /imports build\release\libduckdb_java.so_windows_arm64
dumpbin.exe /exports build\release\libduckdb_java.so_windows_arm64

Expand Down
19 changes: 19 additions & 0 deletions src/duckdb/extension/parquet/column_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,25 @@ ColumnWriter::ColumnWriter(ParquetWriter &writer, ParquetColumnSchema &&column_s
ColumnWriter::~ColumnWriter() {
}

void ColumnWriter::MarkRepetitionRequired() {
if (column_schema.repetition_type == duckdb_parquet::FieldRepetitionType::REQUIRED) {
return;
}
D_ASSERT(column_schema.repetition_type == duckdb_parquet::FieldRepetitionType::OPTIONAL);
D_ASSERT(can_have_nulls);
column_schema.repetition_type = duckdb_parquet::FieldRepetitionType::REQUIRED;
can_have_nulls = false;
DecrementMaxDefineRecursive();
}

void ColumnWriter::DecrementMaxDefineRecursive() {
D_ASSERT(column_schema.max_define > 0);
column_schema.max_define--;
for (auto &child : child_writers) {
child->DecrementMaxDefineRecursive();
}
}

ColumnWriterState::~ColumnWriterState() {
}

Expand Down
5 changes: 5 additions & 0 deletions src/duckdb/extension/parquet/include/column_writer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ class ColumnWriter {
return child_writers;
}

//! Mark this writer (and descendants) as REQUIRED in the Parquet schema
void MarkRepetitionRequired();

virtual void AnalyzeSchema(ParquetAnalyzeSchemaState &state, Vector &input, idx_t count) {
throw NotImplementedException("Writer doesn't require an AnalyzeSchema pass");
}
Expand Down Expand Up @@ -199,6 +202,8 @@ class ColumnWriter {
void CompressPage(MemoryStream &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
AllocatedData &compressed_buf);

void DecrementMaxDefineRecursive();

public:
ParquetWriter &writer;
//! The parent writer (if this is a nested field)
Expand Down
5 changes: 1 addition & 4 deletions src/duckdb/extension/parquet/parquet_shredding.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@ bool ShreddingType::operator==(const ShreddingType &other) const {
if (set != other.set) {
return false;
}
if (!set) {
return true;
}
if (type != other.type) {
if (set && type != other.type) {
return false;
}
if (!children.types && !other.children.types) {
Expand Down
40 changes: 40 additions & 0 deletions src/duckdb/extension/parquet/writer/variant/convert_variant.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,40 @@ static void ToParquetVariant(DataChunk &input, ExpressionState &state, Vector &r
}
}

static optional_ptr<ColumnWriter> FindChildWriterByName(ColumnWriter &parent, const char *name) {
for (auto &child : parent.ChildWriters()) {
if (child->Schema().name == name) {
return *child;
}
}
return nullptr;
}

static void MarkTypedValueShreddingGroupsRequired(ColumnWriter &typed_value);

// Field/element group: mark REQUIRED, then recurse into nested typed_value if present
static void MarkShreddedGroupRequired(ColumnWriter &group) {
D_ASSERT(group.Type().id() == LogicalTypeId::STRUCT);
group.MarkRepetitionRequired();
auto nested_typed_value = FindChildWriterByName(group, "typed_value");
if (nested_typed_value) {
MarkTypedValueShreddingGroupsRequired(*nested_typed_value);
}
}

// Field/element groups under typed_value must be REQUIRED (VariantShredding.md)
static void MarkTypedValueShreddingGroupsRequired(ColumnWriter &typed_value) {
auto type_id = typed_value.Type().id();
if (type_id == LogicalTypeId::STRUCT) {
for (auto &field_group : typed_value.ChildWriters()) {
MarkShreddedGroupRequired(*field_group);
}
} else if (type_id == LogicalTypeId::LIST || type_id == LogicalTypeId::ARRAY) {
D_ASSERT(typed_value.ChildWriters().size() == 1);
MarkShreddedGroupRequired(*typed_value.ChildWriters()[0]);
}
}

idx_t VariantColumnWriter::FinalizeSchema(vector<duckdb_parquet::SchemaElement> &schemas) {
idx_t schema_idx = schemas.size();

Expand All @@ -895,6 +929,12 @@ idx_t VariantColumnWriter::FinalizeSchema(vector<duckdb_parquet::SchemaElement>
auto &name = schema.name;
auto &field_id = schema.field_id;

// Mark shredded field/element groups REQUIRED (reserved Parquet names only)
auto typed_value = FindChildWriterByName(*this, "typed_value");
if (typed_value) {
MarkTypedValueShreddingGroupsRequired(*typed_value);
}

// variant group
duckdb_parquet::SchemaElement top_element;
top_element.repetition_type = repetition_type;
Expand Down
6 changes: 3 additions & 3 deletions src/duckdb/src/function/table/version/pragma_version.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#ifndef DUCKDB_PATCH_VERSION
#define DUCKDB_PATCH_VERSION "6-dev212"
#define DUCKDB_PATCH_VERSION "6"
#endif
#ifndef DUCKDB_MINOR_VERSION
#define DUCKDB_MINOR_VERSION 5
Expand All @@ -8,10 +8,10 @@
#define DUCKDB_MAJOR_VERSION 1
#endif
#ifndef DUCKDB_VERSION
#define DUCKDB_VERSION "v1.5.6-dev212"
#define DUCKDB_VERSION "v1.5.6"
#endif
#ifndef DUCKDB_SOURCE_ID
#define DUCKDB_SOURCE_ID "46f1c14caa"
#define DUCKDB_SOURCE_ID "069cc9f9b5"
#endif
#include "duckdb/function/table/system_functions.hpp"
#include "duckdb/main/database.hpp"
Expand Down
Loading
Loading