Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions be/src/runtime/descriptors.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
auto convert_to_thrift_column_access_path = [](const PColumnAccessPath& pb_path) {
TColumnAccessPath thrift_path;
thrift_path.type = (TAccessPathType::type)pb_path.type();
if (pb_path.has_version()) {
thrift_path.__set_version(pb_path.version());
}
if (pb_path.has_data_access_path()) {
thrift_path.__isset.data_access_path = true;
for (int i = 0; i < pb_path.data_access_path().path_size(); ++i) {
Expand Down Expand Up @@ -160,6 +163,9 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
doris::PColumnAccessPath* pb_path) {
pb_path->Clear();
pb_path->set_type((PAccessPathType)thrift_path.type); // 使用 reinterpret_cast 进行类型转换
if (thrift_path.__isset.version) {
pb_path->set_version(thrift_path.version);
}
if (thrift_path.__isset.data_access_path) {
auto* pb_data = pb_path->mutable_data_access_path();
pb_data->Clear();
Expand Down
787 changes: 443 additions & 344 deletions be/src/storage/segment/column_reader.cpp

Large diffs are not rendered by default.

58 changes: 46 additions & 12 deletions be/src/storage/segment/column_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

#include <cstddef> // for size_t
#include <cstdint> // for uint32_t
#include <functional>
#include <map>
#include <memory> // for unique_ptr
#include <optional>
Expand Down Expand Up @@ -500,18 +501,48 @@ class ColumnIterator {
}

protected:
struct AccessPathSplit {
TColumnAccessPaths descendant_paths;
bool reads_current_data = false;
MetaReadMode current_meta_mode = MetaReadMode::DEFAULT;

bool has_descendant_paths() const { return !descendant_paths.empty(); }
};

// Nested columns share the same current-level access-path planning, while their data-child
// topology and descendant routing remain container-specific.
struct NestedAccessPathPlan {
AccessPathSplit all;
AccessPathSplit predicate;
bool skip_data_descendants = false;
};

// At their current level, Struct supports null-map metadata. Map and Array additionally
// support offsets.
enum class NestedMetaSupport { NULL_MAP, NULL_MAP_AND_OFFSET };

void _convert_to_place_holder_column(MutableColumnPtr& dst, size_t count);

void _recovery_from_place_holder_column(MutableColumnPtr& dst);

// Derive current-level meta-only read mode from access paths. Meta-only is valid only when
// this iterator had no data-read requirement before applying the current paths, and every
// visible path at this level is NULL/OFFSET metadata.
void _check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path,
const TColumnAccessPaths& sub_all_access_paths);

Result<TColumnAccessPaths> _get_sub_access_paths(TColumnAccessPaths access_paths,
bool is_predicate = false);
// Derive current-level meta-only read mode from an explicit access-path split. Meta-only is
// valid only when this iterator had no data-read requirement before applying the current paths,
// no current DATA path exists, and no path must be routed to a descendant iterator.
Status _check_and_set_meta_read_mode(ReadRequirement requirement_before_access_path,
const AccessPathSplit& all_access_paths);

// Apply the common current-level access-path state transitions and select a supported
// parent-owned meta-only mode. When that mode skips data descendants, synchronously invoke the
// callback once with SKIP before returning the routing plan. The callback is never retained.
Result<NestedAccessPathPlan> _prepare_nested_access_paths(
const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths, NestedMetaSupport meta_support,
const std::function<void(ReadRequirement)>& set_all_data_descendants_read_requirement);

// Normalize the wire encoding, strip this iterator's column name, and explicitly partition
// paths consumed by this iterator from paths that must be routed to descendants. This helper is
// intentionally side-effect free; callers apply DATA/predicate read requirements explicitly.
Result<AccessPathSplit> _split_access_paths(TColumnAccessPaths access_paths) const;
ColumnIteratorOptions _opts;

ReadRequirement _read_requirement {ReadRequirement::NORMAL};
Expand Down Expand Up @@ -542,6 +573,9 @@ class FileColumnIterator : public ColumnIterator {
Status read_by_rowids(const rowid_t* rowids, const size_t count,
MutableColumnPtr& dst) override;

Status set_access_paths(const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths) override;

ordinal_t get_current_ordinal() const override { return _current_ordinal; }

// get row ranges by zone map
Expand Down Expand Up @@ -618,10 +652,10 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};

// StringFileColumnIterator extends FileColumnIterator with meta-only reading
// support for string/binary column types. When the OFFSET path is detected in
// set_access_paths, it sets only_read_offsets on the ColumnIteratorOptions so
// that the BinaryPlainPageDecoder skips chars memcpy and only fills offsets.
// StringFileColumnIterator extends FileColumnIterator's NULL metadata support with OFFSET-only
// reading for string/binary column types. When the OFFSET path is detected in set_access_paths, it
// sets only_read_offsets on the ColumnIteratorOptions so that the BinaryPlainPageDecoder skips
// chars memcpy and only fills offsets.
class StringFileColumnIterator final : public FileColumnIterator {
public:
explicit StringFileColumnIterator(std::shared_ptr<ColumnReader> reader);
Expand Down
59 changes: 58 additions & 1 deletion be/test/runtime/descriptor_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.

#include <gen_cpp/Descriptors_constants.h>
#include <gen_cpp/Exprs_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/descriptors.pb.h>
#include <gtest/gtest.h>

#include "common/exception.h"
Expand Down Expand Up @@ -196,4 +198,59 @@ TEST_F(SlotDescriptorTest, DebugString) {
EXPECT_TRUE(debug_str2.find("is_virtual=true") != std::string::npos);
}

} // namespace doris
TEST_F(SlotDescriptorTest, AccessPathsPreservedThroughProtobuf) {
TColumnAccessPath data_path;
data_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED);
data_path.__set_type(TAccessPathType::DATA);
TDataAccessPath data_payload;
data_payload.__set_path({"s", "field"});
data_path.__set_data_access_path(data_payload);

TColumnAccessPath meta_path;
meta_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED);
meta_path.__set_type(TAccessPathType::META);
TMetaAccessPath meta_payload;
meta_payload.__set_path({"s", "field", "NULL"});
meta_path.__set_meta_access_path(meta_payload);

TColumnAccessPath legacy_path;
legacy_path.__set_type(TAccessPathType::DATA);
data_payload.__set_path({"s", "legacy", "NULL"});
legacy_path.__set_data_access_path(data_payload);

TColumnAccessPath explicit_legacy_path;
explicit_legacy_path.__set_version(g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY);
explicit_legacy_path.__set_type(TAccessPathType::DATA);
data_payload.__set_path({"s", "legacy", "VALUES"});
explicit_legacy_path.__set_data_access_path(data_payload);

TSlotDescriptor thrift_descriptor = create_basic_slot_descriptor();
thrift_descriptor.__set_all_access_paths(
{data_path, meta_path, legacy_path, explicit_legacy_path});
thrift_descriptor.__set_predicate_access_paths({meta_path});

SlotDescriptor original(thrift_descriptor);
PSlotDescriptor protobuf_descriptor;
original.to_protobuf(&protobuf_descriptor);
ASSERT_EQ(protobuf_descriptor.all_access_paths_size(), 4);
EXPECT_TRUE(protobuf_descriptor.all_access_paths(0).has_version());
EXPECT_EQ(protobuf_descriptor.all_access_paths(0).version(),
g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_TYPED);
EXPECT_TRUE(protobuf_descriptor.all_access_paths(1).has_version());
EXPECT_FALSE(protobuf_descriptor.all_access_paths(2).has_version());
EXPECT_TRUE(protobuf_descriptor.all_access_paths(3).has_version());
EXPECT_EQ(protobuf_descriptor.all_access_paths(3).version(),
g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY);
SlotDescriptor round_trip(protobuf_descriptor);

EXPECT_EQ(round_trip.all_access_paths(), thrift_descriptor.all_access_paths);
EXPECT_EQ(round_trip.predicate_access_paths(), thrift_descriptor.predicate_access_paths);
EXPECT_FALSE(round_trip.all_access_paths()[1].__isset.data_access_path);
EXPECT_FALSE(round_trip.predicate_access_paths()[0].__isset.data_access_path);
EXPECT_FALSE(round_trip.all_access_paths()[2].__isset.version);
EXPECT_TRUE(round_trip.all_access_paths()[3].__isset.version);
EXPECT_EQ(round_trip.all_access_paths()[3].version,
g_Descriptors_constants.TCOLUMN_ACCESS_PATH_VERSION_LEGACY);
}

} // namespace doris
Loading
Loading