From 129e7214478f73624f844402dfe6c5457e65a4ec Mon Sep 17 00:00:00 2001 From: wyxxxcat <1520358997@qq.com> Date: Sun, 7 Jul 2024 20:41:21 +0800 Subject: [PATCH] fix --- be/src/agent/task_worker_pool.cpp | 3 +++ be/src/cloud/cloud_tablet.cpp | 1 + .../io/cache/block_file_cache_downloader.cpp | 1 + be/src/io/file_factory.cpp | 6 +++++- be/src/olap/rowset/beta_rowset.cpp | 2 ++ be/src/olap/rowset/beta_rowset_writer.cpp | 4 +++- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- be/src/runtime/load_stream.cpp | 1 + .../runtime/workload_group/workload_group.cpp | 4 ++-- be/src/util/s3_util.cpp | 1 + .../vec/exec/format/table/paimon_reader.cpp | 1 + be/src/vec/exec/scan/new_olap_scanner.cpp | 19 +++++++++++++++++++ .../iceberg/viceberg_partition_writer.cpp | 3 ++- .../sink/writer/vhive_partition_writer.cpp | 3 ++- be/src/vec/sink/writer/vtablet_writer_v2.cpp | 1 + 15 files changed, 45 insertions(+), 7 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 9b0a959295028e..bd552b8ee4b4a7 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1392,9 +1392,12 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr auto client = static_cast(existed_fs.get())->client_holder(); auto new_s3_conf = S3Conf::get_s3_conf(param.s3_storage_param); S3ClientConf conf { + .endpoint {}, + .region {}, .ak = std::move(new_s3_conf.client_conf.ak), .sk = std::move(new_s3_conf.client_conf.sk), .token = std::move(new_s3_conf.client_conf.token), + .bucket {}, .provider = new_s3_conf.client_conf.provider, }; st = client->reset(conf); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 5f77c7f44c1fcb..44635a41053d04 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -227,6 +227,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ { .expiration_time = expiration_time, }, + .download_done {}, }); } #endif diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index ca8e27567120d1..14e591b7faa198 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -191,6 +191,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met FileReaderOptions opts { .cache_type = FileCachePolicy::FILE_BLOCK_CACHE, .is_doris_table = true, + .cache_base_path {}, .file_size = meta.file_size, }; auto st = meta.file_system->open_file(meta.path, &file_reader, &opts); diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 0c84c2eb74c7d8..7f64ea50710268 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -55,7 +55,11 @@ constexpr std::string_view RANDOM_CACHE_BASE_PATH = "random"; io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, const io::FileDescription& fd) { - io::FileReaderOptions opts {.file_size = fd.file_size, .mtime = fd.mtime}; + io::FileReaderOptions opts { + .cache_base_path {}, + .file_size = fd.file_size, + .mtime = fd.mtime, + }; if (config::enable_file_cache && state != nullptr && state->query_options().__isset.enable_file_cache && state->query_options().enable_file_cache) { diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index a76cbe636eef90..d16c1146142d4e 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -174,6 +174,7 @@ Status BetaRowset::load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* se .cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE : io::FileCachePolicy::NO_CACHE, .is_doris_table = true, + .cache_base_path = "", .file_size = _rowset_meta->segment_file_size(seg_id), }; auto s = segment_v2::Segment::open(fs, seg_path, seg_id, rowset_id(), _schema, reader_options, @@ -532,6 +533,7 @@ Status BetaRowset::check_current_rowset_segment() { .cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE : io::FileCachePolicy::NO_CACHE, .is_doris_table = true, + .cache_base_path {}, .file_size = _rowset_meta->segment_file_size(seg_id), }; auto s = segment_v2::Segment::open(fs, seg_path, seg_id, rowset_id(), _schema, diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 17801ec16fd9da..1f07ce098fa25a 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -303,7 +303,9 @@ Status BetaRowsetWriter::_load_noncompacted_segment(segment_v2::SegmentSharedPtr ? (config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE : io::FileCachePolicy::NO_CACHE) : io::FileCachePolicy::NO_CACHE, - .is_doris_table = true}; + .is_doris_table = true, + .cache_base_path {}, + }; auto s = segment_v2::Segment::open(io::global_local_filesystem(), path, segment_id, rowset_id(), _context.tablet_schema, reader_options, &segment); if (!s.ok()) { diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 30b7e3b37508f8..3b0cdad5133f66 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -91,7 +91,7 @@ struct ColumnIteratorOptions { // for page cache allocation // page types are divided into DATA_PAGE & INDEX_PAGE // INDEX_PAGE including index_page, dict_page and short_key_page - PageTypePB type; + PageTypePB type = PageTypePB::UNKNOWN_PAGE_TYPE; io::FileReader* file_reader = nullptr; // Ref // reader statistics OlapReaderStatistics* stats = nullptr; // Ref diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp index 0a35bf6008e807..0a4dece073aa87 100644 --- a/be/src/runtime/load_stream.cpp +++ b/be/src/runtime/load_stream.cpp @@ -84,6 +84,7 @@ Status TabletStream::init(std::shared_ptr schema, int64_t .load_id = _load_id, .table_schema_param = schema, // TODO(plat1ko): write_file_cache + .storage_vault_id {}, }; _load_stream_writer = std::make_shared(&req, _profile); diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index 64a5c7aeffb8b6..fd45093758e858 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -252,7 +252,7 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( if (tworkload_group_info.__isset.id) { tg_id = tworkload_group_info.id; } else { - return {.valid = false}; + return {.name = "", .valid = false}; } // 2 name @@ -266,7 +266,7 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( if (tworkload_group_info.__isset.version) { version = tworkload_group_info.version; } else { - return {.valid = false}; + return {.name {}, .valid = false}; } // 4 cpu_share diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 630a998bd477ed..e24cd1e18a757b 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -351,6 +351,7 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { .region = info.region(), .ak = info.ak(), .sk = info.sk(), + .token {}, .bucket = info.bucket(), .provider = io::ObjStorageType::AWS, }, diff --git a/be/src/vec/exec/format/table/paimon_reader.cpp b/be/src/vec/exec/format/table/paimon_reader.cpp index 6a9bad3fb1dd01..d7611d0abb0709 100644 --- a/be/src/vec/exec/format/table/paimon_reader.cpp +++ b/be/src/vec/exec/format/table/paimon_reader.cpp @@ -45,6 +45,7 @@ Status PaimonReader::init_row_filters(const TFileRangeDesc& range) { .system_type = _params.file_type, .properties = _params.properties, .hdfs_params = _params.hdfs_params, + .broker_addresses {}, }; if (range.__isset.file_type) { // for compatibility diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index d539c0d7864f19..36e1c3d41a5440 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -75,8 +75,27 @@ NewOlapScanner::NewOlapScanner(pipeline::ScanLocalStateBase* parent, _key_ranges(std::move(params.key_ranges)), _tablet_reader_params({ .tablet = std::move(params.tablet), + .tablet_schema {}, .aggregation = params.aggregation, .version = {0, params.version}, + .start_key {}, + .end_key {}, + .conditions {}, + .bloom_filters {}, + .bitmap_filters {}, + .in_filters {}, + .conditions_except_leafnode_of_andnode {}, + .function_filters {}, + .delete_predicates {}, + .target_cast_type_for_variants {}, + .rs_splits {}, + .return_columns {}, + .output_columns {}, + .remaining_conjunct_roots {}, + .common_expr_ctxs_push_down {}, + .topn_filter_source_node_ids {}, + .filter_block_conjuncts {}, + .key_group_cluster_key_idxes {}, }) { _tablet_reader_params.set_read_source(std::move(params.read_source)); _is_init = false; diff --git a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp index da5758fd604873..9cf7af32204e2f 100644 --- a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -53,7 +53,8 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil io::FSPropertiesRef fs_properties(_write_info.file_type); fs_properties.properties = &_hadoop_conf; io::FileDescription file_description = { - .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name())}; + .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), + .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); diff --git a/be/src/vec/sink/writer/vhive_partition_writer.cpp b/be/src/vec/sink/writer/vhive_partition_writer.cpp index 1e8b28ef3aeb2c..5322bde57f13e2 100644 --- a/be/src/vec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/vec/sink/writer/vhive_partition_writer.cpp @@ -56,7 +56,8 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile) io::FSPropertiesRef fs_properties(_write_info.file_type); fs_properties.properties = &_hadoop_conf; io::FileDescription file_description = { - .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name())}; + .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), + .fs_name {}}; // If the destination path contains a schema, use the schema directly. // If not, use defaultFS. // Otherwise a write error will occur. diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp index fbefd7a6d836cd..c221849c08e0de 100644 --- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp @@ -443,6 +443,7 @@ Status VTabletWriterV2::_write_memtable(std::shared_ptr block .table_schema_param = _schema, .is_high_priority = _is_high_priority, .write_file_cache = _write_file_cache, + .storage_vault_id {}, }; bool index_not_found = true; for (const auto& index : _schema->indexes()) {