From 61db2e21e893bcb2665644ab49a84c9142887fdc Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:33:34 +0800 Subject: [PATCH] [Bug](column) fix append_data_by_selector_impl reserve too mush useless memory (#39581) ## Proposed changes ``` for (auto* place : local_state._value_places) { SCOPED_TIMER(local_state._selector_block_timer); RETURN_IF_ERROR(place->append_block_by_selector(input_block, eos)); } for (int i = 0; i < mutable_columns.size(); ++i) { columns[i]->append_data_by_selector(mutable_columns[i], _selector); } ``` the columns[I] size is 4096, and _selector size is 1; so the (4096 - 1) rows is useless; in a usercase, the block have 4096 rows and 34 columns. so need bytes is: 1 block * 4096 rows * 4096 reserve * 16 bytes * 34 column * 48 instance = 438086664192 bytes; it's will cause the query canceled as not have enough memory. --- be/src/vec/columns/column_impl.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/be/src/vec/columns/column_impl.h b/be/src/vec/columns/column_impl.h index f0a157f41972f3..470825851e1e9d 100644 --- a/be/src/vec/columns/column_impl.h +++ b/be/src/vec/columns/column_impl.h @@ -42,8 +42,11 @@ void IColumn::append_data_by_selector_impl(MutablePtr& res, const Selector& sele "Size of selector: {} is larger than size of column: {}", selector.size(), num_rows); } - - res->reserve(num_rows); + DCHECK_GE(end, begin); + // here wants insert some value from this column, and the nums is (end - begin) + // and many be this column num_rows is 4096, but only need insert num is (1 - 0) = 1 + // so can't call res->reserve(num_rows), it's will be too mush waste memory + res->reserve(res->size() + (end - begin)); for (size_t i = begin; i < end; ++i) { static_cast(*res).insert_from(*this, selector[i]);