Skip to content

Commit

Permalink
[Bug][Vectorized] Fix insert bimmap column with nullable column (#9408)
Browse files Browse the repository at this point in the history 
Co-authored-by: lihaopeng <lihaopeng@baidu.com>
  • Loading branch information
@HappenLee @morningman
2 people authored and morningman committed May 18, 2022
1 parent c41618a commit ca952c9
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 108 deletions.
209 changes: 102 additions & 107 deletions be/src/vec/sink/vtablet_sink.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block)
// because of "data unqualified"
return Status::EndOfFile("Encountered unqualified data, stop processing");
}
_convert_to_dest_desc_block(&block);
}

BlockRow block_row;
Expand Down Expand Up @@ -176,132 +177,105 @@ Status VOlapTableSink::_validate_data(RuntimeState* state, vectorized::Block* bl
block->get_by_position(i).column->convert_to_full_column_if_const();
const auto& column = block->get_by_position(i).column;

if (desc->type() == TYPE_OBJECT && column->is_nullable()) {
const auto& null_map =
vectorized::check_and_get_column<vectorized::ColumnNullable>(*column)
->get_null_map_data();
fmt::format_to(error_msg, "null is not allowed for bitmap column, column_name: {}; ",
desc->col_name());
auto column_ptr = vectorized::check_and_get_column<vectorized::ColumnNullable>(*column);
auto& real_column_ptr =
column_ptr == nullptr ? column : (column_ptr->get_nested_column_ptr());

switch (desc->type().type) {
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
const auto column_string =
assert_cast<const vectorized::ColumnString*>(real_column_ptr.get());

size_t limit = std::min(config::string_type_length_soft_limit_bytes, desc->type().len);
for (int j = 0; j < num_rows; ++j) {
if (!filter_bitmap->Get(j)) {
if (null_map[j]) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
}
}
} else {
auto column_ptr = vectorized::check_and_get_column<vectorized::ColumnNullable>(*column);
auto& real_column_ptr =
column_ptr == nullptr ? column : (column_ptr->get_nested_column_ptr());

switch (desc->type().type) {
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
const auto column_string =
assert_cast<const vectorized::ColumnString*>(real_column_ptr.get());

size_t limit =
std::min(config::string_type_length_soft_limit_bytes, desc->type().len);
for (int j = 0; j < num_rows; ++j) {
if (!filter_bitmap->Get(j)) {
auto str_val = column_string->get_data_at(j);
bool invalid = str_val.size > limit;
auto str_val = column_string->get_data_at(j);
bool invalid = str_val.size > limit;

error_msg.clear();
if (str_val.size > desc->type().len) {
fmt::format_to(error_msg, "{}",
"the length of input is too long than schema. ");
fmt::format_to(error_msg, "column_name: {}; ", desc->col_name());
fmt::format_to(error_msg, "input str: [{}] ", str_val.to_prefix(10));
fmt::format_to(error_msg, "schema length: {}; ", desc->type().len);
fmt::format_to(error_msg, "actual length: {}; ", str_val.size);
} else if (str_val.size > limit) {
fmt::format_to(
error_msg, "{}",
"the length of input string is too long than vec schema. ");
fmt::format_to(error_msg, "column_name: {}; ", desc->col_name());
fmt::format_to(error_msg, "input str: [{}] ", str_val.to_prefix(10));
fmt::format_to(error_msg, "schema length: {}; ", desc->type().len);
fmt::format_to(error_msg, "limit length: {}; ", limit);
fmt::format_to(error_msg, "actual length: {}; ", str_val.size);
}
error_msg.clear();
if (str_val.size > desc->type().len) {
fmt::format_to(error_msg, "{}",
"the length of input is too long than schema. ");
fmt::format_to(error_msg, "column_name: {}; ", desc->col_name());
fmt::format_to(error_msg, "input str: [{}] ", str_val.to_prefix(10));
fmt::format_to(error_msg, "schema length: {}; ", desc->type().len);
fmt::format_to(error_msg, "actual length: {}; ", str_val.size);
} else if (str_val.size > limit) {
fmt::format_to(error_msg, "{}",
"the length of input string is too long than vec schema. ");
fmt::format_to(error_msg, "column_name: {}; ", desc->col_name());
fmt::format_to(error_msg, "input str: [{}] ", str_val.to_prefix(10));
fmt::format_to(error_msg, "schema length: {}; ", desc->type().len);
fmt::format_to(error_msg, "limit length: {}; ", limit);
fmt::format_to(error_msg, "actual length: {}; ", str_val.size);
}

if (invalid) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
if (invalid) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
}
break;
}
case TYPE_DECIMALV2: {
auto column_decimal = const_cast<
vectorized::ColumnDecimal<vectorized::Decimal128>*>(
assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>*>(
real_column_ptr.get()));
break;
}
case TYPE_DECIMALV2: {
auto column_decimal = const_cast<vectorized::ColumnDecimal<vectorized::Decimal128>*>(
assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>*>(
real_column_ptr.get()));

for (int j = 0; j < num_rows; ++j) {
if (!filter_bitmap->Get(j)) {
auto dec_val = binary_cast<vectorized::Int128, DecimalV2Value>(
column_decimal->get_data()[j]);
error_msg.clear();
bool invalid = false;
for (int j = 0; j < num_rows; ++j) {
if (!filter_bitmap->Get(j)) {
auto dec_val = binary_cast<vectorized::Int128, DecimalV2Value>(
column_decimal->get_data()[j]);
error_msg.clear();
bool invalid = false;

if (dec_val.greater_than_scale(desc->type().scale)) {
auto code = dec_val.round(&dec_val, desc->type().scale, HALF_UP);
column_decimal->get_data()[j] =
binary_cast<DecimalV2Value, vectorized::Int128>(dec_val);
if (dec_val.greater_than_scale(desc->type().scale)) {
auto code = dec_val.round(&dec_val, desc->type().scale, HALF_UP);
column_decimal->get_data()[j] =
binary_cast<DecimalV2Value, vectorized::Int128>(dec_val);

if (code != E_DEC_OK) {
fmt::format_to(error_msg, "round one decimal failed.value={}; ",
dec_val.to_string());
invalid = true;
}
}
if (dec_val > _max_decimalv2_val[i] || dec_val < _min_decimalv2_val[i]) {
fmt::format_to(error_msg,
"decimal value is not valid for definition, column={}",
desc->col_name());
fmt::format_to(error_msg, ", value={}", dec_val.to_string());
fmt::format_to(error_msg, ", precision={}, scale={}; ",
desc->type().precision, desc->type().scale);
if (code != E_DEC_OK) {
fmt::format_to(error_msg, "round one decimal failed.value={}; ",
dec_val.to_string());
invalid = true;
}
}
if (dec_val > _max_decimalv2_val[i] || dec_val < _min_decimalv2_val[i]) {
fmt::format_to(error_msg,
"decimal value is not valid for definition, column={}",
desc->col_name());
fmt::format_to(error_msg, ", value={}", dec_val.to_string());
fmt::format_to(error_msg, ", precision={}, scale={}; ",
desc->type().precision, desc->type().scale);
invalid = true;
}

if (invalid) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
if (invalid) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
}
break;
}
default:
break;
}

// Dispose the nullable column not match problem here, convert to nullable column
if (desc->is_nullable() && !column_ptr) {
block->get_by_position(i).column = vectorized::make_nullable(column);
block->get_by_position(i).type =
vectorized::make_nullable(block->get_by_position(i).type);
}
break;
}
default:
break;
}

// Dispose the nullable column not match problem here, convert to not nullable column
if (!desc->is_nullable() && column_ptr) {
const auto& null_map = column_ptr->get_null_map_data();
for (int j = 0; j < null_map.size(); ++j) {
fmt::format_to(error_msg, "null value for not null column, column={}; ",
desc->col_name());
if (null_map[j] && !filter_bitmap->Get(j)) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
// Dispose the the column should do not contain the NULL value
// Only tow case:
// 1. column is nullable but the desc is not nullable
// 2. desc->type is BITMAP
if ((!desc->is_nullable() || desc->type() == TYPE_OBJECT) && column_ptr) {
const auto& null_map = column_ptr->get_null_map_data();
for (int j = 0; j < null_map.size(); ++j) {
fmt::format_to(error_msg,
"null value for not null column/or bitmap column, column={}; ",
desc->col_name());
if (null_map[j] && !filter_bitmap->Get(j)) {
RETURN_IF_ERROR(set_invalid_and_append_error_msg(j));
}
block->get_by_position(i).column = column_ptr->get_nested_column_ptr();
block->get_by_position(i).type =
(reinterpret_cast<const vectorized::DataTypeNullable*>(
block->get_by_position(i).type.get()))
->get_nested_type();
}
}
}
Expand All @@ -313,5 +287,26 @@ Status VOlapTableSink::_validate_data(RuntimeState* state, vectorized::Block* bl
return Status::OK();
}

void VOlapTableSink::_convert_to_dest_desc_block(doris::vectorized::Block* block) {
for (int i = 0; i < _output_tuple_desc->slots().size(); ++i) {
SlotDescriptor* desc = _output_tuple_desc->slots()[i];
if (desc->is_nullable() != block->get_by_position(i).type->is_nullable()) {
if (desc->is_nullable()) {
block->get_by_position(i).type =
vectorized::make_nullable(block->get_by_position(i).type);
block->get_by_position(i).column =
vectorized::make_nullable(block->get_by_position(i).column);
} else {
block->get_by_position(i).type = assert_cast<const vectorized::DataTypeNullable&>(
*block->get_by_position(i).type)
.get_nested_type();
block->get_by_position(i).column = assert_cast<const vectorized::ColumnNullable&>(
*block->get_by_position(i).column)
.get_nested_column_ptr();
}
}
}
}

} // namespace stream_load
} // namespace doris
6 changes: 5 additions & 1 deletion be/src/vec/sink/vtablet_sink.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,13 @@ class VOlapTableSink : public OlapTableSink {
Status _validate_data(RuntimeState* state, vectorized::Block* block, Bitmap* filter_bitmap,
int* filtered_rows, bool* stop_processing);

// some output column of output expr may have different nullable property with dest slot desc
// so here need to do the convert operation
void _convert_to_dest_desc_block(vectorized::Block* block);

VOlapTablePartitionParam* _vpartition = nullptr;
std::vector<vectorized::VExprContext*> _output_vexpr_ctxs;
};

} // namespace stream_load
} // namespace doris
} // namespace doris

0 comments on commit ca952c9

Please sign in to comment.