diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 480a3c6e46bd263..92f2431a57b56de 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -394,6 +394,7 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER") if (USE_LLD) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}") endif () if (USE_LIBCPP AND COMPILER_CLANG) diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index b6b96e279c7e80d..7292c3498cda757 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -359,7 +359,8 @@ void DeltaWriter::_build_current_tablet_schema(const POlapTableSchemaParam& ptab const TabletSchema& ori_tablet_schema) { *_tablet_schema = ori_tablet_schema; //new tablet schame if new table - if (ptable_schema_param.columns_size() != 0 && ptable_schema_param.columns(0).unique_id() >= 0) { + if (ptable_schema_param.columns_size() != 0 && + ptable_schema_param.columns(0).unique_id() >= 0) { _tablet_schema->build_current_tablet_schema(ptable_schema_param, ori_tablet_schema); } } diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index c0b5fb004ee72a0..7e636e524450937 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1841,8 +1841,13 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams std::unique_ptr rowset_writer; Status status = new_tablet->create_rowset_writer( rs_reader->version(), VISIBLE, +<<<<<<< HEAD rs_reader->rowset()->rowset_meta()->segments_overlap(), &new_tablet->tablet_schema(), &rowset_writer); +======= + rs_reader->rowset()->rowset_meta()->segments_overlap(), &new_tablet->tablet_schema(), + &rowset_writer); +>>>>>>> df84fd6ba ([Schema Change] Merge selectdb/master (#90)) if (!status.ok()) { res = Status::OLAPInternalError(OLAP_ERR_ROWSET_BUILDER_INIT); goto PROCESS_ALTER_EXIT; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index b2cfd0e4377f79e..48d7d1817735564 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1589,4 +1589,101 @@ const TabletSchema& Tablet::tablet_schema() const { return *last_rowset->tablet_schema(); } +Status Tablet::create_initial_rowset(const int64_t req_version) { + Status res = Status::OK(); + if (req_version < 1) { + LOG(WARNING) << "init version of tablet should at least 1. req.ver=" << req_version; + return Status::OLAPInternalError(OLAP_ERR_CE_CMD_PARAMS_ERROR); + } + Version version(0, req_version); + RowsetSharedPtr new_rowset; + do { + // there is no data in init rowset, so overlapping info is unknown. + std::unique_ptr rs_writer; + res = create_rowset_writer(version, VISIBLE, OVERLAP_UNKNOWN, &tablet_schema(), &rs_writer); + if (!res.ok()) { + LOG(WARNING) << "failed to init rowset writer for tablet " << full_name(); + break; + } + res = rs_writer->flush(); + if (!res.ok()) { + LOG(WARNING) << "failed to flush rowset writer for tablet " << full_name(); + break; + } + + new_rowset = rs_writer->build(); + res = add_rowset(new_rowset); + if (!res.ok()) { + LOG(WARNING) << "failed to add rowset for tablet " << full_name(); + break; + } + } while (0); + + // Unregister index and delete files(index and data) if failed + if (!res.ok()) { + LOG(WARNING) << "fail to create initial rowset. res=" << res << " version=" << req_version; + StorageEngine::instance()->add_unused_rowset(new_rowset); + return res; + } + set_cumulative_layer_point(req_version + 1); + return res; +} + +Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, + const SegmentsOverlapPB& overlap, + const TabletSchema* tablet_schema, + std::unique_ptr* rowset_writer) { + RowsetWriterContext context; + context.version = version; + context.rowset_state = rowset_state; + context.segments_overlap = overlap; + context.tablet_schema = tablet_schema; + _init_context_common_fields(context); + return RowsetFactory::create_rowset_writer(context, rowset_writer); +} + +Status Tablet::create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, + const RowsetStatePB& rowset_state, + const SegmentsOverlapPB& overlap, + const TabletSchema* tablet_schema, + std::unique_ptr* rowset_writer) { + RowsetWriterContext context; + context.txn_id = txn_id; + context.load_id = load_id; + context.rowset_state = rowset_state; + context.segments_overlap = overlap; + context.tablet_schema = tablet_schema; + _init_context_common_fields(context); + return RowsetFactory::create_rowset_writer(context, rowset_writer); +} + +void Tablet::_init_context_common_fields(RowsetWriterContext& context) { + context.rowset_id = StorageEngine::instance()->next_rowset_id(); + context.tablet_uid = tablet_uid(); + + context.tablet_id = tablet_id(); + context.partition_id = partition_id(); + context.tablet_schema_hash = schema_hash(); + context.rowset_type = tablet_meta()->preferred_rowset_type(); + // Alpha Rowset will be removed in the future, so that if the tablet's default rowset type is + // alpah rowset, then set the newly created rowset to storage engine's default rowset. + if (context.rowset_type == ALPHA_ROWSET) { + context.rowset_type = StorageEngine::instance()->default_rowset_type(); + } + context.path_desc = tablet_path_desc(); + context.data_dir = data_dir(); +} + +Status Tablet::create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset) { + return RowsetFactory::create_rowset(&tablet_schema(), tablet_path_desc(), rowset_meta, rowset); +} + +std::shared_ptr& Tablet::get_compaction_mem_tracker(CompactionType compaction_type) { + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + return _cumulative_compaction->get_mem_tracker(); + } else { + return _base_compaction->get_mem_tracker(); + } +} + } // namespace doris diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 6c0cf6ec8dd9635..81ed5e0a5712fcb 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -34,6 +34,7 @@ #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_reader.h" #include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" #include "olap/tuple.h" #include "olap/utils.h" #include "olap/version_graph.h" @@ -275,6 +276,16 @@ class Tablet : public BaseTablet { Status create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); const TabletSchema& tablet_schema() const override; + Status create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, + const SegmentsOverlapPB& overlap, const TabletSchema* tablet_schema, + std::unique_ptr* rowset_writer); + + Status create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, + const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, + const TabletSchema* tablet_schema, + std::unique_ptr* rowset_writer); + + Status create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); private: Status _init_once_action(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 506a9a32f61ce47..82784e573d82709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -369,6 +369,7 @@ public TColumn toThrift() { toChildrenThrift(this, tColumn); tColumn.setColUniqueId(uniqueId); + // ATTN: // Currently, this `toThrift()` method is only used from CreateReplicaTask. // And CreateReplicaTask does not need `defineExpr` field. diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 5a3c482262eeabe..920665a445d1dea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -659,6 +659,16 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_CREATE_POLICY: { + data = Policy.read(in); + isRead = true; + break; + } + case OperationType.OP_DROP_POLICY: { + data = DropPolicyLog.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 2201c216545d3a5..eab20bdffab08e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -827,6 +827,16 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { catalog.replayModifyTableAddOrDropColumns(info); break; } + case OperationType.OP_CREATE_POLICY: { + Policy log = (Policy) journal.getData(); + catalog.getPolicyMgr().replayCreate(log); + break; + } + case OperationType.OP_DROP_POLICY: { + DropPolicyLog log = (DropPolicyLog) journal.getData(); + catalog.getPolicyMgr().replayDrop(log); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -1445,4 +1455,12 @@ public void logDropPolicy(DropPolicyLog log) { public void logModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) { logEdit(OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS, info); } + + public void logCreatePolicy(Policy policy) { + logEdit(OperationType.OP_CREATE_POLICY, policy); + } + + public void logDropPolicy(DropPolicyLog log) { + logEdit(OperationType.OP_DROP_POLICY, log); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 06023f4b4f69824..cc617ff8cf9d930 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -222,7 +222,8 @@ public class OperationType { // policy 310-320 public static final short OP_CREATE_POLICY = 310; public static final short OP_DROP_POLICY = 311; - //schema change for add and drop columns 320-329 + + // schema change for add and drop columns 320-329 public static final short OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS = 320; // get opcode name by op codeStri