Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](catalog) modify some meta cache logic #38506

Merged
merged 2 commits into from
Aug 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -1979,16 +1979,20 @@ public class Config extends ConfigBase {
* Max cache num of hive partition.
* Decrease this value if FE's memory is small
*/
@ConfField(mutable = false, masterOnly = false)
public static long max_hive_partition_cache_num = 100000;
@ConfField(description = {"Hive Metastore 表级别分区缓存的最大数量。",
"Max cache number of partition at table level in Hive Metastore."})
public static long max_hive_partition_cache_num = 10000;

@ConfField(mutable = false, masterOnly = false, description = {"Hive表名缓存的最大数量。",
"Max cache number of hive table name list."})
public static long max_hive_table_cache_num = 1000;
@ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。",
"Max cache number of hudi/iceberg table."})
public static long max_external_table_cache_num = 1000;

@ConfField(mutable = false, masterOnly = false, description = {
"Hive分区表缓存的最大数量", "Max cache number of hive partition table"
})
@ConfField(description = {"External Catalog 中,Database 和 Table 的实例缓存的最大数量。",
"Max cache number of database and table instance in external catalog."})
public static long max_meta_object_cache_num = 1000;

@ConfField(description = {"Hive分区表缓存的最大数量",
"Max cache number of hive partition table"})
public static long max_hive_partition_table_cache_num = 1000;

@ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量,-1代表没有限制。",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ public final synchronized void makeSureInitialized() {
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_table_cache_num,
Config.max_meta_object_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)),
Expand Down Expand Up @@ -660,8 +660,6 @@ protected ExternalDatabase<? extends ExternalTable> buildDbForInit(String dbName
return new IcebergExternalDatabase(this, dbId, dbName);
case MAX_COMPUTE:
return new MaxComputeExternalDatabase(this, dbId, dbName);
//case HUDI:
//return new HudiExternalDatabase(this, dbId, dbName);
case LAKESOUL:
return new LakeSoulExternalDatabase(this, dbId, dbName);
case TEST:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public final synchronized void makeSureInitialized() {
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_table_cache_num,
Config.max_meta_object_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
buildTableForInit(tableName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public void setNewFileCache() {

CacheFactory fileCacheFactory = new CacheFactory(
OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE
? fileMetaCacheTtlSecond : 86400L),
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ public HudiCachedPartitionProcessor(long catalogId, ExecutorService executor) {
this.catalogId = catalogId;
this.executor = executor;
CacheFactory partitionCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,17 @@ public class IcebergMetadataCache {

public IcebergMetadataCache(ExecutorService executor) {
CacheFactory snapshotListCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor);

CacheFactory tableCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public MetaCache(String name,
CacheFactory namesCacheFactory = new CacheFactory(
expireAfterWriteSec,
refreshAfterWriteSec,
maxSize,
1, // names cache has one and only one entry
true,
null);
CacheFactory objCacheFactory = new CacheFactory(
Expand Down
Loading