Skip to content

Commit

Permalink
[fix](memory) Fix free page not tracking correctly (apache#36768)
Browse files Browse the repository at this point in the history
## Proposed changes

apache#36235 introduced, index page is not freed in query thread, so saved
tracker ptr when alloc page, so that page only needs to be freed before
query ends.

```
F20240624 18:44:51.880192 162249 mem_tracker_limiter.cpp:125] mem tracker label: Query#Id=f0511107829e465e-a87a985234018b77, consumption: 94, peak consumption: 636540, mem tracker not equal to 0 when mem tracker destruct, this usually means that memory tracking is inaccurate and SCOPED_ATTACH_TASK and SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER are not used correctly. 1. For query and load, memory leaks may have occurred, it is expected that the query mem tracker will be bound to the thread context using SCOPED_ATTACH_TASK and SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER before all memory alloc and free. 2. If a memory alloc is recorded by this tracker, it is expected that be recorded in this tracker when memory is freed. 3. Merge the remaining memory tracking value by this tracker into Orphan, if you observe that Orphan is not equal to 0 in the mem tracker web or log, this indicates that there may be a memory leak. 4. If you need to transfer memory tracking value between two trackers, can use transfer_to..[Address Sanitizer]:
    0x605000160b30, size 46, strack trace:
        0#  Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long)
        1#  doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, bool, doris::segment_v2::PageTypePB)
        2#  doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*)
        3#  doris::segment_v2::IndexedColumnReader::read_page(doris::segment_v2::PagePointer const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*, doris::segment_v2::PageTypePB, doris::BlockCompressionCodec*, bool) const
        4#  doris::segment_v2::IndexedColumnReader::load_index_page(doris::segment_v2::PagePointerPB const&, doris::segment_v2::PageHandle*, doris::segment_v2::IndexPageReader*)
        5#  doris::segment_v2::IndexedColumnReader::load(bool, bool)
        6#  doris::segment_v2::BitmapIndexReader::_load(bool, bool, std::unique_ptr<doris::segment_v2::BitmapIndexPB, std::default_delete<doris::segment_v2::BitmapIndexPB> >)
        7#  doris::segment_v2::BitmapIndexReader::load(bool, bool)
        8#  doris::segment_v2::ColumnReader::_load_bitmap_index(bool, bool)
        9#  doris::segment_v2::ColumnReader::new_bitmap_index_iterator(doris::segment_v2::BitmapIndexIterator**)
        10# doris::segment_v2::Segment::new_bitmap_index_iterator(doris::TabletColumn const&, std::unique_ptr<doris::segment_v2::BitmapIndexIterator, std::default_delete<doris::segment_v2::BitmapIndexIterator> >*)
        11# doris::segment_v2::SegmentIterator::_init_bitmap_index_iterators()
        12# doris::segment_v2::SegmentIterator::init_iterators()
        13# doris::segment_v2::SegmentIterator::_init_impl(doris::StorageReadOptions const&)
        14# doris::segment_v2::SegmentIterator::init(doris::StorageReadOptions const&)
        15# doris::segment_v2::Segment::new_iterator(std::shared_ptr<doris::Schema const>, doris::StorageReadOptions const&, std::unique_ptr<doris::RowwiseIterator, std::default_delete<doris::RowwiseIterator> >*)
        16# doris::segment_v2::LazyInitSegmentIterator::init(doris::StorageReadOptions const&)
        17# doris::BetaRowsetReader::_init_iterator()
        18# doris::BetaRowsetReader::_init_iterator_once()
        19# doris::BetaRowsetReader::next_block(doris::vectorized::Block*)
        20# doris::vectorized::VCollectIterator::_topn_next(doris::vectorized::Block*)
        21# doris::vectorized::VCollectIterator::next(doris::vectorized::Block*)
        22# doris::vectorized::BlockReader::_direct_next_block(doris::vectorized::Block*, bool*)
        23# doris::vectorized::BlockReader::next_block_with_aggregation(doris::vectorized::Block*, bool*)
        24# doris::vectorized::NewOlapScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*)
        25# doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*)
        26# doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*)
        27# doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)
        28# std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&)
        29# doris::ThreadPool::dispatch_thread()
        30# doris::Thread::supervise_thread(void*)
```
  • Loading branch information
xinyiZzz committed Jul 12, 2024
1 parent 326b40c commit dcaee24
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 18 deletions.
25 changes: 9 additions & 16 deletions be/src/olap/page_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,14 @@
namespace doris {
template <typename TAllocator>
PageBase<TAllocator>::PageBase(size_t b, bool use_cache, segment_v2::PageTypePB page_type)
: LRUCacheValueBase(),
_size(b),
_capacity(b),
_use_cache(use_cache),
_page_type(page_type) {
if (_use_cache) {
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
StoragePageCache::instance()->mem_tracker(_page_type));
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity, ALLOCATOR_ALIGNMENT_16));
: LRUCacheValueBase(), _size(b), _capacity(b) {
if (use_cache) {
_mem_tracker_by_allocator = StoragePageCache::instance()->mem_tracker(page_type);
} else {
_mem_tracker_by_allocator = thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker();
}
{
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity, ALLOCATOR_ALIGNMENT_16));
}
}
Expand All @@ -44,13 +42,8 @@ template <typename TAllocator>
PageBase<TAllocator>::~PageBase() {
if (_data != nullptr) {
DCHECK(_capacity != 0 && _size != 0);
if (_use_cache) {
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
StoragePageCache::instance()->mem_tracker(_page_type));
TAllocator::free(_data, _capacity);
} else {
TAllocator::free(_data, _capacity);
}
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
TAllocator::free(_data, _capacity);
}
}

Expand Down
3 changes: 1 addition & 2 deletions be/src/olap/page_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,7 @@ class PageBase : private TAllocator, public LRUCacheValueBase {
// Effective size, smaller than capacity, such as data page remove checksum suffix.
size_t _size = 0;
size_t _capacity = 0;
bool _use_cache;
segment_v2::PageTypePB _page_type;
std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
};

using DataPage = PageBase<Allocator<false>>;
Expand Down

0 comments on commit dcaee24

Please sign in to comment.