From 8a2c098fffb6e65dd3c29905a0cb47682189fb0d Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Mon, 26 Aug 2024 09:05:34 -0700 Subject: [PATCH] [MC][NFC] Count pseudo probes and function records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-parse pseudo probes section counting the number of probes and function records. These numbers are used in follow-up diff to pre-allocate vectors for decoded probes and inline tree nodes. Additional benefit is avoiding error handling during parsing. This pre-parsing is fast: for a 404MiB .pseudo_probe section with 43373881 probes and 25228770 function records, it only takes 0.68±0.01s. The total time of buildAddress2ProbeMap is 21s. Reviewers: dcci, maksfb, rafaelauler, wlei-llvm, ayermolo Reviewed By: wlei-llvm Pull Request: https://github.com/llvm/llvm-project/pull/102774 --- bolt/lib/Rewrite/PseudoProbeRewriter.cpp | 1 - llvm/include/llvm/MC/MCPseudoProbe.h | 6 + llvm/lib/MC/MCPseudoProbe.cpp | 143 +++++++++++++++++------ 3 files changed, 113 insertions(+), 37 deletions(-) diff --git a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp index 886bbdbf9d686e..37a5b937ebcaa3 100644 --- a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp +++ b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp @@ -143,7 +143,6 @@ void PseudoProbeRewriter::parsePseudoProbe() { if (!ProbeDecoder.buildAddress2ProbeMap( reinterpret_cast(Contents.data()), Contents.size(), GuidFilter, FuncStartAddrs)) { - ProbeDecoder.getAddress2ProbesMap().clear(); errs() << "BOLT-WARNING: fail in building Address2ProbeMap\n"; return; } diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h index 13ad1c38f3b3b0..3dd10c0717679b 100644 --- a/llvm/include/llvm/MC/MCPseudoProbe.h +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -370,6 +370,12 @@ class MCPseudoProbeDecoder { // Decode pseudo_probe_desc section to build GUID to PseudoProbeFuncDesc map. bool buildGUID2FuncDescMap(const uint8_t *Start, std::size_t Size); + // Decode pseudo_probe section to count the number of probes and inlined + // function records for each function record. + template + bool countRecords(bool &Discard, uint32_t &ProbeCount, uint32_t &InlinedCount, + const Uint64Set &GuidFilter); + // Decode pseudo_probe section to build address to probes map for specifed // functions only. bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size, diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp index 3f6f605149b479..77ac1fee4120f5 100644 --- a/llvm/lib/MC/MCPseudoProbe.cpp +++ b/llvm/lib/MC/MCPseudoProbe.cpp @@ -18,6 +18,7 @@ #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/MD5.h" #include "llvm/Support/raw_ostream.h" @@ -429,17 +430,11 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap( Index = Cur->getChildren().size(); } else { // Read inline site for inlinees - auto ErrorOrIndex = readUnsignedNumber(); - if (!ErrorOrIndex) - return false; - Index = std::move(*ErrorOrIndex); + Index = cantFail(errorOrToExpected(readUnsignedNumber())); } // Read guid - auto ErrorOrCurGuid = readUnencodedNumber(); - if (!ErrorOrCurGuid) - return false; - uint64_t Guid = std::move(*ErrorOrCurGuid); + uint64_t Guid = cantFail(errorOrToExpected(readUnencodedNumber())); // Decide if top-level node should be disgarded. if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid)) @@ -457,41 +452,27 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap( } // Read number of probes in the current node. - auto ErrorOrNodeCount = readUnsignedNumber(); - if (!ErrorOrNodeCount) - return false; - uint32_t NodeCount = std::move(*ErrorOrNodeCount); + uint32_t NodeCount = + cantFail(errorOrToExpected(readUnsignedNumber())); // Read number of direct inlinees - auto ErrorOrCurChildrenToProcess = readUnsignedNumber(); - if (!ErrorOrCurChildrenToProcess) - return false; + uint32_t ChildrenToProcess = + cantFail(errorOrToExpected(readUnsignedNumber())); // Read all probes in this node for (std::size_t I = 0; I < NodeCount; I++) { // Read index - auto ErrorOrIndex = readUnsignedNumber(); - if (!ErrorOrIndex) - return false; - uint32_t Index = std::move(*ErrorOrIndex); + uint32_t Index = + cantFail(errorOrToExpected(readUnsignedNumber())); // Read type | flag. - auto ErrorOrValue = readUnencodedNumber(); - if (!ErrorOrValue) - return false; - uint8_t Value = std::move(*ErrorOrValue); + uint8_t Value = cantFail(errorOrToExpected(readUnencodedNumber())); uint8_t Kind = Value & 0xf; uint8_t Attr = (Value & 0x70) >> 4; // Read address uint64_t Addr = 0; if (Value & 0x80) { - auto ErrorOrOffset = readSignedNumber(); - if (!ErrorOrOffset) - return false; - int64_t Offset = std::move(*ErrorOrOffset); + int64_t Offset = cantFail(errorOrToExpected(readSignedNumber())); Addr = LastAddr + Offset; } else { - auto ErrorOrAddr = readUnencodedNumber(); - if (!ErrorOrAddr) - return false; - Addr = std::move(*ErrorOrAddr); + Addr = cantFail(errorOrToExpected(readUnencodedNumber())); if (isSentinelProbe(Attr)) { // For sentinel probe, the addr field actually stores the GUID of the // split function. Convert it to the real address. @@ -508,10 +489,8 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap( uint32_t Discriminator = 0; if (hasDiscriminator(Attr)) { - auto ErrorOrDiscriminator = readUnsignedNumber(); - if (!ErrorOrDiscriminator) - return false; - Discriminator = std::move(*ErrorOrDiscriminator); + Discriminator = + cantFail(errorOrToExpected(readUnsignedNumber())); } if (Cur && !isSentinelProbe(Attr)) { @@ -524,17 +503,109 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap( LastAddr = Addr; } - uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); for (uint32_t I = 0; I < ChildrenToProcess; I++) { buildAddress2ProbeMap(Cur, LastAddr, GuidFilter, FuncStartAddrs); } + return true; +} + +template +bool MCPseudoProbeDecoder::countRecords(bool &Discard, uint32_t &ProbeCount, + uint32_t &InlinedCount, + const Uint64Set &GuidFilter) { + if (!IsTopLevelFunc) + // Read inline site for inlinees + if (!readUnsignedNumber()) + return false; + + // Read guid + auto ErrorOrCurGuid = readUnencodedNumber(); + if (!ErrorOrCurGuid) + return false; + uint64_t Guid = std::move(*ErrorOrCurGuid); + + // Decide if top-level node should be disgarded. + if (IsTopLevelFunc) { + Discard = !GuidFilter.empty() && !GuidFilter.count(Guid); + if (!Discard) + // Allocate an entry for top-level function record. + ++InlinedCount; + } + + // Read number of probes in the current node. + auto ErrorOrNodeCount = readUnsignedNumber(); + if (!ErrorOrNodeCount) + return false; + uint32_t NodeCount = std::move(*ErrorOrNodeCount); + uint32_t CurrentProbeCount = 0; + + // Read number of direct inlinees + auto ErrorOrCurChildrenToProcess = readUnsignedNumber(); + if (!ErrorOrCurChildrenToProcess) + return false; + uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess); + + // Read all probes in this node + for (std::size_t I = 0; I < NodeCount; I++) { + // Read index + if (!readUnsignedNumber()) + return false; + + // Read type | flag. + auto ErrorOrValue = readUnencodedNumber(); + if (!ErrorOrValue) + return false; + uint8_t Value = std::move(*ErrorOrValue); + + uint8_t Attr = (Value & 0x70) >> 4; + if (Value & 0x80) { + // Offset + if (!readSignedNumber()) + return false; + } else { + // Addr + if (!readUnencodedNumber()) + return false; + } + + if (hasDiscriminator(Attr)) + // Discriminator + if (!readUnsignedNumber()) + return false; + + if (!Discard && !isSentinelProbe(Attr)) + ++CurrentProbeCount; + } + if (!Discard) { + ProbeCount += CurrentProbeCount; + InlinedCount += ChildrenToProcess; + } + + for (uint32_t I = 0; I < ChildrenToProcess; I++) + if (!countRecords(Discard, ProbeCount, InlinedCount, GuidFilter)) + return false; return true; } bool MCPseudoProbeDecoder::buildAddress2ProbeMap( const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs) { + // For function records in the order of their appearance in the encoded data + // (DFS), count the number of contained probes and inlined function records. + uint32_t ProbeCount = 0; + uint32_t InlinedCount = 0; + uint32_t TopLevelFuncs = 0; + Data = Start; + End = Data + Size; + bool Discard = false; + while (Data < End) { + if (!countRecords(Discard, ProbeCount, InlinedCount, GuidFilter)) + return false; + TopLevelFuncs += !Discard; + } + assert(Data == End && "Have unprocessed data in pseudo_probe section"); + Data = Start; End = Data + Size; uint64_t LastAddr = 0;