JIT: Return multiple likely classes in getLikelyClass (for better GDV)

src/coreclr/inc/jiteeversionguid.h

@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier =  { /* 5ed35c58-857b-48dd-a818-7c0136dc9f73 */
-    0x5ed35c58,
-    0x857b,
-    0x48dd,
-    {0xa8, 0x18, 0x7c, 0x01, 0x36, 0xdc, 0x9f, 0x73}
+constexpr GUID JITEEVersionIdentifier =  { /* 4ef06a0e-e58d-4796-abb7-0cda6460610c */
+    0x4ef06a0e,
+    0xe58d,
+    0x4796,
+    {0xab, 0xb7, 0xc, 0xda, 0x64, 0x60, 0x61, 0xc}
 };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/jit/ClrJit.PAL.exports

@@ -1,3 +1,3 @@
 getJit
 jitStartup
-getLikelyClass
+getLikelyClasses

src/coreclr/jit/ClrJit.exports

@@ -4,4 +4,4 @@
 EXPORTS
     getJit
     jitStartup
-    getLikelyClass
+    getLikelyClasses

src/coreclr/jit/importer.cpp

@@ -21338,13 +21338,17 @@ void Compiler::impDevirtualizeCall(GenTreeCall*            call,
         (isExact || objClassIsFinal) && (fgPgoSource == ICorJitInfo::PgoSource::Dynamic) && !compIsForInlining();
     if (JitConfig.JitCrossCheckDevirtualizationAndPGO() && canSensiblyCheck)
     {
-        unsigned likelihood      = 0;
-        unsigned numberOfClasses = 0;
+        // We only can handle a single likely class for now
+        const int         maxLikelyClasses = 1;
+        LikelyClassRecord likelyClasses[maxLikelyClasses];
 
-        CORINFO_CLASS_HANDLE likelyClass =
-            getLikelyClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, &likelihood, &numberOfClasses);
+        UINT32 numberOfClasses =
+            getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset);
+        UINT32 likelihood = likelyClasses[0].likelihood;
 
-        if (likelyClass != NO_CLASS_HANDLE)
+        CORINFO_CLASS_HANDLE likelyClass = likelyClasses[0].clsHandle;
+
+        if (numberOfClasses > 0)
         {
             // PGO had better agree the class we devirtualized to is plausible.
             //
@@ -21905,6 +21909,9 @@ void Compiler::considerGuardedDevirtualization(
 
     bool doRandomDevirt = false;
 
+    const int         maxLikelyClasses = 32;
+    LikelyClassRecord likelyClasses[maxLikelyClasses];
+
 #ifdef DEBUG
     // Optional stress mode to pick a random known class, rather than
     // the most likely known class.
@@ -21917,24 +21924,40 @@ void Compiler::considerGuardedDevirtualization(
         //
         CLRRandom* const random =
             impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization());
-        likelihood      = 100;
-        numberOfClasses = 1;
-        likelyClass     = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random);
+        likelyClasses[0].clsHandle  = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random);
+        likelyClasses[0].likelihood = 100;
+        if (likelyClasses[0].clsHandle != NO_CLASS_HANDLE)
+        {
+            numberOfClasses = 1;
+        }
     }
     else
 #endif
     {
-        likelyClass = getLikelyClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, &likelihood, &numberOfClasses);
+        numberOfClasses =
+            getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset);
     }
 
-    if (likelyClass == NO_CLASS_HANDLE)
+    // For now we only use the most popular type
+
+    likelihood  = likelyClasses[0].likelihood;
+    likelyClass = likelyClasses[0].clsHandle;
+
+    if (numberOfClasses < 1)
     {
         JITDUMP("No likely class, sorry\n");
         return;
     }
 
-    JITDUMP("%s class for %p (%s) is %p (%s) [likelihood:%u classes seen:%u]\n", doRandomDevirt ? "Random" : "Likely",
-            dspPtr(objClass), objClassName, likelyClass, eeGetClassName(likelyClass), likelihood, numberOfClasses);
+    assert(likelyClass != NO_CLASS_HANDLE);
+
+    // Print all likely classes
+    JITDUMP("%s classes for %p (%s):\n", doRandomDevirt ? "Random" : "Likely", dspPtr(objClass), objClassName)
+    for (UINT32 i = 0; i < numberOfClasses; i++)
+    {
+        JITDUMP("  %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].clsHandle,
+                eeGetClassName(likelyClasses[i].clsHandle), likelyClasses[i].likelihood);
+    }
 
     // Todo: a more advanced heuristic using likelihood, number of
     // classes, and the profile count for this block.

src/coreclr/jit/jit.h

@@ -633,10 +633,10 @@ inline size_t unsigned_abs(ssize_t x)
 
 /*****************************************************************************/
 
-#if CALL_ARG_STATS || COUNT_BASIC_BLOCKS || COUNT_LOOPS || EMITTER_STATS || MEASURE_NODE_SIZE || MEASURE_MEM_ALLOC
-
 #define HISTOGRAM_MAX_SIZE_COUNT 64
 
+#if CALL_ARG_STATS || COUNT_BASIC_BLOCKS || COUNT_LOOPS || EMITTER_STATS || MEASURE_NODE_SIZE || MEASURE_MEM_ALLOC
+
 class Histogram
 {
 public:
@@ -836,12 +836,18 @@ T dspOffset(T o)
 
 #endif // !defined(DEBUG)
 
-extern "C" CORINFO_CLASS_HANDLE WINAPI getLikelyClass(ICorJitInfo::PgoInstrumentationSchema* schema,
-                                                      UINT32                                 countSchemaItems,
-                                                      BYTE*                                  pInstrumentationData,
-                                                      int32_t                                ilOffset,
-                                                      UINT32*                                pLikelihood,
-                                                      UINT32*                                pNumberOfClasses);
+struct LikelyClassRecord
+{
+    CORINFO_CLASS_HANDLE clsHandle;
+    UINT32               likelihood;
+};
+
+extern "C" UINT32 WINAPI getLikelyClasses(LikelyClassRecord*                     pLikelyClasses,
+                                          UINT32                                 maxLikelyClasses,
+                                          ICorJitInfo::PgoInstrumentationSchema* schema,
+                                          UINT32                                 countSchemaItems,
+                                          BYTE*                                  pInstrumentationData,
+                                          int32_t                                ilOffset);
 
 /*****************************************************************************/
 #endif //_JIT_H_

src/coreclr/jit/likelyclass.cpp

@@ -18,6 +18,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #pragma hdrstop
 #endif
 
+#include <algorithm.h>
+
 #ifndef DLLEXPORT
 #define DLLEXPORT
 #endif // !DLLEXPORT
@@ -44,7 +46,7 @@ struct LikelyClassHistogram
     // Rough guess at count of unknown types
     unsigned m_unknownTypes;
     // Histogram entries, in no particular order.
-    LikelyClassHistogramEntry m_histogram[64];
+    LikelyClassHistogramEntry m_histogram[HISTOGRAM_MAX_SIZE_COUNT];
     UINT32                    countHistogramElements = 0;
 
     LikelyClassHistogramEntry HistogramEntryAt(unsigned index)
@@ -104,18 +106,24 @@ LikelyClassHistogram::LikelyClassHistogram(INT_PTR* histogramEntries, unsigned e
 }
 
 //------------------------------------------------------------------------
-// getLikelyClass: find class profile data for an IL offset, and return the most likely class
+// getLikelyClasses: find class profile data for an IL offset, and return the most likely classes
 //
 // Arguments:
+//    pLikelyClasses - [OUT] array of likely classes sorted by likelihood (descending). It must be
+//                     at least of 'maxLikelyClasses' (next argument) length.
+//                     The array consists of pairs "clsHandle - likelihood" ordered by likelihood
+//                     (descending) where likelihood can be any value in [0..100] range. clsHandle
+//                     is never null for [0..<return value of this function>) range, Items in
+//                     [<return value of this function>..maxLikelyClasses) are zeroed if the number
+//                     of classes seen is less than maxLikelyClasses provided.
+//    maxLikelyClasses - limit for likely classes to output
 //    schema - profile schema
 //    countSchemaItems - number of items in the schema
 //    pInstrumentationData - associated data
 //    ilOffset - il offset of the callvirt
-//    pLikelihood - [OUT] likelihood of observing that entry [0...100]
-//    pNumberOfClasses - [OUT] estimated number of classes seen at runtime
 //
 // Returns:
-//    Class handle for the most likely class, or nullptr
+//    Estimated number of classes seen at runtime
 //
 // Notes:
 //    A "monomorphic" call site will return likelihood 100 and number of entries = 1.
@@ -126,37 +134,40 @@ LikelyClassHistogram::LikelyClassHistogram(INT_PTR* histogramEntries, unsigned e
 //   This code can runs without a jit instance present, so JITDUMP and related
 //   cannot be used.
 //
-extern "C" DLLEXPORT CORINFO_CLASS_HANDLE WINAPI getLikelyClass(ICorJitInfo::PgoInstrumentationSchema* schema,
-                                                                UINT32                                 countSchemaItems,
-                                                                BYTE*   pInstrumentationData,
-                                                                int32_t ilOffset,
-                                                                UINT32* pLikelihood,
-                                                                UINT32* pNumberOfClasses)
+extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*                     pLikelyClasses,
+                                                    UINT32                                 maxLikelyClasses,
+                                                    ICorJitInfo::PgoInstrumentationSchema* schema,
+                                                    UINT32                                 countSchemaItems,
+                                                    BYTE*                                  pInstrumentationData,
+                                                    int32_t                                ilOffset)
 {
-    *pLikelihood      = 0;
-    *pNumberOfClasses = 0;
+    ZeroMemory(pLikelyClasses, maxLikelyClasses * sizeof(*pLikelyClasses));
 
-    if (schema == NULL)
-        return NULL;
+    if (schema == nullptr)
+    {
+        return 0;
+    }
 
     for (COUNT_T i = 0; i < countSchemaItems; i++)
     {
-        if (schema[i].ILOffset != (int32_t)ilOffset)
+        if (schema[i].ILOffset != ilOffset)
             continue;
 
         if ((schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass) &&
             (schema[i].Count == 1))
         {
-            *pNumberOfClasses = (UINT32)schema[i].Other >> 8;
-            *pLikelihood      = (UINT32)(schema[i].Other & 0xFF);
-            INT_PTR result    = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
+            INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
             if (ICorJitInfo::IsUnknownTypeHandle(result))
-                return NULL;
-            else
-                return (CORINFO_CLASS_HANDLE)result;
+            {
+                return 0;
+            }
+            assert(result != 0); // we don't expect zero in GetLikelyClass
+            pLikelyClasses[0].likelihood = (UINT32)(schema[i].Other & 0xFF);
+            pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)result;
+            return 1;
         }
 
-        bool isHistogramCount =
+        const bool isHistogramCount =
             (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) ||
             (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount);
 
@@ -168,84 +179,100 @@ extern "C" DLLEXPORT CORINFO_CLASS_HANDLE WINAPI getLikelyClass(ICorJitInfo::Pgo
             LikelyClassHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count);
 
             // Use histogram count as number of classes estimate
-            //
-            *pNumberOfClasses = (uint32_t)h.countHistogramElements + h.m_unknownTypes;
-
             // Report back what we've learned
             // (perhaps, use count to augment likelihood?)
             //
-            switch (*pNumberOfClasses)
+            switch (h.countHistogramElements)
             {
                 case 0:
-                {
-                    return NULL;
-                }
-                break;
+                    return 0;
 
                 case 1:
                 {
-                    if (ICorJitInfo::IsUnknownTypeHandle(h.HistogramEntryAt(0).m_mt))
+                    LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
+                    // Fast path for monomorphic cases
+                    if (ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
                     {
-                        return NULL;
+                        return 0;
                     }
-                    *pLikelihood = 100;
-                    return (CORINFO_CLASS_HANDLE)h.HistogramEntryAt(0).m_mt;
+                    pLikelyClasses[0].likelihood = 100;
+                    pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)hist0.m_mt;
+                    return 1;
                 }
-                break;
 
                 case 2:
                 {
-                    if ((h.HistogramEntryAt(0).m_count >= h.HistogramEntryAt(1).m_count) &&
-                        !ICorJitInfo::IsUnknownTypeHandle(h.HistogramEntryAt(0).m_mt))
-                    {
-                        *pLikelihood = (100 * h.HistogramEntryAt(0).m_count) / h.m_totalCount;
-                        return (CORINFO_CLASS_HANDLE)h.HistogramEntryAt(0).m_mt;
-                    }
-                    else if (!ICorJitInfo::IsUnknownTypeHandle(h.HistogramEntryAt(1).m_mt))
+                    LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
+                    LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1);
+                    // Fast path for two classes
+                    if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
                     {
-                        *pLikelihood = (100 * h.HistogramEntryAt(1).m_count) / h.m_totalCount;
-                        return (CORINFO_CLASS_HANDLE)h.HistogramEntryAt(1).m_mt;
+                        pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount;
+                        pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)hist0.m_mt;
+
+                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
+                        {
+                            pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount;
+                            pLikelyClasses[1].clsHandle  = (CORINFO_CLASS_HANDLE)hist1.m_mt;
+                            return 2;
+                        }
+                        return 1;
                     }
-                    else
+
+                    if (!ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
                     {
-                        return NULL;
+                        pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount;
+                        pLikelyClasses[0].clsHandle  = (CORINFO_CLASS_HANDLE)hist1.m_mt;
+
+                        if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
+                        {
+                            pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount;
+                            pLikelyClasses[1].clsHandle  = (CORINFO_CLASS_HANDLE)hist0.m_mt;
+                            return 2;
+                        }
+                        return 1;
                     }
+                    return 0;
                 }
-                break;
 
                 default:
                 {
-                    // Find maximum entry and return it
-                    //
-                    unsigned maxKnownIndex = 0;
-                    unsigned maxKnownCount = 0;
+                    LikelyClassHistogramEntry sortedEntries[HISTOGRAM_MAX_SIZE_COUNT];
 
+                    // Since this method can be invoked without a jit instance we can't use any existing allocators
+                    unsigned knownHandles = 0;
                     for (unsigned m = 0; m < h.countHistogramElements; m++)
                     {
-                        if ((h.HistogramEntryAt(m).m_count > maxKnownCount) &&
-                            !ICorJitInfo::IsUnknownTypeHandle(h.HistogramEntryAt(m).m_mt))
+                        LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m);
+                        if (!ICorJitInfo::IsUnknownTypeHandle(hist.m_mt))
                         {
-                            maxKnownIndex = m;
-                            maxKnownCount = h.HistogramEntryAt(m).m_count;
+                            sortedEntries[knownHandles++] = hist;
                         }
                     }
 
-                    if (maxKnownCount > 0)
+                    // sort by m_count (descending)
+                    jitstd::sort(sortedEntries, sortedEntries + knownHandles,
+                                 [](const LikelyClassHistogramEntry& h1, const LikelyClassHistogramEntry& h2) -> bool {
+                                     return h1.m_count > h2.m_count;
+                                 });
+
+                    const UINT32 numberOfClasses = min(knownHandles, maxLikelyClasses);
+
+                    for (size_t hIdx = 0; hIdx < numberOfClasses; hIdx++)
                     {
-                        *pLikelihood = (100 * maxKnownCount) / h.m_totalCount;
-                        return (CORINFO_CLASS_HANDLE)h.HistogramEntryAt(maxKnownIndex).m_mt;
+                        LikelyClassHistogramEntry const hc = sortedEntries[hIdx];
+                        pLikelyClasses[hIdx].clsHandle     = (CORINFO_CLASS_HANDLE)hc.m_mt;
+                        pLikelyClasses[hIdx].likelihood    = hc.m_count * 100 / h.m_totalCount;
                     }
-
-                    return NULL;
+                    return numberOfClasses;
                 }
-                break;
             }
         }
     }
 
     // Failed to find histogram data for this method
     //
-    return NULL;
+    return 0;
 }
 
 //------------------------------------------------------------------------