Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Bulk copy of byrefs #101761

Merged
merged 5 commits into from
May 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/coreclr/inc/corinfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ enum CorInfoHelpFunc
CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap.

CORINFO_HELP_ASSIGN_BYREF,
CORINFO_HELP_ASSIGN_STRUCT,
CORINFO_HELP_BULK_WRITEBARRIER,


/* Accessing fields */
Expand Down
10 changes: 5 additions & 5 deletions src/coreclr/inc/jiteeversionguid.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
#define GUID_DEFINED
#endif // !GUID_DEFINED

constexpr GUID JITEEVersionIdentifier = { /* 32d71f8e-c1f5-41cb-88cc-4e8504cabf40 */
0x32d71f8e,
0xc1f5,
0x41cb,
{0x88, 0xcc, 0x4e, 0x85, 0x04, 0xca, 0xbf, 0x40}
constexpr GUID JITEEVersionIdentifier = { /* bd8c41d4-8531-49c1-a600-0ae9bfe05de1 */
0xbd8c41d4,
0x8531,
0x49c1,
{0xa6, 0x00, 0x0a, 0xe9, 0xbf, 0xe0, 0x5d, 0xe1}
};

//////////////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down
3 changes: 1 addition & 2 deletions src/coreclr/inc/jithelpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,7 @@
JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY)

DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB)

JITHELPER(CORINFO_HELP_ASSIGN_STRUCT, JIT_StructWriteBarrier,CORINFO_HELP_SIG_4_STACK)
DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, CORINFO_HELP_SIG_REG_ONLY)

// Accessing fields
JITHELPER(CORINFO_HELP_GETFIELD8, JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY)
Expand Down
4 changes: 3 additions & 1 deletion src/coreclr/inc/readytorun.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
// If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION`
// and handle pending work.
#define READYTORUN_MAJOR_VERSION 0x0009
#define READYTORUN_MINOR_VERSION 0x0002
#define READYTORUN_MINOR_VERSION 0x0003

#define MINIMUM_READYTORUN_MAJOR_VERSION 0x009

Expand All @@ -34,6 +34,7 @@
// R2R Version 9.0 adds support for the Vector512 type
// R2R Version 9.1 adds new helpers to allocate objects on frozen segments
// R2R Version 9.2 adds MemZero and NativeMemSet helpers
// R2R Version 9.3 adds BulkWriteBarrier helper


struct READYTORUN_CORE_HEADER
Expand Down Expand Up @@ -321,6 +322,7 @@ enum ReadyToRunHelper
READYTORUN_HELPER_WriteBarrier = 0x30,
READYTORUN_HELPER_CheckedWriteBarrier = 0x31,
READYTORUN_HELPER_ByRefWriteBarrier = 0x32,
READYTORUN_HELPER_BulkWriteBarrier = 0x33,

// Array helpers
READYTORUN_HELPER_Stelem_Ref = 0x38,
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/inc/readytorunhelpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero, CORINFO_HELP_THROWDIVZERO,
HELPER(READYTORUN_HELPER_WriteBarrier, CORINFO_HELP_ASSIGN_REF, )
HELPER(READYTORUN_HELPER_CheckedWriteBarrier, CORINFO_HELP_CHECKED_ASSIGN_REF, )
HELPER(READYTORUN_HELPER_ByRefWriteBarrier, CORINFO_HELP_ASSIGN_BYREF, )
HELPER(READYTORUN_HELPER_BulkWriteBarrier, CORINFO_HELP_BULK_WRITEBARRIER, )

HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, )
HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, )
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/codegencommon.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1772,7 +1772,7 @@ void CodeGen::genGenerateCode(void** codePtr, uint32_t* nativeSizeOfCode)
if (genWriteBarrierUsed && JitConfig.EnableExtraSuperPmiQueries() && !compiler->opts.IsReadyToRun())
{
void* ignored;
for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_ASSIGN_STRUCT; i++)
for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_BULK_WRITEBARRIER; i++)
{
compiler->compGetHelperFtn((CorInfoHelpFunc)i, &ignored);
}
Expand Down
111 changes: 111 additions & 0 deletions src/coreclr/jit/lower.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8212,6 +8212,117 @@ void Lowering::ContainCheckBitCast(GenTree* node)
}
}

//------------------------------------------------------------------------
// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_BULK_WRITEBARRIER call
//
// Arguments:
// blkNode - The block store node to lower
//
bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk)
{
if (comp->opts.OptimizationDisabled())
{
return false;
}

// Replace STORE_BLK (struct copy) with CORINFO_HELP_BULK_WRITEBARRIER which performs
// bulk copy for byrefs.
const unsigned bulkCopyThreshold = 4;
MichalStrehovsky marked this conversation as resolved.
Show resolved Hide resolved
if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || blk->IsVolatile() ||
(blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold))
{
return false;
}

GenTree* dest = blk->Addr();
GenTree* data = blk->Data();

if (data->OperIs(GT_IND))
{
if (data->AsIndir()->IsVolatile())
{
return false;
}

// Drop GT_IND nodes
BlockRange().Remove(data);
data = data->AsIndir()->Addr();
}
else
{
assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD));

// Convert local to LCL_ADDR
unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs();
data->ChangeOper(GT_LCL_ADDR);
data->ChangeType(TYP_I_IMPL);
data->AsLclFld()->SetLclOffs(lclOffset);
data->ClearContained();
}

// Size is a constant
GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)blk->GetLayout()->GetSize(), TYP_I_IMPL);
BlockRange().InsertBefore(data, size);

// A hacky way to safely call fgMorphTree in Lower
GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet());
GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data));
GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size));

GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_BULK_WRITEBARRIER, TYP_VOID, destPlaceholder,
dataPlaceholder, sizePlaceholder);
comp->fgMorphArgs(call);

LIR::Range range = LIR::SeqTree(comp, call);
GenTree* rangeStart = range.FirstNode();
GenTree* rangeEnd = range.LastNode();

BlockRange().InsertBefore(blk, std::move(range));
blk->gtBashToNOP();

LIR::Use destUse;
LIR::Use sizeUse;
BlockRange().TryGetUse(destPlaceholder, &destUse);
BlockRange().TryGetUse(sizePlaceholder, &sizeUse);
destUse.ReplaceWith(dest);
sizeUse.ReplaceWith(size);
destPlaceholder->SetUnusedValue();
sizePlaceholder->SetUnusedValue();

LIR::Use dataUse;
BlockRange().TryGetUse(dataPlaceholder, &dataUse);
dataUse.ReplaceWith(data);
dataPlaceholder->SetUnusedValue();

LowerRange(rangeStart, rangeEnd);

// Finally move all GT_PUTARG_* nodes
// Re-use the existing logic for CFG call args here
MoveCFGCallArgs(call);

BlockRange().Remove(destPlaceholder);
BlockRange().Remove(sizePlaceholder);
BlockRange().Remove(dataPlaceholder);

// Add implicit nullchecks for dest and data if needed:
//
auto wrapWithNullcheck = [&](GenTree* node) {
if (comp->fgAddrCouldBeNull(node))
{
LIR::Use nodeUse;
BlockRange().TryGetUse(node, &nodeUse);
GenTree* nodeClone = comp->gtNewLclvNode(nodeUse.ReplaceWithLclVar(comp), genActualType(node));
GenTree* nullcheck = comp->gtNewNullCheck(nodeClone, comp->compCurBB);
BlockRange().InsertAfter(nodeUse.Def(), nodeClone, nullcheck);
LowerNode(nullcheck);
}
};
wrapWithNullcheck(dest);
wrapWithNullcheck(data);

return true;
}

//------------------------------------------------------------------------
// LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call
//
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/lower.h
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,7 @@ class Lowering final : public Phase
void LowerBlockStore(GenTreeBlk* blkNode);
void LowerBlockStoreCommon(GenTreeBlk* blkNode);
void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode);
bool TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode);
void LowerLclHeap(GenTree* node);
void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent);
void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode);
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/jit/lowerarmarch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)

if (doCpObj)
{
// Try to use bulk copy helper
if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
{
return;
}

assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll;
}
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/jit/lowerloongarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
// CopyObj or CopyBlk
if (doCpObj)
{
// Try to use bulk copy helper
if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
{
return;
}

assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll;
}
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/jit/lowerriscv64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
// CopyObj or CopyBlk
if (doCpObj)
{
// Try to use bulk copy helper
if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
{
return;
}

assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll;
}
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/jit/lowerxarch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)

if (doCpObj)
{
// Try to use bulk copy helper
if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
{
return;
}

assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));

// If we have a long enough sequence of slots that do not require write barriers then
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1746,7 +1746,7 @@ void HelperCallProperties::init()
case CORINFO_HELP_CHECKED_ASSIGN_REF:
case CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP:
case CORINFO_HELP_ASSIGN_BYREF:
case CORINFO_HELP_ASSIGN_STRUCT:
case CORINFO_HELP_BULK_WRITEBARRIER:

mutatesHeap = true;
break;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants
static const uint32_t Signature = 0x00525452; // 'RTR'

static const uint32_t CurrentMajorVersion = 9;
static const uint32_t CurrentMinorVersion = 2;
static const uint32_t CurrentMinorVersion = 3;
};

struct ReadyToRunHeader
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants
public const uint Signature = 0x00525452; // 'RTR'

public const ushort CurrentMajorVersion = 9;
public const ushort CurrentMinorVersion = 2;
public const ushort CurrentMinorVersion = 3;
}
#if READYTORUN
#pragma warning disable 0169
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ public enum ReadyToRunHelper
WriteBarrier = 0x30,
CheckedWriteBarrier = 0x31,
ByRefWriteBarrier = 0x32,
BulkWriteBarrier = 0x33,

// Array helpers
Stelem_Ref = 0x38,
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ which is the right helper to use to allocate an object of a given type. */
CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap.

CORINFO_HELP_ASSIGN_BYREF,
CORINFO_HELP_ASSIGN_STRUCT,
CORINFO_HELP_BULK_WRITEBARRIER,


/* Accessing fields */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ public static void GetEntryPoint(TypeSystemContext context, ReadyToRunHelper id,
case ReadyToRunHelper.CheckedWriteBarrier:
mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpCheckedAssignRefArm64" : "RhpCheckedAssignRef";
break;
case ReadyToRunHelper.BulkWriteBarrier:
mangledName = "RhBuffer_BulkMoveWithWriteBarrier";
break;
case ReadyToRunHelper.ByRefWriteBarrier:
mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpByRefAssignRefArm64" : "RhpByRefAssignRef";
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF:
id = ReadyToRunHelper.ByRefWriteBarrier;
break;
case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER:
id = ReadyToRunHelper.BulkWriteBarrier;
break;

case CorInfoHelpFunc.CORINFO_HELP_ARRADDR_ST:
id = ReadyToRunHelper.Stelem_Ref;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1672,6 +1672,10 @@ private void ParseHelper(StringBuilder builder)
builder.Append("BYREF_WRITE_BARRIER");
break;

case ReadyToRunHelper.BulkWriteBarrier:
builder.Append("BULK_WRITE_BARRIER");
break;

// Array helpers
case ReadyToRunHelper.Stelem_Ref:
builder.Append("STELEM_REF");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
case CorInfoHelpFunc.CORINFO_HELP_CHECKED_ASSIGN_REF:
id = ReadyToRunHelper.CheckedWriteBarrier;
break;
case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER:
id = ReadyToRunHelper.BulkWriteBarrier;
break;
case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF:
id = ReadyToRunHelper.ByRefWriteBarrier;
break;
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/vm/corelib.h
Original file line number Diff line number Diff line change
Expand Up @@ -891,6 +891,7 @@ DEFINE_METHOD(DEBUGGER, BREAK, Break,
DEFINE_CLASS(BUFFER, System, Buffer)
DEFINE_METHOD(BUFFER, MEMCPY_PTRBYTE_ARRBYTE, Memcpy, SM_PtrByte_Int_ArrByte_Int_Int_RetVoid)
DEFINE_METHOD(BUFFER, MEMCPY, Memcpy, SM_PtrByte_PtrByte_Int_RetVoid)
DEFINE_METHOD(BUFFER, MEMCOPYGC, BulkMoveWithWriteBarrier, SM_RefByte_RefByte_UIntPtr_RetVoid)

DEFINE_CLASS(STUBHELPERS, StubHelpers, StubHelpers)
DEFINE_METHOD(STUBHELPERS, GET_DELEGATE_TARGET, GetDelegateTarget, SM_Delegate_RetIntPtr)
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/vm/ecall.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,10 @@ void ECall::PopulateManagedHelpers()
pDest = pMD->GetMultiCallableAddrOfCode();
SetJitHelperFunction(CORINFO_HELP_MEMCPY, pDest);

pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__BUFFER__MEMCOPYGC));
pDest = pMD->GetMultiCallableAddrOfCode();
SetJitHelperFunction(CORINFO_HELP_BULK_WRITEBARRIER, pDest);

pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__MATH__ROUND));
pDest = pMD->GetMultiCallableAddrOfCode();
SetJitHelperFunction(CORINFO_HELP_DBLROUND, pDest);
Expand Down
17 changes: 1 addition & 16 deletions src/coreclr/vm/jithelpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@
#ifndef FEATURE_EH_FUNCLETS
#include "excep.h"
#endif

#include "exinfo.h"
#include "arraynative.inl"

using std::isfinite;
using std::isnan;
Expand Down Expand Up @@ -4779,21 +4779,6 @@ HCIMPLEND
//
//========================================================================

/*************************************************************/
HCIMPL3(VOID, JIT_StructWriteBarrier, void *dest, void* src, CORINFO_CLASS_HANDLE typeHnd_)
{
FCALL_CONTRACT;

TypeHandle typeHnd(typeHnd_);
MethodTable *pMT = typeHnd.AsMethodTable();

HELPER_METHOD_FRAME_BEGIN_NOPOLL(); // Set up a frame
CopyValueClass(dest, src, pMT);
HELPER_METHOD_FRAME_END_POLL();

}
HCIMPLEND

/*************************************************************/
// Slow helper to tailcall from the fast one
NOINLINE HCIMPL0(void, JIT_PollGC_Framed)
Expand Down
Loading
Loading