From 278081fdd9cbc738a30867651a2e9f1af8d5fa4f Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Fri, 3 May 2024 17:54:35 +0200 Subject: [PATCH] JIT: Bulk copy of byrefs (#101761) --- src/coreclr/inc/corinfo.h | 2 +- src/coreclr/inc/jiteeversionguid.h | 10 +- src/coreclr/inc/jithelpers.h | 3 +- src/coreclr/inc/readytorun.h | 4 +- src/coreclr/inc/readytorunhelpers.h | 1 + src/coreclr/jit/codegencommon.cpp | 2 +- src/coreclr/jit/lower.cpp | 111 ++++++++++++++++++ src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerarmarch.cpp | 6 + src/coreclr/jit/lowerloongarch64.cpp | 6 + src/coreclr/jit/lowerriscv64.cpp | 6 + src/coreclr/jit/lowerxarch.cpp | 6 + src/coreclr/jit/utils.cpp | 2 +- .../nativeaot/Runtime/inc/ModuleHeaders.h | 2 +- .../Common/Internal/Runtime/ModuleHeaders.cs | 2 +- .../Internal/Runtime/ReadyToRunConstants.cs | 1 + .../Common/JitInterface/CorInfoHelpFunc.cs | 2 +- .../ILCompiler.Compiler/Compiler/JitHelper.cs | 3 + .../JitInterface/CorInfoImpl.ReadyToRun.cs | 3 + .../ReadyToRunSignature.cs | 4 + .../JitInterface/CorInfoImpl.RyuJit.cs | 3 + src/coreclr/vm/corelib.h | 1 + src/coreclr/vm/ecall.cpp | 4 + src/coreclr/vm/jithelpers.cpp | 17 +-- .../src/System/Buffer.cs | 3 + 25 files changed, 175 insertions(+), 30 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index ff82759f6aab64..2d526105164b6b 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -496,7 +496,7 @@ enum CorInfoHelpFunc CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap. CORINFO_HELP_ASSIGN_BYREF, - CORINFO_HELP_ASSIGN_STRUCT, + CORINFO_HELP_BULK_WRITEBARRIER, /* Accessing fields */ diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 79b6397a9111e2..122906341d00b4 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* 32d71f8e-c1f5-41cb-88cc-4e8504cabf40 */ - 0x32d71f8e, - 0xc1f5, - 0x41cb, - {0x88, 0xcc, 0x4e, 0x85, 0x04, 0xca, 0xbf, 0x40} +constexpr GUID JITEEVersionIdentifier = { /* bd8c41d4-8531-49c1-a600-0ae9bfe05de1 */ + 0xbd8c41d4, + 0x8531, + 0x49c1, + {0xa6, 0x00, 0x0a, 0xe9, 0xbf, 0xe0, 0x5d, 0xe1} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index f1711a9acfd9b2..2ee49994538592 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -153,8 +153,7 @@ JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY) DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB) - - JITHELPER(CORINFO_HELP_ASSIGN_STRUCT, JIT_StructWriteBarrier,CORINFO_HELP_SIG_4_STACK) + DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, CORINFO_HELP_SIG_REG_ONLY) // Accessing fields JITHELPER(CORINFO_HELP_GETFIELD8, JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 88219146a123a4..1c3ce8237ef7fc 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -20,7 +20,7 @@ // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION` // and handle pending work. #define READYTORUN_MAJOR_VERSION 0x0009 -#define READYTORUN_MINOR_VERSION 0x0002 +#define READYTORUN_MINOR_VERSION 0x0003 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x009 @@ -34,6 +34,7 @@ // R2R Version 9.0 adds support for the Vector512 type // R2R Version 9.1 adds new helpers to allocate objects on frozen segments // R2R Version 9.2 adds MemZero and NativeMemSet helpers +// R2R Version 9.3 adds BulkWriteBarrier helper struct READYTORUN_CORE_HEADER @@ -321,6 +322,7 @@ enum ReadyToRunHelper READYTORUN_HELPER_WriteBarrier = 0x30, READYTORUN_HELPER_CheckedWriteBarrier = 0x31, READYTORUN_HELPER_ByRefWriteBarrier = 0x32, + READYTORUN_HELPER_BulkWriteBarrier = 0x33, // Array helpers READYTORUN_HELPER_Stelem_Ref = 0x38, diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h index bbb586e8eb4a30..a1fcef8fbaf835 100644 --- a/src/coreclr/inc/readytorunhelpers.h +++ b/src/coreclr/inc/readytorunhelpers.h @@ -24,6 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero, CORINFO_HELP_THROWDIVZERO, HELPER(READYTORUN_HELPER_WriteBarrier, CORINFO_HELP_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_CheckedWriteBarrier, CORINFO_HELP_CHECKED_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_ByRefWriteBarrier, CORINFO_HELP_ASSIGN_BYREF, ) +HELPER(READYTORUN_HELPER_BulkWriteBarrier, CORINFO_HELP_BULK_WRITEBARRIER, ) HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, ) HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, ) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 6ec296a1391bb5..6dcbba13b48533 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -1772,7 +1772,7 @@ void CodeGen::genGenerateCode(void** codePtr, uint32_t* nativeSizeOfCode) if (genWriteBarrierUsed && JitConfig.EnableExtraSuperPmiQueries() && !compiler->opts.IsReadyToRun()) { void* ignored; - for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_ASSIGN_STRUCT; i++) + for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_BULK_WRITEBARRIER; i++) { compiler->compGetHelperFtn((CorInfoHelpFunc)i, &ignored); } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index a35c921facfcf0..28538fc2b9053f 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8212,6 +8212,117 @@ void Lowering::ContainCheckBitCast(GenTree* node) } } +//------------------------------------------------------------------------ +// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_BULK_WRITEBARRIER call +// +// Arguments: +// blkNode - The block store node to lower +// +bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) +{ + if (comp->opts.OptimizationDisabled()) + { + return false; + } + + // Replace STORE_BLK (struct copy) with CORINFO_HELP_BULK_WRITEBARRIER which performs + // bulk copy for byrefs. + const unsigned bulkCopyThreshold = 4; + if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || blk->IsVolatile() || + (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) + { + return false; + } + + GenTree* dest = blk->Addr(); + GenTree* data = blk->Data(); + + if (data->OperIs(GT_IND)) + { + if (data->AsIndir()->IsVolatile()) + { + return false; + } + + // Drop GT_IND nodes + BlockRange().Remove(data); + data = data->AsIndir()->Addr(); + } + else + { + assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); + + // Convert local to LCL_ADDR + unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); + data->ChangeOper(GT_LCL_ADDR); + data->ChangeType(TYP_I_IMPL); + data->AsLclFld()->SetLclOffs(lclOffset); + data->ClearContained(); + } + + // Size is a constant + GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)blk->GetLayout()->GetSize(), TYP_I_IMPL); + BlockRange().InsertBefore(data, size); + + // A hacky way to safely call fgMorphTree in Lower + GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet()); + GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); + GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); + + GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_BULK_WRITEBARRIER, TYP_VOID, destPlaceholder, + dataPlaceholder, sizePlaceholder); + comp->fgMorphArgs(call); + + LIR::Range range = LIR::SeqTree(comp, call); + GenTree* rangeStart = range.FirstNode(); + GenTree* rangeEnd = range.LastNode(); + + BlockRange().InsertBefore(blk, std::move(range)); + blk->gtBashToNOP(); + + LIR::Use destUse; + LIR::Use sizeUse; + BlockRange().TryGetUse(destPlaceholder, &destUse); + BlockRange().TryGetUse(sizePlaceholder, &sizeUse); + destUse.ReplaceWith(dest); + sizeUse.ReplaceWith(size); + destPlaceholder->SetUnusedValue(); + sizePlaceholder->SetUnusedValue(); + + LIR::Use dataUse; + BlockRange().TryGetUse(dataPlaceholder, &dataUse); + dataUse.ReplaceWith(data); + dataPlaceholder->SetUnusedValue(); + + LowerRange(rangeStart, rangeEnd); + + // Finally move all GT_PUTARG_* nodes + // Re-use the existing logic for CFG call args here + MoveCFGCallArgs(call); + + BlockRange().Remove(destPlaceholder); + BlockRange().Remove(sizePlaceholder); + BlockRange().Remove(dataPlaceholder); + + // Add implicit nullchecks for dest and data if needed: + // + auto wrapWithNullcheck = [&](GenTree* node) { + if (comp->fgAddrCouldBeNull(node)) + { + LIR::Use nodeUse; + BlockRange().TryGetUse(node, &nodeUse); + GenTree* nodeClone = comp->gtNewLclvNode(nodeUse.ReplaceWithLclVar(comp), genActualType(node)); + GenTree* nullcheck = comp->gtNewNullCheck(nodeClone, comp->compCurBB); + BlockRange().InsertAfter(nodeUse.Def(), nodeClone, nullcheck); + LowerNode(nullcheck); + } + }; + wrapWithNullcheck(dest); + wrapWithNullcheck(data); + + return true; +} + //------------------------------------------------------------------------ // LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call // diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index f13f9c2eeab106..dc32f7941f53dc 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -363,6 +363,7 @@ class Lowering final : public Phase void LowerBlockStore(GenTreeBlk* blkNode); void LowerBlockStoreCommon(GenTreeBlk* blkNode); void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); + bool TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode); void LowerLclHeap(GenTree* node); void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent); void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 6456b2b972ea9a..36b5e08f0afd67 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -737,6 +737,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index 4e826be0b22574..d3552e478fdfef 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -370,6 +370,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index aa8342ee1af5a4..5b0bd99df484f3 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -319,6 +319,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0e18c5685066fc..156992b89e9d46 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -457,6 +457,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); // If we have a long enough sequence of slots that do not require write barriers then diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index e3b99463b466de..07ca1fb6dc09c9 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -1746,7 +1746,7 @@ void HelperCallProperties::init() case CORINFO_HELP_CHECKED_ASSIGN_REF: case CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP: case CORINFO_HELP_ASSIGN_BYREF: - case CORINFO_HELP_ASSIGN_STRUCT: + case CORINFO_HELP_BULK_WRITEBARRIER: mutatesHeap = true; break; diff --git a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h index 6a3b24a3944870..d4360273d28ac8 100644 --- a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h +++ b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h @@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants static const uint32_t Signature = 0x00525452; // 'RTR' static const uint32_t CurrentMajorVersion = 9; - static const uint32_t CurrentMinorVersion = 2; + static const uint32_t CurrentMinorVersion = 3; }; struct ReadyToRunHeader diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 6fc5d9542e1609..8420c9a4803b28 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants public const uint Signature = 0x00525452; // 'RTR' public const ushort CurrentMajorVersion = 9; - public const ushort CurrentMinorVersion = 2; + public const ushort CurrentMinorVersion = 3; } #if READYTORUN #pragma warning disable 0169 diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs index 0ca8b74c85229b..af86b107e5b31e 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs @@ -233,6 +233,7 @@ public enum ReadyToRunHelper WriteBarrier = 0x30, CheckedWriteBarrier = 0x31, ByRefWriteBarrier = 0x32, + BulkWriteBarrier = 0x33, // Array helpers Stelem_Ref = 0x38, diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 5346806c1aff60..14bf90a9aaaed4 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -138,7 +138,7 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap. CORINFO_HELP_ASSIGN_BYREF, - CORINFO_HELP_ASSIGN_STRUCT, + CORINFO_HELP_BULK_WRITEBARRIER, /* Accessing fields */ diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs index 8faf4cf3fcfe7c..67cfd6cbe4f7a8 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs @@ -67,6 +67,9 @@ public static void GetEntryPoint(TypeSystemContext context, ReadyToRunHelper id, case ReadyToRunHelper.CheckedWriteBarrier: mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpCheckedAssignRefArm64" : "RhpCheckedAssignRef"; break; + case ReadyToRunHelper.BulkWriteBarrier: + mangledName = "RhBuffer_BulkMoveWithWriteBarrier"; + break; case ReadyToRunHelper.ByRefWriteBarrier: mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpByRefAssignRefArm64" : "RhpByRefAssignRef"; break; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs index e1b1b359c0878a..7f702fd6b764aa 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs @@ -1000,6 +1000,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; break; + case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER: + id = ReadyToRunHelper.BulkWriteBarrier; + break; case CorInfoHelpFunc.CORINFO_HELP_ARRADDR_ST: id = ReadyToRunHelper.Stelem_Ref; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs index 0eae2f10cb8f00..329b95f5d3c4e2 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs @@ -1672,6 +1672,10 @@ private void ParseHelper(StringBuilder builder) builder.Append("BYREF_WRITE_BARRIER"); break; + case ReadyToRunHelper.BulkWriteBarrier: + builder.Append("BULK_WRITE_BARRIER"); + break; + // Array helpers case ReadyToRunHelper.Stelem_Ref: builder.Append("STELEM_REF"); diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index 2dfa5d9d88f67d..d344dd431fb557 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -497,6 +497,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_CHECKED_ASSIGN_REF: id = ReadyToRunHelper.CheckedWriteBarrier; break; + case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER: + id = ReadyToRunHelper.BulkWriteBarrier; + break; case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; break; diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index d5af8bdb3cb64e..2f9049aea8d710 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -891,6 +891,7 @@ DEFINE_METHOD(DEBUGGER, BREAK, Break, DEFINE_CLASS(BUFFER, System, Buffer) DEFINE_METHOD(BUFFER, MEMCPY_PTRBYTE_ARRBYTE, Memcpy, SM_PtrByte_Int_ArrByte_Int_Int_RetVoid) DEFINE_METHOD(BUFFER, MEMCPY, Memcpy, SM_PtrByte_PtrByte_Int_RetVoid) +DEFINE_METHOD(BUFFER, MEMCOPYGC, BulkMoveWithWriteBarrier, SM_RefByte_RefByte_UIntPtr_RetVoid) DEFINE_CLASS(STUBHELPERS, StubHelpers, StubHelpers) DEFINE_METHOD(STUBHELPERS, GET_DELEGATE_TARGET, GetDelegateTarget, SM_Delegate_RetIntPtr) diff --git a/src/coreclr/vm/ecall.cpp b/src/coreclr/vm/ecall.cpp index 35a5d36eae4d76..56a28a7a57ddb8 100644 --- a/src/coreclr/vm/ecall.cpp +++ b/src/coreclr/vm/ecall.cpp @@ -157,6 +157,10 @@ void ECall::PopulateManagedHelpers() pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_MEMCPY, pDest); + pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__BUFFER__MEMCOPYGC)); + pDest = pMD->GetMultiCallableAddrOfCode(); + SetJitHelperFunction(CORINFO_HELP_BULK_WRITEBARRIER, pDest); + pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__MATH__ROUND)); pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_DBLROUND, pDest); diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 98d6780258a6c6..05d2ad18b5bd94 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -55,8 +55,8 @@ #ifndef FEATURE_EH_FUNCLETS #include "excep.h" #endif - #include "exinfo.h" +#include "arraynative.inl" using std::isfinite; using std::isnan; @@ -4779,21 +4779,6 @@ HCIMPLEND // //======================================================================== -/*************************************************************/ -HCIMPL3(VOID, JIT_StructWriteBarrier, void *dest, void* src, CORINFO_CLASS_HANDLE typeHnd_) -{ - FCALL_CONTRACT; - - TypeHandle typeHnd(typeHnd_); - MethodTable *pMT = typeHnd.AsMethodTable(); - - HELPER_METHOD_FRAME_BEGIN_NOPOLL(); // Set up a frame - CopyValueClass(dest, src, pMT); - HELPER_METHOD_FRAME_END_POLL(); - -} -HCIMPLEND - /*************************************************************/ // Slow helper to tailcall from the fast one NOINLINE HCIMPL0(void, JIT_PollGC_Framed) diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs index 24f8794d852afd..d07df58bb23769 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs @@ -177,6 +177,9 @@ ref Unsafe.As(ref source), private const uint BulkMoveWithWriteBarrierChunk = 0x4000; #endif +#if NATIVEAOT + [System.Runtime.RuntimeExport("RhBuffer_BulkMoveWithWriteBarrier")] +#endif internal static void BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) { if (byteCount <= BulkMoveWithWriteBarrierChunk)