diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index c2bc30d00cd5a..a95d275b5f2ad 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8212,9 +8212,15 @@ void Lowering::ContainCheckBitCast(GenTree* node) } } +//------------------------------------------------------------------------ +// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_ASSIGN_STRUCT call +// +// Arguments: +// blkNode - The block store node to lower +// bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) { - if (comp->opts.OptimizationDisabled() || !ISMETHOD("Test")) + if (comp->opts.OptimizationDisabled()) { return false; } @@ -8222,79 +8228,93 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs // bulk copy for byrefs. const unsigned bulkCopyThreshold = 4; - if (blk->OperIs(GT_STORE_BLK) && !blk->OperIsInitBlkOp() && - (blk->GetLayout()->GetGCPtrCount() >= bulkCopyThreshold)) + if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) { - GenTree* addr = blk->Addr(); - GenTree* data = blk->Data(); + return false; + } - const unsigned gcPtrs = blk->GetLayout()->GetGCPtrCount(); - if (!CheckedOps::MulOverflows((int)gcPtrs, TARGET_POINTER_SIZE, true)) - { - if (data->OperIs(GT_IND)) - { - // Drop GT_IND nodes - BlockRange().Remove(data); - data = data->AsIndir()->Addr(); - } - else - { - assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); - - // Convert local to LCL_ADDR - unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); - data->ChangeOper(GT_LCL_ADDR); - data->ChangeType(TYP_I_IMPL); - data->AsLclFld()->SetLclOffs(lclOffset); - data->ClearContained(); - } + GenTree* dest = blk->Addr(); + GenTree* data = blk->Data(); + + if (data->OperIs(GT_IND)) + { + // Drop GT_IND nodes + BlockRange().Remove(data); + data = data->AsIndir()->Addr(); + } + else + { + assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); + + // Convert local to LCL_ADDR + unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); + data->ChangeOper(GT_LCL_ADDR); + data->ChangeType(TYP_I_IMPL); + data->AsLclFld()->SetLclOffs(lclOffset); + data->ClearContained(); + } - // Size is a constant - GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)gcPtrs * TARGET_POINTER_SIZE, TYP_I_IMPL); - BlockRange().InsertBefore(data, size); + // Size is a constant + GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)blk->GetLayout()->GetSize(), TYP_I_IMPL); + BlockRange().InsertBefore(data, size); + + // A hacky way to safely call fgMorphTree in Lower + GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet()); + GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); + GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); - // A hacky way to safely call fgMorphTree in Lower - GenTree* destPlaceholder = comp->gtNewZeroConNode(addr->TypeGet()); - GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); - GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); + GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, + dataPlaceholder, sizePlaceholder); + comp->fgMorphArgs(call); - GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder); - comp->fgMorphArgs(call); + LIR::Range range = LIR::SeqTree(comp, call); + GenTree* rangeStart = range.FirstNode(); + GenTree* rangeEnd = range.LastNode(); - LIR::Range range = LIR::SeqTree(comp, call); - GenTree* rangeStart = range.FirstNode(); - GenTree* rangeEnd = range.LastNode(); + BlockRange().InsertBefore(blk, std::move(range)); + blk->gtBashToNOP(); - BlockRange().InsertBefore(blk, std::move(range)); - blk->gtBashToNOP(); + LIR::Use destUse; + LIR::Use sizeUse; + BlockRange().TryGetUse(destPlaceholder, &destUse); + BlockRange().TryGetUse(sizePlaceholder, &sizeUse); + destUse.ReplaceWith(dest); + sizeUse.ReplaceWith(size); + destPlaceholder->SetUnusedValue(); + sizePlaceholder->SetUnusedValue(); - LIR::Use destUse; - LIR::Use sizeUse; - BlockRange().TryGetUse(destPlaceholder, &destUse); - BlockRange().TryGetUse(sizePlaceholder, &sizeUse); - destUse.ReplaceWith(addr); - sizeUse.ReplaceWith(size); - destPlaceholder->SetUnusedValue(); - sizePlaceholder->SetUnusedValue(); + LIR::Use dataUse; + BlockRange().TryGetUse(dataPlaceholder, &dataUse); + dataUse.ReplaceWith(data); + dataPlaceholder->SetUnusedValue(); - LIR::Use dataUse; - BlockRange().TryGetUse(dataPlaceholder, &dataUse); - dataUse.ReplaceWith(data); - dataPlaceholder->SetUnusedValue(); + LowerRange(rangeStart, rangeEnd); - LowerRange(rangeStart, rangeEnd); + // Finally move all GT_PUTARG_* nodes + // Re-use the existing logic for CFG call args here + MoveCFGCallArgs(call); - // Finally move all GT_PUTARG_* nodes - // Re-use the existing logic for CFG call args here - MoveCFGCallArgs(call); + BlockRange().Remove(destPlaceholder); + BlockRange().Remove(sizePlaceholder); + BlockRange().Remove(dataPlaceholder); - BlockRange().Remove(destPlaceholder); - BlockRange().Remove(sizePlaceholder); - BlockRange().Remove(dataPlaceholder); - return true; + // Add implicit nullchecks for dest and data if needed: + // + auto wrapWithNullcheck = [&](GenTree* node) { + if (comp->fgAddrCouldBeNull(node)) + { + LIR::Use nodeUse; + BlockRange().TryGetUse(node, &nodeUse); + GenTree* nodeClone = comp->gtNewLclvNode(nodeUse.ReplaceWithLclVar(comp), genActualType(node)); + GenTree* nullcheck = comp->gtNewNullCheck(nodeClone, comp->compCurBB); + BlockRange().InsertAfter(nodeUse.Def(), nodeClone, nullcheck); + LowerNode(nullcheck); } - } - return false; + }; + wrapWithNullcheck(dest); + wrapWithNullcheck(data); + + return true; } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 6456b2b972ea9..36b5e08f0afd6 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -737,6 +737,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index 4e826be0b2257..d3552e478fdfe 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -370,6 +370,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index aa8342ee1af5a..5b0bd99df484f 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -319,6 +319,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 45ea924530180..156992b89e9d4 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -457,6 +457,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) { return;