Revamp the generation of runtime division checks on ARM64

Fixes #64795

This patch introduces a new compilation phase that passes over the
GenTrees looking for GT_DIV/GT_UDIV nodes on integral types, and
morphs the code to introduce the necessary conformance checks
(overflow/divide-by-zero) early on in the compilation pipeline.
Currently these are added during the Emit phase, meaning optimizations
don't run on any code introduced.

The aim is to allow the compiler to make decisions on code position
and instruction selection for these checks. For example on ARM64 this
enables certain scenarios to choose the cbz instruction over cmp/beq,
can lead to more compact code. It also allows some of the comparisons
in the checks to be hoisted out of loops.

src/coreclr/jit/CMakeLists.txt

@@ -93,6 +93,7 @@ endif(CLR_CMAKE_TARGET_WIN32)
 set( JIT_SOURCES
   abi.cpp
   alloc.cpp
+  arithchecks.cpp
   assertionprop.cpp
   bitset.cpp
   block.cpp

src/coreclr/jit/arithchecks.cpp

@@ -0,0 +1,270 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "jitpch.h"
+#include "compiler.h"
+#include "gentree.h"
+#ifdef _MSC_VER
+#pragma hdrstop
+#endif
+
+jitstd::vector<unsigned>& Compiler::getVisitedDivNodes()
+{
+    if (visitedDivNodes == nullptr)
+    {
+        visitedDivNodes = new (getAllocator(CMK_Unknown)) jitstd::vector<unsigned>(getAllocator(CMK_Unknown));
+    }
+    return *visitedDivNodes;
+}
+
+jitstd::vector<Compiler::ThrowHelper>& Compiler::getThrowHelperSet()
+{
+    if (throwHelperSet == nullptr)
+    {
+        throwHelperSet = new (getAllocator(CMK_Unknown)) jitstd::vector<ThrowHelper>(getAllocator(CMK_Unknown));
+    }
+    return *throwHelperSet;
+}
+
+void Compiler::fgInsertConditionalThrowException(Compiler::TreeCoord& code, SpecialCodeKind codeKind)
+{
+    // Create a new basic block, splitting before the code containing the division.
+    Statement* newStmt      = nullptr;
+    GenTree**  splitNodeUse = nullptr;
+    JITDUMP("[%06d] contains the division\n", code.tree->gtTreeID);
+    BasicBlock* divBlock = fgSplitBlockBeforeTree(code.block, code.stmt, code.tree, &newStmt, &splitNodeUse);
+    GenTree*    tree     = *splitNodeUse;
+    JITDUMP("[%06d] contains the division after split\n", tree->gtTreeID);
+    BasicBlock* checkBlock = code.block;
+
+    // The checking block is added after the original block that has been split. This is for convenience of
+    // adding the checking statement to an empty basic block.
+    checkBlock = fgNewBBafter(BBJ_ALWAYS, code.block, true);
+    fgRedirectTargetEdge(code.block, checkBlock);
+
+    GenTree* dividend = tree->gtGetOp1();
+    GenTree* divisor  = tree->gtGetOp2();
+    assert(varTypeIsIntegral(divisor) && varTypeIsIntegral(dividend));
+
+    GenTree* divisorCopy  = nullptr;
+    GenTree* dividendCopy = nullptr;
+
+    GenTree*        condition = nullptr;
+    CorInfoHelpFunc helper    = CORINFO_HELP_UNDEF;
+    switch (codeKind)
+    {
+        case SCK_DIV_BY_ZERO:
+        {
+            assert(codeKind == SCK_DIV_BY_ZERO);
+            impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
+
+            // (divisor == 0)
+            condition = gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(0, genActualType(divisorCopy)));
+
+            helper = CORINFO_HELP_THROWDIVZERO;
+            break;
+        }
+        case SCK_OVERFLOW:
+        {
+            impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
+            impCloneExpr(dividend, &dividendCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("cloned for runtime check"));
+
+            // (dividend < 0 && divisor == -1)
+            GenTreeOp* const divisorIsMinusOne =
+                gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(-1, genActualType(divisorCopy)));
+            GenTreeOp* const dividendIsNegative =
+                gtNewOperNode(GT_LT, TYP_INT, dividendCopy, gtNewIconNode(0, genActualType(dividendCopy)));
+            GenTreeOp* const combinedTest = gtNewOperNode(GT_AND, TYP_INT, divisorIsMinusOne, dividendIsNegative);
+            condition                     = gtNewOperNode(GT_EQ, TYP_INT, combinedTest, gtNewTrue());
+
+            helper = CORINFO_HELP_OVERFLOW;
+            break;
+        }
+        default:
+            noway_assert(!"unexpected exception code kind");
+    }
+    assert(condition != nullptr);
+    assert(helper != CORINFO_HELP_UNDEF);
+
+    // Add another block near the checking block, this will be the throw block.
+    // This block is explicitly placed after the checking block to produce debuggable code
+    // when optimizations aren't enabled.
+    BasicBlock* throwBlock = nullptr;
+
+    // Search for an already created throw block. We don't do this for debug code
+    // so the throw code will stay in-line.
+    if (!opts.compDbgCode)
+    {
+        for (const ThrowHelper& th : getThrowHelperSet())
+        {
+            if (th.For(helper, divBlock))
+            {
+                throwBlock = th.Block();
+                break;
+            }
+        }
+    }
+
+    if (throwBlock == nullptr)
+    {
+        // Insert a throw block close by, in the same EH region.
+        throwBlock = fgNewBBafter(BBJ_THROW, checkBlock, true);
+
+        // Add exception raising code to the throw block.
+        GenTreeCall* const thrw      = gtNewHelperCallNode(helper, TYP_VOID);
+        Statement*         throwStmt = gtNewStmt(thrw);
+        fgInsertStmtAtEnd(throwBlock, throwStmt);
+        assert(throwBlock->lastStmt() == throwStmt);
+
+        if (!opts.compDbgCode)
+            getThrowHelperSet().push_back(ThrowHelper(helper, throwBlock));
+    }
+
+    // Add the compare and jump to the end of the checking block.
+    condition->gtFlags |= GTF_RELOP_JMP_USED;
+    GenTree* const cndJmp  = gtNewOperNode(GT_JTRUE, TYP_VOID, condition);
+    Statement*     jmpStmt = gtNewStmt(cndJmp);
+    fgInsertStmtAtEnd(checkBlock, jmpStmt);
+    assert(checkBlock->lastStmt() == jmpStmt);
+
+    // Set the true edge to the exception block, false edge to the block where division occurs.
+    FlowEdge* trueEdge  = fgAddRefPred(throwBlock, checkBlock);
+    FlowEdge* falseEdge = fgAddRefPred(divBlock, checkBlock);
+
+    // The exception case is considered unlikely in comparison to normal control flow.
+    trueEdge->setLikelihood(0.01);
+    falseEdge->setLikelihood(0.99);
+
+    checkBlock->SetCond(trueEdge, falseEdge);
+
+    JITDUMP(FMT_BB " contains the division\n", divBlock->bbNum);
+    JITDUMP(FMT_BB " contains the check\n", checkBlock->bbNum);
+    JITDUMP(FMT_BB " contains the throw\n", throwBlock->bbNum);
+
+    code.block = divBlock;
+    code.stmt  = divBlock->firstStmt();
+    code.tree  = tree;
+}
+
+bool Compiler::fgInsertDivisionChecks(BasicBlock* block, Statement* stmt, GenTree* tree)
+{
+    assert(tree->OperIs(GT_DIV, GT_UDIV));
+
+    // Check we haven't processed this DIV before.
+    bool found = false;
+    for (const unsigned& id : getVisitedDivNodes())
+    {
+        if (tree->gtTreeID == id)
+        {
+            found = true;
+            break;
+        }
+    }
+    if (found)
+        return false;
+
+    // Only integral divisions will throw
+    if (!(varTypeIsIntegral(tree->gtGetOp1()) && varTypeIsIntegral(tree->gtGetOp2())))
+    {
+        return false;
+    }
+
+    bool modified = false;
+
+    TreeCoord divCode(block, stmt, tree);
+
+    // Check for division by -1 when the dividend is signed.
+    // (NegativeVal / -1) => OverflowException
+    if ((divCode.tree->OperExceptions(this) & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None)
+    {
+        assert(tree->OperIs(GT_DIV));
+        JITDUMP(FMT_BB " " FMT_STMT " needs a signed overflow check\n", divCode.block->bbNum, divCode.stmt->GetID());
+        fgInsertConditionalThrowException(divCode, SCK_OVERFLOW);
+        modified = true;
+    }
+
+    // Check for division by 0 - both unsigned and signed are affected.
+    // (AnyVal / 0) => DivideByZeroException
+    if ((divCode.tree->OperExceptions(this) & ExceptionSetFlags::DivideByZeroException) != ExceptionSetFlags::None)
+    {
+        JITDUMP(FMT_BB " " FMT_STMT " needs a divide by zero check\n", divCode.block->bbNum, divCode.stmt->GetID());
+        fgInsertConditionalThrowException(divCode, SCK_DIV_BY_ZERO);
+        modified = true;
+    }
+
+    JITDUMP("Marking [%06d] as visited.\n", divCode.tree->gtTreeID);
+    getVisitedDivNodes().push_back(divCode.tree->gtTreeID);
+
+    return modified;
+}
+
+PhaseStatus Compiler::fgInsertArithmeticExceptions()
+{
+#if defined(TARGET_ARM64)
+    struct Visitor : GenTreeVisitor<Visitor>
+    {
+        enum
+        {
+            DoPreOrder = true,
+        };
+
+        Visitor(Compiler* comp, BasicBlock* block, Statement* stmt)
+            : GenTreeVisitor(comp)
+            , m_block(block)
+            , m_stmt(stmt)
+        {
+        }
+
+        fgWalkResult PreOrderVisit(GenTree** use, GenTree* user)
+        {
+            if (!(*use)->OperIs(GT_DIV, GT_UDIV))
+            {
+                return WALK_CONTINUE;
+            }
+
+            modified = m_compiler->fgInsertDivisionChecks(m_block, m_stmt, *use);
+
+            // If checks were inserted, the tree was split and the graph was modified,
+            // so traversal needs to start again.
+            if (modified)
+            {
+                return WALK_ABORT;
+            }
+
+            return WALK_CONTINUE;
+        }
+
+        BasicBlock* m_block;
+        Statement*  m_stmt;
+    public:
+        bool modified = false;
+    };
+
+    bool lastModified = false;
+    bool anyModified  = false;
+    do
+    {
+        lastModified = false;
+        for (BasicBlock* const block : Blocks())
+        {
+            for (Statement* const stmt : block->Statements())
+            {
+                GenTree* tree = stmt->GetRootNode();
+                Visitor  visitor(this, block, stmt);
+                visitor.WalkTree(&tree, nullptr);
+                lastModified = visitor.modified;
+                anyModified |= lastModified;
+
+                if (lastModified)
+                    break;
+            }
+            if (lastModified)
+                break;
+        }
+    } while (lastModified);
+
+    return anyModified ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
+#else
+    return PhaseStatus::MODIFIED_NOTHING;
+#endif
+}

src/coreclr/jit/codegen.h

@@ -794,7 +794,9 @@ class CodeGen final : public CodeGenInterface
     void genCodeForLongUMod(GenTreeOp* node);
 #endif // TARGET_X86
 
+#if !defined(TARGET_ARM64)
     void genCodeForDivMod(GenTreeOp* treeNode);
+#endif
     void genCodeForMul(GenTreeOp* treeNode);
     void genCodeForIncSaturate(GenTree* treeNode);
     void genCodeForMulHi(GenTreeOp* treeNode);

src/coreclr/jit/codegenarm64.cpp

@@ -3535,95 +3535,6 @@ void CodeGen::genCodeForBswap(GenTree* tree)
     genProduceReg(tree);
 }
 
-//------------------------------------------------------------------------
-// genCodeForDivMod: Produce code for a GT_DIV/GT_UDIV node. We don't see MOD:
-// (1) integer MOD is morphed into a sequence of sub, mul, div in fgMorph;
-// (2) float/double MOD is morphed into a helper call by front-end.
-//
-// Arguments:
-//    tree - the node
-//
-void CodeGen::genCodeForDivMod(GenTreeOp* tree)
-{
-    assert(tree->OperIs(GT_DIV, GT_UDIV));
-
-    var_types targetType = tree->TypeGet();
-    emitter*  emit       = GetEmitter();
-
-    genConsumeOperands(tree);
-
-    if (varTypeIsFloating(targetType))
-    {
-        // Floating point divide never raises an exception
-        genCodeForBinary(tree);
-    }
-    else // an integer divide operation
-    {
-        // Generate the require runtime checks for GT_DIV or GT_UDIV.
-
-        GenTree* divisorOp = tree->gtGetOp2();
-        emitAttr size      = EA_ATTR(genTypeSize(genActualType(tree->TypeGet())));
-
-        regNumber divisorReg = divisorOp->GetRegNum();
-
-        ExceptionSetFlags exSetFlags = tree->OperExceptions(compiler);
-
-        // (AnyVal / 0) => DivideByZeroException
-        if ((exSetFlags & ExceptionSetFlags::DivideByZeroException) != ExceptionSetFlags::None)
-        {
-            if (divisorOp->IsIntegralConst(0))
-            {
-                // We unconditionally throw a divide by zero exception
-                genJumpToThrowHlpBlk(EJ_jmp, SCK_DIV_BY_ZERO);
-
-                // We still need to call genProduceReg
-                genProduceReg(tree);
-
-                return;
-            }
-            else
-            {
-                // Check if the divisor is zero throw a DivideByZeroException
-                emit->emitIns_R_I(INS_cmp, size, divisorReg, 0);
-                genJumpToThrowHlpBlk(EJ_eq, SCK_DIV_BY_ZERO);
-            }
-        }
-
-        // (MinInt / -1) => ArithmeticException
-        if ((exSetFlags & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None)
-        {
-            // Signed-division might overflow.
-
-            assert(tree->OperIs(GT_DIV));
-            assert(!divisorOp->IsIntegralConst(0));
-
-            BasicBlock* sdivLabel  = genCreateTempLabel();
-            GenTree*    dividendOp = tree->gtGetOp1();
-
-            // Check if the divisor is not -1 branch to 'sdivLabel'
-            emit->emitIns_R_I(INS_cmp, size, divisorReg, -1);
-
-            inst_JMP(EJ_ne, sdivLabel);
-            // If control flow continues past here the 'divisorReg' is known to be -1
-
-            regNumber dividendReg = dividendOp->GetRegNum();
-            // At this point the divisor is known to be -1
-            //
-            // Issue the 'cmp dividendReg, 1' instruction.
-            // This is an alias to 'subs zr, dividendReg, 1' on ARM64 itself.
-            // This will set the V (overflow) flags only when dividendReg is MinInt
-            //
-            emit->emitIns_R_I(INS_cmp, size, dividendReg, 1);
-            genJumpToThrowHlpBlk(EJ_vs, SCK_ARITH_EXCPN); // if the V flags is set throw
-                                                          // ArithmeticException
-
-            genDefineTempLabel(sdivLabel);
-        }
-
-        genCodeForBinary(tree); // Generate the sdiv instruction
-    }
-}
-
 // Generate code for CpObj nodes which copy structs that have interleaved
 // GC pointers.
 // For this case we'll generate a sequence of loads/stores in the case of struct