diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index eebe47377666b6..1b4b4789279d03 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -623,7 +623,7 @@ class CodeGen final : public CodeGenInterface } #endif // !FEATURE_EH_FUNCLETS - + void genUpdateLiveRangesForTruncatedIGs(); void genGeneratePrologsAndEpilogs(); #if defined(DEBUG) && defined(TARGET_ARM64) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 0eccb2abfc8e5e..c2cd239b2225b7 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -1969,6 +1969,8 @@ void CodeGen::genGenerateMachineCode() GetEmitter()->emitJumpDistBind(); + genUpdateLiveRangesForTruncatedIGs(); + #if FEATURE_LOOP_ALIGN /* Perform alignment adjustments */ @@ -1978,6 +1980,113 @@ void CodeGen::genGenerateMachineCode() /* The code is now complete and final; it should not change after this. */ } +void CodeGen::genUpdateLiveRangesForTruncatedIGs() +{ +#ifdef DEBUG + if (verbose) + { + printf("*************** In genUpdateLiveRangesForTruncatedIGs()\n"); + } +#endif + bool updatedGroup = false; + + if (compiler->lvaCount > 0) + { + insGroup* ig = GetEmitter()->emitIGlist; + while (ig != nullptr) + { + if (ig->igFlags & IGF_UPD_ICOUNT) + { + + unsigned int liveVarCount = varLiveKeeper->getVarCount(); + if (liveVarCount > 0) + { +#ifdef DEBUG + if (compiler->verbose) + { + JITDUMP("IG%02u is marked with IGF_UPD_ICOUNT and has %d live variables\n", ig->igNum, + liveVarCount); + } +#endif + for (unsigned int varNum = 0; varNum < liveVarCount; varNum++) + { + if (compiler->compMap2ILvarNum(varNum) == (unsigned int)ICorDebugInfo::UNKNOWN_ILNUM) + { + continue; + } + + for (int rangeIndex = 0; rangeIndex < 2; rangeIndex++) + { + VariableLiveKeeper::LiveRangeList* liveRanges; + if (rangeIndex == 0) + { + liveRanges = varLiveKeeper->getLiveRangesForVarForProlog(varNum); + } + else + { + liveRanges = varLiveKeeper->getLiveRangesForVarForBody(varNum); + } + + for (VariableLiveKeeper::VariableLiveRange& liveRange : *liveRanges) + { + if (liveRange.m_StartEmitLocation.GetIG()->igNum == ig->igNum && + liveRange.m_StartEmitLocation.GetInsNum() > ig->igInsCnt) + { +#ifdef DEBUG + if (compiler->verbose) + { + JITDUMP("IG%02u varNum %d StartEmitLocation InsCnt changed from %d to %d\n", ig, + liveVarCount, liveRange.m_StartEmitLocation.GetInsNum(), ig->igInsCnt); + } +#endif + liveRange.m_StartEmitLocation.SetInsNum(ig->igInsCnt); + assert(liveRange.m_StartEmitLocation.GetInsNum() == ig->igInsCnt); + updatedGroup = true; + } + + if (liveRange.m_EndEmitLocation.GetIG()->igNum == ig->igNum && + liveRange.m_EndEmitLocation.GetInsNum() > ig->igInsCnt) + { + liveRange.m_EndEmitLocation.SetInsNum(ig->igInsCnt); +#ifdef DEBUG + if (compiler->verbose) + { + JITDUMP("IG%02u varNum %d EndEmitLocation InsCnt changed from %d to %d\n", ig, + liveVarCount, liveRange.m_StartEmitLocation.GetInsNum(), ig->igInsCnt); + } +#endif + assert(liveRange.m_EndEmitLocation.GetInsNum() == ig->igInsCnt); + updatedGroup = true; + } + } + } + } + } + else + { +#ifdef DEBUG + if (compiler->verbose) + { + JITDUMP("IG%02u is marked with IGF_UPD_ICOUNT but has no live variables\n", ig); + } +#endif + } + + ig->igFlags ^= IGF_UPD_ICOUNT; + } + + ig = ig->igNext; + } + } +#ifdef DEBUG + if (updatedGroup && verbose) + { + printf("\nlvaTable after genUpdateLiveRangesForTruncatedIGs\n"); + compiler->lvaTableDump(); + } +#endif +} + //---------------------------------------------------------------------- // genEmitMachineCode -- emit the actual machine instruction code // @@ -7174,7 +7283,6 @@ void CodeGen::genSetScopeInfoUsingVariableRanges() reportRange(curLoc, curStart, curEnd); } } - compiler->eeVarsCount = liveRangeIndex; } #endif // USING_VARIABLE_LIVE_RANGE @@ -9279,7 +9387,7 @@ CodeGenInterface::VariableLiveKeeper::LiveRangeList* CodeGenInterface::VariableL } //------------------------------------------------------------------------ -// getLiveRangesCount: Returns the count of variable locations reported for the tracked +// getLiveRangesCount: Returns the size of variable locations reported for the tracked // variables, which are arguments, special arguments, and local IL variables. // // Return Value: @@ -9311,6 +9419,30 @@ size_t CodeGenInterface::VariableLiveKeeper::getLiveRangesCount() const return liveRangesCount; } +//------------------------------------------------------------------------ +// getVarCount: Returns the count of variable locations reported for the tracked +// variables, which are arguments, special arguments, and local IL variables. +// +// Return Value: +// unsigned int - the count of variables +// +unsigned int CodeGenInterface::VariableLiveKeeper::getVarCount() const +{ + unsigned int liveRangesCount = 0; + + if (m_Compiler->opts.compDbgInfo) + { + for (unsigned int varNum = 0; varNum < m_LiveDscCount; varNum++) + { + if (m_Compiler->compMap2ILvarNum(varNum) != (unsigned int)ICorDebugInfo::UNKNOWN_ILNUM) + { + liveRangesCount += 1; + } + } + } + return liveRangesCount; +}; + //------------------------------------------------------------------------ // psiStartVariableLiveRange: Reports the given variable as being born. // diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index dbd53ffbad46f7..a452549c7c9cef 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -743,7 +743,8 @@ class CodeGenInterface LiveRangeList* getLiveRangesForVarForBody(unsigned int varNum) const; LiveRangeList* getLiveRangesForVarForProlog(unsigned int varNum) const; - size_t getLiveRangesCount() const; + size_t getLiveRangesCount() const; + unsigned int getVarCount() const; // For parameters locations on prolog void psiStartVariableLiveRange(CodeGenInterface::siVarLoc varLocation, unsigned int varNum); diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index dd17b6febd6109..f87f7328556a17 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -29,7 +29,6 @@ void emitLocation::CaptureLocation(emitter* emit) { ig = emit->emitCurIG; codePos = emit->emitCurOffset(); - assert(Valid()); } @@ -50,6 +49,13 @@ int emitLocation::GetInsNum() const return emitGetInsNumFromCodePos(codePos); } +void emitLocation::SetInsNum(int insNum) +{ + assert((unsigned int)insNum == emitGetInsNumFromCodePos(insNum)); + codePos = emitGetInsNumFromCodePos(insNum) + (emitGetInsOfsFromCodePos(codePos) << 16); + assert(Valid()); +} + // Get the instruction offset in the current instruction group, which must be a funclet prolog group. // This is used to find an instruction offset used in unwind data. // TODO-AMD64-Bug?: We only support a single main function prolog group, but allow for multiple funclet prolog @@ -3613,6 +3619,10 @@ void emitter::emitDispIGflags(unsigned flags) { printf(", isz"); } + if (flags & IGF_UPD_ICOUNT) + { + printf(", ict"); + } if (flags & IGF_EXTEND) { printf(", extend"); @@ -4139,6 +4149,10 @@ void emitter::emitJumpDistBind() #endif // DEBUG int jmp_iteration = 1; +#ifdef TARGET_XARCH + int jmp_removal = 1; + bool jmpRemoved = false; +#endif /*****************************************************************************/ /* If we iterate to look for more jumps to shorten, we start again here. */ @@ -4165,6 +4179,110 @@ void emitter::emitJumpDistBind() adjIG = 0; minShortExtra = (UNATIVE_OFFSET)-1; +#ifdef TARGET_XARCH + // try to remove unconditional jumps at the end of a group to the next group + + { + jmp = emitJumpList; + instrDescJmp* previousJmp = nullptr; + while (jmp) + { + // if the jump is unconditional then it is a candidate + if (jmp->idInsFmt() == IF_LABEL && emitIsUncondJump(jmp) && !jmp->idjKeepLong) + { + // target group is not bound yet so use the cookie to fetch it + insGroup* targetGroup = (insGroup*)emitCodeGetCookie(jmp->idAddr()->iiaBBlabel); + + // if the target group is the next instruction group + if (targetGroup != nullptr && jmp->idjIG->igNext == targetGroup) + { + insGroup* group = jmp->idjIG; + unsigned instructionCount = group->igInsCnt; + + if (instructionCount > 0) + { + BYTE* dataPtr = group->igData; + instrDesc* instructionDescriptor = nullptr; + + do + { + instructionDescriptor = (instrDesc*)dataPtr; + dataPtr += emitSizeOfInsDsc(instructionDescriptor); + } while (--instructionCount); + + assert(instructionDescriptor != nullptr); + // instructionDescriptor is now the last instruction in the group + + if (instructionDescriptor->idIns() == jmp->idIns() && jmp == instructionDescriptor) + { +// the last instruction in the group is the jmp we're looking for +// and it jumps to the next instruction group so we don't really need it +#ifdef DEBUG + if (EMITVERBOSE) + { + printf("Removing unconditional jump [%08X/%03u] from the last instruction in IG%02u to " + "the next IG IG%02u label %s\n", + dspPtr(jmp), jmp->idDebugOnlyInfo()->idNum, group->igNum, targetGroup->igNum, + emitLabelString(targetGroup)); + } +#endif + UNATIVE_OFFSET sizeRemoved = instructionDescriptor->idCodeSize(); + + // set state needed at the end of the loop in ADJUST_GROUP + adjIG = sizeRemoved; + lstIG = group; + jmpRemoved = true; + + // unlink the jump + if (previousJmp != nullptr) + { + previousJmp->idjNext = jmp->idjNext; + } + else + { + assert(jmp == emitJumpList); + emitJumpList = jmp->idjNext; + } + jmp->idjNext = nullptr; + +#if DEBUG + // clear the instruction data + memset((BYTE*)instructionDescriptor, 0, instructionDescriptor->idCodeSize()); +#endif + + // remove the instruction from the group + group->igInsCnt -= 1; + group->igSize -= sizeRemoved; + emitTotalCodeSize -= sizeRemoved; + + // flag the group as resized and recounted + group->igFlags |= (IGF_UPD_ISZ | IGF_UPD_ICOUNT); + + // cleanup + instructionDescriptor = nullptr; + jmp = nullptr; + + // jump to adjusting the size + goto ADJUST_GROUP; + } + } + } + } + + previousJmp = jmp; + if (jmp != nullptr) + { + jmp = jmp->idjNext; + } + } + + jmp = nullptr; + lstIG = nullptr; + adjLJ = 0; + adjIG = 0; + } +#endif + #if defined(TARGET_ARM) minMediumExtra = (UNATIVE_OFFSET)-1; #endif // TARGET_ARM @@ -4784,8 +4902,11 @@ void emitter::emitJumpDistBind() } // end for each jump - /* Did we shorten any jumps? */ +#ifdef TARGET_XARCH +ADJUST_GROUP: +#endif + /* Did we shorten any jumps? */ if (adjIG) { /* Adjust offsets of any remaining blocks */ @@ -4829,6 +4950,22 @@ void emitter::emitJumpDistBind() #endif #endif +#ifdef TARGET_XARCH + if (jmpRemoved) + { + + jmp_removal++; +#ifdef DEBUG + if (EMITVERBOSE) + { + printf("Iterating branch removal. Iteration = %d\n", jmp_removal); + } +#endif + jmpRemoved = false; + goto AGAIN; + } +#endif + if ((minShortExtra <= adjIG) #if defined(TARGET_ARM) || (minMediumExtra <= adjIG) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index e06b9552f7e81d..7b0fb7230e797f 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -159,7 +159,8 @@ class emitLocation return ig; } - int GetInsNum() const; + int GetInsNum() const; + void SetInsNum(int insNum); bool operator!=(const emitLocation& other) const { @@ -277,6 +278,9 @@ struct insGroup // and the emitter should continue to track GC info as if there was no new block. #define IGF_HAS_ALIGN 0x0400 // this group contains an alignment instruction(s) at the end to align either the next // IG, or, if this IG contains with an unconditional branch, some subsequent IG. +#define IGF_UPD_ICOUNT \ + 0x0800 // instruction was removed from the marked group and the live range end will not match this ig instruction + // count // Mask of IGF_* flags that should be propagated to new blocks when they are created. // This allows prologs and epilogs to be any number of IGs, but still be