diff --git a/src/CodeGenerator.cpp b/src/CodeGenerator.cpp index fc76b46..874967d 100644 --- a/src/CodeGenerator.cpp +++ b/src/CodeGenerator.cpp @@ -951,6 +951,7 @@ CodeGenerator::CodeGenerator() m_strictScheduleSlots = false; m_enableUpperMoves = false; m_ignoredImplicitWawResources = VU_RESOURCE_NONE; + m_branchTargetStatesGrew = false; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1056,6 +1057,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) m_ignoredImplicitWawResources = VU_RESOURCE_NONE; const bool emitTypedScheduleDirectly = m_strictScheduleSlots || !m_knownLoopOptimizations; + VuScheduledProgram scheduledProgram; if( emitTypedScheduleDirectly ) { if( !appliedGenericSoftwarePipeline ) @@ -1063,8 +1065,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) fillBranchDelaySlots(workTokens); fillDeadFallthroughBranchDelaySlots(workTokens); } - if( !emitStrictScheduledProgram(workTokens, exitWritten) ) - return false; + scheduledProgram = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(workTokens); } else { @@ -1072,6 +1073,41 @@ bool CodeGenerator::beginProcess(const std::list& tokens) workTokens.swap(scheduledTokens); fillBranchDelaySlots(workTokens); fillDeadFallthroughBranchDelaySlots(workTokens); + } + + // The schedule pads each block for the results still in flight from the + // code before it in the program, not for those a branch into it leaves + // in flight. Each branch records what its target inherits, and the + // program is emitted again with that merged in at the labels, until the + // records stop growing: padding only ever moves a branch further from + // the writes before it, so they can grow a bounded number of times. + const std::list emittedTokens = workTokens; + const size_t headerLines = m_codeLines.size(); + for( unsigned int pass = 0; ; ++pass ) + { + if( pass == 64 ) + { + Error::Display( Error( "Internal error: latencies carried to branch targets did not settle" ) ); + return false; + } + + m_codeLines.resize(headerLines); + m_sourceComments.clear(); + m_currentCycle = 0; + m_latencyTracker.reset(); + m_branchTargetStatesGrew = false; + exitWritten = true; + + if( emitTypedScheduleDirectly ) + { + if( !emitStrictScheduledProgram(workTokens, scheduledProgram, exitWritten) ) + return false; + } + else + { + // the lookahead below takes the tokens it emits out of the list + workTokens.clear(); + workTokens.insert(workTokens.end(), emittedTokens.begin(), emittedTokens.end()); for( std::list::iterator k = workTokens.begin(); k != workTokens.end(); ) { @@ -1087,7 +1123,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) //handle label if(token.label().length()>0) - m_codeLines.push_back(token.label() + ":"); + addLabelLine(token.label()); if(!token.operand()) { @@ -1444,6 +1480,10 @@ bool CodeGenerator::beginProcess(const std::list& tokens) } } + if( !m_branchTargetStatesGrew ) + break; + } + if( !exitWritten ) { // Auto-terminate with an exit pair if the last instruction wasn't closed @@ -1493,6 +1533,17 @@ void CodeGenerator::addSourceComment( const Token& token ) m_sourceComments.push_back( std::make_pair( m_codeLines.size(), s.str() ) ); } +void CodeGenerator::addLabelLine( const std::string& label ) +{ + m_codeLines.push_back( label + ":" ); + + // the code after the label waits for what the branches to it leave in + // flight, as well as for what the code before it does + std::map::const_iterator state = m_branchTargetStates.find( label ); + if( state != m_branchTargetStates.end() ) + m_latencyTracker.merge( state->second, m_currentCycle, m_currentCycle ); +} + void CodeGenerator::emitWaitQ() { const int nextCycle = m_currentCycle + 1; @@ -1610,6 +1661,7 @@ void CodeGenerator::emitSingleToken( const Token& token ) addNopLine(); m_currentCycle++; } + recordBranchTargetState( token ); } else { @@ -1685,6 +1737,7 @@ void CodeGenerator::emitBranchWithDelayFiller( const Token& branch, const Token& m_codeLines.push_back(fillerLine); recordRegisterWrites(filler, m_currentCycle); m_currentCycle++; + recordBranchTargetState( branch ); } void CodeGenerator::emitPairedTokens( const Token& a, const Token& b ) @@ -1713,6 +1766,8 @@ void CodeGenerator::emitPairedTokens( const Token& a, const Token& b ) addNopLine(); m_currentCycle++; } + recordBranchTargetState( a ); + recordBranchTargetState( b ); } void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler ) @@ -1764,6 +1819,8 @@ void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token m_codeLines.push_back(fillerLine); recordRegisterWrites(filler, m_currentCycle); m_currentCycle++; + recordBranchTargetState( a ); + recordBranchTargetState( b ); } namespace @@ -1983,7 +2040,7 @@ bool CodeGenerator::prepareStrictScheduledToken( const Token& token, bool allowBranchDelayFiller ) { if( token.label().length() > 0 ) - m_codeLines.push_back(token.label() + ":"); + addLabelLine(token.label()); if( !token.operand() ) return false; @@ -2054,9 +2111,8 @@ void CodeGenerator::emitStrictScheduledPadding( const VuScheduledIssueSlot& slot } } -bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, bool& exitWritten ) +bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, const VuScheduledProgram& program, bool& exitWritten ) { - VuScheduledProgram program = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(tokens); std::vector slots; for( std::vector::const_iterator block = program.blocks.begin(); block != program.blocks.end(); @@ -2134,7 +2190,7 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, if( bake ) { // Emit the MAIN_LOOP label. - m_codeLines.push_back( bake->mainLabelTok->label() + ":" ); + addLabelLine( bake->mainLabelTok->label() ); // Smart-tail substitution: if the last cycle has an empty // lower lane, place the back-edge branch there to save a @@ -2230,6 +2286,14 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, emitUpperWithWait(*first, slot.paddingKind == VU_SCHEDULED_PADDING_WAITQ); continue; } + + // the schedule's padding covers the block's own latencies; this adds + // what a branch into the block left in flight + if( firstEmits ) + padForReadHazards(*first, secondEmits ? second : NULL); + else if( secondEmits ) + padForReadHazards(*second, NULL); + const Token* branch = branchTokenInScheduledSlot(slot); const Token* delayFiller = NULL; unsigned int delayFillerSlot = i + 1; @@ -2366,6 +2430,18 @@ void CodeGenerator::recordRegisterWrites( const Token& token, int issueCycle ) m_latencyTracker.recordWrites( token, issueCycle, emitsAsUpperMove( token ) ); } +void CodeGenerator::recordBranchTargetState( const Token& branch ) +{ + std::string label; + if( vuTokenBranchDelaySlots( branch ) == 0 || !branchTargetLabel( branch, label ) ) + return; + + // the branch and its delay slot are emitted: the target's first + // instruction issues at m_currentCycle + if( m_branchTargetStates[label].merge( m_latencyTracker, -m_currentCycle, 0 ) ) + m_branchTargetStatesGrew = true; +} + unsigned int CodeGenerator::ignoredImplicitWawResourcesForRemaining( std::list::const_iterator begin, std::list::const_iterator end ) const { diff --git a/src/CodeGenerator.h b/src/CodeGenerator.h index fd8362f..c397660 100644 --- a/src/CodeGenerator.h +++ b/src/CodeGenerator.h @@ -69,6 +69,7 @@ class CodeGenerator void addNopLine(); void addSourceComment( const Token& token ); + void addLabelLine( const std::string& label ); void emitWaitQ(); void emitWaitP(); void emitUpperWithWait( const Token& token, bool waitQ ); @@ -78,12 +79,13 @@ class CodeGenerator void emitBranchWithDelayFiller( const Token& branch, const Token& filler ); void emitPairedTokens( const Token& a, const Token& b ); void emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler ); - bool emitStrictScheduledProgram( const std::list& tokens, bool& exitWritten ); + bool emitStrictScheduledProgram( const std::list& tokens, const VuScheduledProgram& program, bool& exitWritten ); void emitStrictScheduledPadding( const VuScheduledIssueSlot& slot ); bool prepareStrictScheduledToken( const Token& token, bool& exitWritten, bool allowBranchDelayFiller ); int readHazardDelay( const Token& token, const Token* partner ) const; void padForReadHazards( const Token& token, const Token* partner ); void recordRegisterWrites( const Token& token, int issueCycle ); + void recordBranchTargetState( const Token& branch ); unsigned int ignoredImplicitWawResourcesForRemaining( std::list::const_iterator begin, std::list::const_iterator end ) const; void fillPreIncrementStoreBranchDelaySlots( std::list& tokens ) const; @@ -208,6 +210,14 @@ class CodeGenerator int m_currentCycle; VuLatencyTracker m_latencyTracker; + // What the code at each label inherits from the branches to it: the + // latencies still pending when the branch is taken, relative to the cycle + // the target's first instruction issues, as the most any of them leaves. + // The program is emitted again with these merged in at the labels, until + // they stop growing. + std::map m_branchTargetStates; + bool m_branchTargetStatesGrew; + // 9.G-1h-4a-2: out-of-band MAIN-body ranges produced by the // generic kernel rewrite. Populated when OPENVCL_USE_GENERIC_KERNEL_REWRITE // is set and at least one plan is rewritten. Consumed in 4a-3. diff --git a/src/VuLatencyTracker.cpp b/src/VuLatencyTracker.cpp index 71ee7cb..6f1bdf6 100644 --- a/src/VuLatencyTracker.cpp +++ b/src/VuLatencyTracker.cpp @@ -31,6 +31,14 @@ namespace const VuInstructionInfo* info = findVuInstructionInfo( mnemonic ); return info ? static_cast( info->latency ) : fallback; } + + bool mergeCycle( int& cycle, int other, int pendingAfter ) + { + if( other <= cycle ) + return false; + cycle = other; + return other > pendingAfter; + } } VuLatencyTracker::VuLatencyTracker() @@ -48,6 +56,44 @@ void VuLatencyTracker::reset() m_registerProducerMnemonic.clear(); } +bool VuLatencyTracker::merge( const VuLatencyTracker& other, int shift, int now ) +{ + bool changed = false; + changed = mergeCycle( m_qReadyCycle, other.m_qReadyCycle + shift, now ) || changed; + changed = mergeCycle( m_pReadyCycle, other.m_pReadyCycle + shift, now ) || changed; + // the flags are read four cycles after they are written + changed = mergeCycle( m_lastFMACCycle, other.m_lastFMACCycle + shift, now - 4 ) || changed; + changed = mergeCycle( m_lastClipwCycle, other.m_lastClipwCycle + shift, now - 4 ) || changed; + + for( std::map::const_iterator i = other.m_registerReadyCycle.begin(); + i != other.m_registerReadyCycle.end(); ++i ) + { + const int ready = i->second + shift; + std::map::const_iterator producer = + other.m_registerProducerMnemonic.find( i->first ); + std::map::iterator mine = m_registerReadyCycle.find( i->first ); + if( mine == m_registerReadyCycle.end() ) + { + m_registerReadyCycle[i->first] = ready; + if( producer != other.m_registerProducerMnemonic.end() ) + m_registerProducerMnemonic[i->first] = producer->second; + changed = (ready > now) || changed; + continue; + } + + changed = mergeCycle( mine->second, ready, now ) || changed; + + // a bypass only applies when the same instruction writes the register + // on every path + std::map::iterator mineProducer = m_registerProducerMnemonic.find( i->first ); + if( mineProducer != m_registerProducerMnemonic.end() + && (producer == other.m_registerProducerMnemonic.end() || producer->second != mineProducer->second) ) + m_registerProducerMnemonic.erase( mineProducer ); + } + + return changed; +} + int VuLatencyTracker::readHazardDelay( const Token& token, const Token* partner, int currentCycle ) const diff --git a/src/VuLatencyTracker.h b/src/VuLatencyTracker.h index 2c14ff6..94fb913 100644 --- a/src/VuLatencyTracker.h +++ b/src/VuLatencyTracker.h @@ -21,6 +21,11 @@ class VuLatencyTracker VuLatencyTracker(); void reset(); + + // Takes on the pending latencies of another tracker, whose cycle c is + // this tracker's cycle c + shift: each ready cycle becomes the later of + // the two. Returns whether anything still pending at cycle `now` grew. + bool merge( const VuLatencyTracker& other, int shift, int now ); int readHazardDelay( const Token& token, const Token* partner, int currentCycle ) const; void recordWrites( const Token& token, int issueCycle, bool forceMacFlagWrite = false ); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df3c04e..bb7c73e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -80,6 +80,7 @@ set(OPENVCL_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_pairing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_literal_register_conflict.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_flag_latency.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_cross_block_latency.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_emit_source.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_continuation_lifetime.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_two_address_coalescing.cpp diff --git a/test/integration/test_cross_block_latency.cpp b/test/integration/test_cross_block_latency.cpp new file mode 100644 index 0000000..dc415eb --- /dev/null +++ b/test/integration/test_cross_block_latency.cpp @@ -0,0 +1,425 @@ +// Regression tests: results still in flight at a label or a branch are +// waited for on every path. +// +// The scheduler pads each basic block for the results still in flight from +// the code before it in the program, and for nothing a branch into the +// block leaves in flight. The clip flags from a clipw on the other side +// of a join, a register loaded just before a loop's label or in a branch's +// delay slot, a VI loaded before a join and tested right after it, a lane +// moved into a vertex colour on the lit path and stored at the join: all +// were read a cycle or more early on one of the paths. + +#include "test_harness.h" +#include "openvcl_runner.h" + +#include +#include +#include +#include + +namespace +{ + // Each path's clipw is a different distance from the fcand after the join. + const char* kClipJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tclipw.xyz fA, fA[w]\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tclipw.xyz fA, fA[w]\n" + "lJoin:\n" + "\tfcand vi01, 0x3f\n" + "\tiaddiu iOut, vi01, 0\n" + "\tisw.x iOut, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + const char* kMacJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tsub.xyz fB, fA, vf00\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tadd.xyz fB, fA, vf00\n" + "lJoin:\n" + "\tfmand iM, iFmt\n" + "\tisw.x iM, 2(vi00)\n" + "\tsq.xyz fB, 3(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded again right before the join on the path that falls into it. + const char* kLoadBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tlq fA, 3(vi00)\n" + "\tibeq iC, vi00, lJoin\n" + "\tlq fA, 1(vi00)\n" + "lJoin:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded right before the branch to the join, or in its delay slot. + const char* kLoadBeforeBranch = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tlq fA, 3(vi00)\n" + "\tibeq iC, vi00, lB\n" + "\tlq fA, 1(vi00)\n" + "\tb lJoin\n" + "lB:\n" + "\tiaddiu iT, vi00, 1\n" + "\tisw.x iT, 4(vi00)\n" + "lJoin:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded right before the loop, and late in each trip. + const char* kLoadBeforeLoop = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tiaddiu iN, vi00, 4\n" + "\tlq fA, 1(vi00)\n" + "lLoop:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\tiaddi iN, iN, -1\n" + "\tlq fA, 3(vi00)\n" + "\tibne iN, vi00, lLoop\n" + "\t--exit\n" + "\t--endexit\n"; + + // iD is loaded right before the join and tested right after it. + const char* kIntegerLoadBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tilw.y iD, 0(vi00)\n" + "\tibeq iC, vi00, lJoin\n" + "\tilw.z iD, 1(vi00)\n" + "lJoin:\n" + "\tibne iD, vi00, lOut\n" + "\tisw.x iD, 3(vi00)\n" + "lOut:\n" + "\tisw.y iD, 4(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // quake2's vertex colour: cleared before the branch, and on the lit path + // rewritten lane by lane right before the store at the join. + const char* kMaskedWritesBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tlq fX, 1(vi00)\n" + "\tilw.x iC, 0(vi00)\n" + "\tmove vCol, vf00\n" + "\tibeq iC, vi00, lSkip\n" + "\tmove.xyz vCol, vf00\n" + "\tmove.w vCol, fX\n" + "lSkip:\n" + "\tsq vCol, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // Nothing read after the join is in flight on either path. + const char* kJoinWithoutHazard = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tiaddiu iT, vi00, 1\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tiaddiu iT, vi00, 2\n" + "lJoin:\n" + "\tisw.x iT, 2(vi00)\n" + "\tsq fA, 3(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // An emitted instruction pair, one issue cycle: its upper and lower + // instructions, and the pairs that can issue next. + struct Pair + { + std::string upper; + std::string lower; + std::vector next; + }; + + std::string mnemonic(const std::string& instruction) + { + return instruction.substr(0, instruction.find(' ')); + } + + // The register an instruction names first: what a load or an operation writes. + std::string destination(const std::string& instruction) + { + const std::string::size_type space = instruction.find(' '); + if (space == std::string::npos) + return std::string(); + const std::string::size_type comma = instruction.find(',', space); + return instruction.substr(space + 1, comma == std::string::npos ? std::string::npos : comma - space - 1); + } + + bool isBranch(const std::string& instruction) + { + const std::string op = mnemonic(instruction); + return op == "b" || op == "bal" || op.compare(0, 2, "ib") == 0 || op == "jr" || op == "jalr"; + } + + // The emitted pairs and the control flow between them: a branch takes + // effect after its delay slot, and the pair after one with the E bit is + // the last to run. + std::vector parse(const std::string& vsm, std::map& labels) + { + std::vector pairs; + std::vector pending; + std::stringstream stream(vsm); + std::string line; + while (std::getline(stream, line)) + { + if (line.compare(0, 20, std::string(20, ' ')) == 0) + { + Pair pair; + const std::string::size_type start = line.find_first_not_of(' '); + const std::string::size_type split = line.find(" ", start); + pair.upper = line.substr(start, split == std::string::npos ? std::string::npos : split - start); + if (split != std::string::npos) + pair.lower = line.substr(line.find_first_not_of(' ', split)); + for (unsigned int i = 0; i < pending.size(); ++i) + labels[pending[i]] = static_cast(pairs.size()); + pending.clear(); + pairs.push_back(pair); + } + else if (!line.empty() && line[0] != ' ' && line[0] != '\t' && line[line.size() - 1] == ':') + pending.push_back(line.substr(0, line.size() - 1)); + } + + for (unsigned int i = 0; i < pairs.size(); ++i) + { + if (i > 0 && pairs[i - 1].upper.find("[E]") != std::string::npos) + continue; + if (i > 0 && isBranch(pairs[i - 1].lower)) + { + const std::string& branch = pairs[i - 1].lower; + const std::map::const_iterator target = labels.find(branch.substr(branch.rfind(' ') + 1)); + if (target != labels.end()) + pairs[i].next.push_back(target->second); + if (mnemonic(branch).compare(0, 2, "ib") == 0 && i + 1 < pairs.size()) + pairs[i].next.push_back(i + 1); + } + else if (i + 1 < pairs.size()) + pairs[i].next.push_back(i + 1); + } + return pairs; + } + + // The first pair with this instruction in either half. + unsigned int find(const std::vector& pairs, const std::string& op) + { + for (unsigned int i = 0; i < pairs.size(); ++i) + { + if (mnemonic(pairs[i].upper) == op || mnemonic(pairs[i].lower) == op) + return i; + } + return static_cast(pairs.size()); + } + + // The fewest cycles from a pair for which `produces` holds to the + // consumer over any path, or -1 when none lies within `limit`. + int nearestProducer(const std::vector& pairs, unsigned int consumer, bool (*produces)(const Pair&), int limit) + { + std::vector< std::vector > previous(pairs.size()); + for (unsigned int i = 0; i < pairs.size(); ++i) + { + for (unsigned int n = 0; n < pairs[i].next.size(); ++n) + previous[pairs[i].next[n]].push_back(i); + } + + int nearest = -1; + std::map seen; + std::vector< std::pair > work; + for (unsigned int p = 0; p < previous[consumer].size(); ++p) + work.push_back(std::make_pair(previous[consumer][p], 1)); + while (!work.empty()) + { + const unsigned int i = work.back().first; + const int distance = work.back().second; + work.pop_back(); + if (distance >= limit || (seen.count(i) && seen[i] <= distance)) + continue; + seen[i] = distance; + if (produces(pairs[i])) + { + if (nearest < 0 || distance < nearest) + nearest = distance; + continue; + } + for (unsigned int p = 0; p < previous[i].size(); ++p) + work.push_back(std::make_pair(previous[i][p], distance + 1)); + } + return nearest; + } + + bool writesClipFlags(const Pair& pair) + { + return mnemonic(pair.upper).compare(0, 5, "clipw") == 0; + } + + bool writesMacFlags(const Pair& pair) + { + return mnemonic(pair.upper) != "nop"; + } + + bool loadsVF01(const Pair& pair) + { + return mnemonic(pair.lower) == "lq" && destination(pair.lower) == "VF01"; + } + + bool loadsVI02(const Pair& pair) + { + return mnemonic(pair.lower).compare(0, 3, "ilw") == 0 && destination(pair.lower) == "VI02"; + } + + // The lane move: a lower instruction with a four-cycle latency. + bool movesIntoVF02(const Pair& pair) + { + return mnemonic(pair.lower).compare(0, 4, "move") == 0 && destination(pair.lower) == "VF02"; + } + + // The field move, which the emitter may turn into an upper max. + bool writesFieldsOfVF02(const Pair& pair) + { + return mnemonic(pair.upper) != "nop" && destination(pair.upper) == "VF02"; + } + + ::test::RunResult compile(const char* source) + { + return ::test::run_openvcl(std::vector(), source); + } +} + +TEST_CASE("openvcl: the clip flags are read four cycles after the clipw on every path into a join") +{ + ::test::RunResult result = compile(kClipJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int fcand = find(code, "fcand"); + REQUIRE(fcand < code.size()); + CHECK(nearestProducer(code, fcand, writesClipFlags, 4) == -1); +} + +TEST_CASE("openvcl: the MAC flags are read four cycles after the operation on every path into a join") +{ + ::test::RunResult result = compile(kMacJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int fmand = find(code, "fmand"); + REQUIRE(fmand < code.size()); + CHECK(nearestProducer(code, fmand, writesMacFlags, 4) == -1); +} + +TEST_CASE("openvcl: a register loaded before a label or a branch is read four cycles later on every path") +{ + const char* sources[] = { kLoadBeforeJoin, kLoadBeforeBranch, kLoadBeforeLoop }; + for (unsigned int i = 0; i < 3; ++i) + { + ::test::RunResult result = compile(sources[i]); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int add = find(code, "add.xyzw"); + REQUIRE(add < code.size()); + CHECK(nearestProducer(code, add, loadsVF01, 4) == -1); + } +} + +TEST_CASE("openvcl: a VI loaded before a join is tested five cycles later on every path") +{ + ::test::RunResult result = compile(kIntegerLoadBeforeJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int ibne = find(code, "ibne"); + REQUIRE(ibne < code.size()); + CHECK(nearestProducer(code, ibne, loadsVI02, 5) == -1); +} + +TEST_CASE("openvcl: a vertex colour written lane by lane before a join is stored after the lanes land") +{ + ::test::RunResult result = compile(kMaskedWritesBeforeJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int clear = find(code, "move"); + const unsigned int branch = find(code, "ibeq"); + const unsigned int store = find(code, "sq"); + REQUIRE(store < code.size()); + CHECK(clear < branch); + CHECK(branch < store); + CHECK(nearestProducer(code, store, movesIntoVF02, 5) == -1); + CHECK(nearestProducer(code, store, writesFieldsOfVF02, 4) == -1); +} + +TEST_CASE("openvcl: a join with nothing in flight on either path gets no padding") +{ + ::test::RunResult result = compile(kJoinWithoutHazard); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + REQUIRE(labels.count("lJoin") == 1); + CHECK(mnemonic(code[labels["lJoin"]].lower) == "isw.x"); +}