From 15f87360062d479e6f576e32d95a832ea808179c Mon Sep 17 00:00:00 2001 From: Guilherme Lampert Date: Thu, 22 Oct 2026 16:50:02 +1100 Subject: [PATCH] Pad the code after a label for what the branches to it leave in flight The scheduler pads each basic block for the results still in flight from the code before it in the program, and for nothing that a branch into the block leaves in flight: the clip flags from a clipw on the other side of a join were read three cycles after it, a register loaded just before a loop's label or in a branch's delay slot one to three cycles after the load, and a VI loaded before a join was tested four cycles after. The padding for the code that falls into a label runs one cycle short as well, so the same reads came one cycle early on that path too: a vertex colour's w lane moved in right before a join was stored a cycle before the move landed, and the store got the lane's old value. Each branch now records what its target inherits: the latencies still pending when it is taken, relative to the cycle the target's first instruction issues, as the most any branch to the label leaves. The program is emitted again with those merged into the emitter's own tracker at the labels, and the emitter pads the scheduled slots for whatever the schedule did not see, until the records stop growing; padding only moves a branch further from the writes before it, so they do. The schedule itself is computed once. Tests: test/integration/test_cross_block_latency.cpp (6 cases, 5 fail on master). Corpus: 16 of the 47 programs that compile change, all by added nop pairs: 11 of the 12 ps2gl fixtures, ps2gl's scei and 4 repros; the rest, quake2 and Tyra among them, are byte-identical. Co-Authored-By: Claude Fable 5.1 --- src/CodeGenerator.cpp | 90 +++- src/CodeGenerator.h | 12 +- src/VuLatencyTracker.cpp | 46 ++ src/VuLatencyTracker.h | 5 + test/CMakeLists.txt | 1 + test/integration/test_cross_block_latency.cpp | 425 ++++++++++++++++++ 6 files changed, 571 insertions(+), 8 deletions(-) create mode 100644 test/integration/test_cross_block_latency.cpp diff --git a/src/CodeGenerator.cpp b/src/CodeGenerator.cpp index fc76b46..874967d 100644 --- a/src/CodeGenerator.cpp +++ b/src/CodeGenerator.cpp @@ -951,6 +951,7 @@ CodeGenerator::CodeGenerator() m_strictScheduleSlots = false; m_enableUpperMoves = false; m_ignoredImplicitWawResources = VU_RESOURCE_NONE; + m_branchTargetStatesGrew = false; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1056,6 +1057,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) m_ignoredImplicitWawResources = VU_RESOURCE_NONE; const bool emitTypedScheduleDirectly = m_strictScheduleSlots || !m_knownLoopOptimizations; + VuScheduledProgram scheduledProgram; if( emitTypedScheduleDirectly ) { if( !appliedGenericSoftwarePipeline ) @@ -1063,8 +1065,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) fillBranchDelaySlots(workTokens); fillDeadFallthroughBranchDelaySlots(workTokens); } - if( !emitStrictScheduledProgram(workTokens, exitWritten) ) - return false; + scheduledProgram = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(workTokens); } else { @@ -1072,6 +1073,41 @@ bool CodeGenerator::beginProcess(const std::list& tokens) workTokens.swap(scheduledTokens); fillBranchDelaySlots(workTokens); fillDeadFallthroughBranchDelaySlots(workTokens); + } + + // The schedule pads each block for the results still in flight from the + // code before it in the program, not for those a branch into it leaves + // in flight. Each branch records what its target inherits, and the + // program is emitted again with that merged in at the labels, until the + // records stop growing: padding only ever moves a branch further from + // the writes before it, so they can grow a bounded number of times. + const std::list emittedTokens = workTokens; + const size_t headerLines = m_codeLines.size(); + for( unsigned int pass = 0; ; ++pass ) + { + if( pass == 64 ) + { + Error::Display( Error( "Internal error: latencies carried to branch targets did not settle" ) ); + return false; + } + + m_codeLines.resize(headerLines); + m_sourceComments.clear(); + m_currentCycle = 0; + m_latencyTracker.reset(); + m_branchTargetStatesGrew = false; + exitWritten = true; + + if( emitTypedScheduleDirectly ) + { + if( !emitStrictScheduledProgram(workTokens, scheduledProgram, exitWritten) ) + return false; + } + else + { + // the lookahead below takes the tokens it emits out of the list + workTokens.clear(); + workTokens.insert(workTokens.end(), emittedTokens.begin(), emittedTokens.end()); for( std::list::iterator k = workTokens.begin(); k != workTokens.end(); ) { @@ -1087,7 +1123,7 @@ bool CodeGenerator::beginProcess(const std::list& tokens) //handle label if(token.label().length()>0) - m_codeLines.push_back(token.label() + ":"); + addLabelLine(token.label()); if(!token.operand()) { @@ -1444,6 +1480,10 @@ bool CodeGenerator::beginProcess(const std::list& tokens) } } + if( !m_branchTargetStatesGrew ) + break; + } + if( !exitWritten ) { // Auto-terminate with an exit pair if the last instruction wasn't closed @@ -1493,6 +1533,17 @@ void CodeGenerator::addSourceComment( const Token& token ) m_sourceComments.push_back( std::make_pair( m_codeLines.size(), s.str() ) ); } +void CodeGenerator::addLabelLine( const std::string& label ) +{ + m_codeLines.push_back( label + ":" ); + + // the code after the label waits for what the branches to it leave in + // flight, as well as for what the code before it does + std::map::const_iterator state = m_branchTargetStates.find( label ); + if( state != m_branchTargetStates.end() ) + m_latencyTracker.merge( state->second, m_currentCycle, m_currentCycle ); +} + void CodeGenerator::emitWaitQ() { const int nextCycle = m_currentCycle + 1; @@ -1610,6 +1661,7 @@ void CodeGenerator::emitSingleToken( const Token& token ) addNopLine(); m_currentCycle++; } + recordBranchTargetState( token ); } else { @@ -1685,6 +1737,7 @@ void CodeGenerator::emitBranchWithDelayFiller( const Token& branch, const Token& m_codeLines.push_back(fillerLine); recordRegisterWrites(filler, m_currentCycle); m_currentCycle++; + recordBranchTargetState( branch ); } void CodeGenerator::emitPairedTokens( const Token& a, const Token& b ) @@ -1713,6 +1766,8 @@ void CodeGenerator::emitPairedTokens( const Token& a, const Token& b ) addNopLine(); m_currentCycle++; } + recordBranchTargetState( a ); + recordBranchTargetState( b ); } void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler ) @@ -1764,6 +1819,8 @@ void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token m_codeLines.push_back(fillerLine); recordRegisterWrites(filler, m_currentCycle); m_currentCycle++; + recordBranchTargetState( a ); + recordBranchTargetState( b ); } namespace @@ -1983,7 +2040,7 @@ bool CodeGenerator::prepareStrictScheduledToken( const Token& token, bool allowBranchDelayFiller ) { if( token.label().length() > 0 ) - m_codeLines.push_back(token.label() + ":"); + addLabelLine(token.label()); if( !token.operand() ) return false; @@ -2054,9 +2111,8 @@ void CodeGenerator::emitStrictScheduledPadding( const VuScheduledIssueSlot& slot } } -bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, bool& exitWritten ) +bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, const VuScheduledProgram& program, bool& exitWritten ) { - VuScheduledProgram program = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(tokens); std::vector slots; for( std::vector::const_iterator block = program.blocks.begin(); block != program.blocks.end(); @@ -2134,7 +2190,7 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, if( bake ) { // Emit the MAIN_LOOP label. - m_codeLines.push_back( bake->mainLabelTok->label() + ":" ); + addLabelLine( bake->mainLabelTok->label() ); // Smart-tail substitution: if the last cycle has an empty // lower lane, place the back-edge branch there to save a @@ -2230,6 +2286,14 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list& tokens, emitUpperWithWait(*first, slot.paddingKind == VU_SCHEDULED_PADDING_WAITQ); continue; } + + // the schedule's padding covers the block's own latencies; this adds + // what a branch into the block left in flight + if( firstEmits ) + padForReadHazards(*first, secondEmits ? second : NULL); + else if( secondEmits ) + padForReadHazards(*second, NULL); + const Token* branch = branchTokenInScheduledSlot(slot); const Token* delayFiller = NULL; unsigned int delayFillerSlot = i + 1; @@ -2366,6 +2430,18 @@ void CodeGenerator::recordRegisterWrites( const Token& token, int issueCycle ) m_latencyTracker.recordWrites( token, issueCycle, emitsAsUpperMove( token ) ); } +void CodeGenerator::recordBranchTargetState( const Token& branch ) +{ + std::string label; + if( vuTokenBranchDelaySlots( branch ) == 0 || !branchTargetLabel( branch, label ) ) + return; + + // the branch and its delay slot are emitted: the target's first + // instruction issues at m_currentCycle + if( m_branchTargetStates[label].merge( m_latencyTracker, -m_currentCycle, 0 ) ) + m_branchTargetStatesGrew = true; +} + unsigned int CodeGenerator::ignoredImplicitWawResourcesForRemaining( std::list::const_iterator begin, std::list::const_iterator end ) const { diff --git a/src/CodeGenerator.h b/src/CodeGenerator.h index fd8362f..c397660 100644 --- a/src/CodeGenerator.h +++ b/src/CodeGenerator.h @@ -69,6 +69,7 @@ class CodeGenerator void addNopLine(); void addSourceComment( const Token& token ); + void addLabelLine( const std::string& label ); void emitWaitQ(); void emitWaitP(); void emitUpperWithWait( const Token& token, bool waitQ ); @@ -78,12 +79,13 @@ class CodeGenerator void emitBranchWithDelayFiller( const Token& branch, const Token& filler ); void emitPairedTokens( const Token& a, const Token& b ); void emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler ); - bool emitStrictScheduledProgram( const std::list& tokens, bool& exitWritten ); + bool emitStrictScheduledProgram( const std::list& tokens, const VuScheduledProgram& program, bool& exitWritten ); void emitStrictScheduledPadding( const VuScheduledIssueSlot& slot ); bool prepareStrictScheduledToken( const Token& token, bool& exitWritten, bool allowBranchDelayFiller ); int readHazardDelay( const Token& token, const Token* partner ) const; void padForReadHazards( const Token& token, const Token* partner ); void recordRegisterWrites( const Token& token, int issueCycle ); + void recordBranchTargetState( const Token& branch ); unsigned int ignoredImplicitWawResourcesForRemaining( std::list::const_iterator begin, std::list::const_iterator end ) const; void fillPreIncrementStoreBranchDelaySlots( std::list& tokens ) const; @@ -208,6 +210,14 @@ class CodeGenerator int m_currentCycle; VuLatencyTracker m_latencyTracker; + // What the code at each label inherits from the branches to it: the + // latencies still pending when the branch is taken, relative to the cycle + // the target's first instruction issues, as the most any of them leaves. + // The program is emitted again with these merged in at the labels, until + // they stop growing. + std::map m_branchTargetStates; + bool m_branchTargetStatesGrew; + // 9.G-1h-4a-2: out-of-band MAIN-body ranges produced by the // generic kernel rewrite. Populated when OPENVCL_USE_GENERIC_KERNEL_REWRITE // is set and at least one plan is rewritten. Consumed in 4a-3. diff --git a/src/VuLatencyTracker.cpp b/src/VuLatencyTracker.cpp index 71ee7cb..6f1bdf6 100644 --- a/src/VuLatencyTracker.cpp +++ b/src/VuLatencyTracker.cpp @@ -31,6 +31,14 @@ namespace const VuInstructionInfo* info = findVuInstructionInfo( mnemonic ); return info ? static_cast( info->latency ) : fallback; } + + bool mergeCycle( int& cycle, int other, int pendingAfter ) + { + if( other <= cycle ) + return false; + cycle = other; + return other > pendingAfter; + } } VuLatencyTracker::VuLatencyTracker() @@ -48,6 +56,44 @@ void VuLatencyTracker::reset() m_registerProducerMnemonic.clear(); } +bool VuLatencyTracker::merge( const VuLatencyTracker& other, int shift, int now ) +{ + bool changed = false; + changed = mergeCycle( m_qReadyCycle, other.m_qReadyCycle + shift, now ) || changed; + changed = mergeCycle( m_pReadyCycle, other.m_pReadyCycle + shift, now ) || changed; + // the flags are read four cycles after they are written + changed = mergeCycle( m_lastFMACCycle, other.m_lastFMACCycle + shift, now - 4 ) || changed; + changed = mergeCycle( m_lastClipwCycle, other.m_lastClipwCycle + shift, now - 4 ) || changed; + + for( std::map::const_iterator i = other.m_registerReadyCycle.begin(); + i != other.m_registerReadyCycle.end(); ++i ) + { + const int ready = i->second + shift; + std::map::const_iterator producer = + other.m_registerProducerMnemonic.find( i->first ); + std::map::iterator mine = m_registerReadyCycle.find( i->first ); + if( mine == m_registerReadyCycle.end() ) + { + m_registerReadyCycle[i->first] = ready; + if( producer != other.m_registerProducerMnemonic.end() ) + m_registerProducerMnemonic[i->first] = producer->second; + changed = (ready > now) || changed; + continue; + } + + changed = mergeCycle( mine->second, ready, now ) || changed; + + // a bypass only applies when the same instruction writes the register + // on every path + std::map::iterator mineProducer = m_registerProducerMnemonic.find( i->first ); + if( mineProducer != m_registerProducerMnemonic.end() + && (producer == other.m_registerProducerMnemonic.end() || producer->second != mineProducer->second) ) + m_registerProducerMnemonic.erase( mineProducer ); + } + + return changed; +} + int VuLatencyTracker::readHazardDelay( const Token& token, const Token* partner, int currentCycle ) const diff --git a/src/VuLatencyTracker.h b/src/VuLatencyTracker.h index 2c14ff6..94fb913 100644 --- a/src/VuLatencyTracker.h +++ b/src/VuLatencyTracker.h @@ -21,6 +21,11 @@ class VuLatencyTracker VuLatencyTracker(); void reset(); + + // Takes on the pending latencies of another tracker, whose cycle c is + // this tracker's cycle c + shift: each ready cycle becomes the later of + // the two. Returns whether anything still pending at cycle `now` grew. + bool merge( const VuLatencyTracker& other, int shift, int now ); int readHazardDelay( const Token& token, const Token* partner, int currentCycle ) const; void recordWrites( const Token& token, int issueCycle, bool forceMacFlagWrite = false ); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df3c04e..bb7c73e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -80,6 +80,7 @@ set(OPENVCL_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_pairing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_literal_register_conflict.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_flag_latency.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_cross_block_latency.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_emit_source.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_continuation_lifetime.cpp ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_two_address_coalescing.cpp diff --git a/test/integration/test_cross_block_latency.cpp b/test/integration/test_cross_block_latency.cpp new file mode 100644 index 0000000..dc415eb --- /dev/null +++ b/test/integration/test_cross_block_latency.cpp @@ -0,0 +1,425 @@ +// Regression tests: results still in flight at a label or a branch are +// waited for on every path. +// +// The scheduler pads each basic block for the results still in flight from +// the code before it in the program, and for nothing a branch into the +// block leaves in flight. The clip flags from a clipw on the other side +// of a join, a register loaded just before a loop's label or in a branch's +// delay slot, a VI loaded before a join and tested right after it, a lane +// moved into a vertex colour on the lit path and stored at the join: all +// were read a cycle or more early on one of the paths. + +#include "test_harness.h" +#include "openvcl_runner.h" + +#include +#include +#include +#include + +namespace +{ + // Each path's clipw is a different distance from the fcand after the join. + const char* kClipJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tclipw.xyz fA, fA[w]\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tclipw.xyz fA, fA[w]\n" + "lJoin:\n" + "\tfcand vi01, 0x3f\n" + "\tiaddiu iOut, vi01, 0\n" + "\tisw.x iOut, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + const char* kMacJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tsub.xyz fB, fA, vf00\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tadd.xyz fB, fA, vf00\n" + "lJoin:\n" + "\tfmand iM, iFmt\n" + "\tisw.x iM, 2(vi00)\n" + "\tsq.xyz fB, 3(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded again right before the join on the path that falls into it. + const char* kLoadBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tlq fA, 3(vi00)\n" + "\tibeq iC, vi00, lJoin\n" + "\tlq fA, 1(vi00)\n" + "lJoin:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded right before the branch to the join, or in its delay slot. + const char* kLoadBeforeBranch = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tlq fA, 3(vi00)\n" + "\tibeq iC, vi00, lB\n" + "\tlq fA, 1(vi00)\n" + "\tb lJoin\n" + "lB:\n" + "\tiaddiu iT, vi00, 1\n" + "\tisw.x iT, 4(vi00)\n" + "lJoin:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // fA is loaded right before the loop, and late in each trip. + const char* kLoadBeforeLoop = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tiaddiu iN, vi00, 4\n" + "\tlq fA, 1(vi00)\n" + "lLoop:\n" + "\tadd fB, fA, fA\n" + "\tsq fB, 2(vi00)\n" + "\tiaddi iN, iN, -1\n" + "\tlq fA, 3(vi00)\n" + "\tibne iN, vi00, lLoop\n" + "\t--exit\n" + "\t--endexit\n"; + + // iD is loaded right before the join and tested right after it. + const char* kIntegerLoadBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iC, 0(vi00)\n" + "\tilw.y iD, 0(vi00)\n" + "\tibeq iC, vi00, lJoin\n" + "\tilw.z iD, 1(vi00)\n" + "lJoin:\n" + "\tibne iD, vi00, lOut\n" + "\tisw.x iD, 3(vi00)\n" + "lOut:\n" + "\tisw.y iD, 4(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // quake2's vertex colour: cleared before the branch, and on the lit path + // rewritten lane by lane right before the store at the join. + const char* kMaskedWritesBeforeJoin = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tlq fX, 1(vi00)\n" + "\tilw.x iC, 0(vi00)\n" + "\tmove vCol, vf00\n" + "\tibeq iC, vi00, lSkip\n" + "\tmove.xyz vCol, vf00\n" + "\tmove.w vCol, fX\n" + "lSkip:\n" + "\tsq vCol, 2(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // Nothing read after the join is in flight on either path. + const char* kJoinWithoutHazard = + "\t.init_vf_all\n" + "\t.init_vi_all\n" + "\t.syntax new\n" + "\t.name vsmCrossBlockLatency\n" + "\t--enter\n" + "\t--endenter\n" + "\tilw.x iFmt, 0(vi00)\n" + "\tlq fA, 1(vi00)\n" + "\tibeq iFmt, vi00, lFmtB\n" + "\tiaddiu iT, vi00, 1\n" + "\tb lJoin\n" + "lFmtB:\n" + "\tiaddiu iT, vi00, 2\n" + "lJoin:\n" + "\tisw.x iT, 2(vi00)\n" + "\tsq fA, 3(vi00)\n" + "\t--exit\n" + "\t--endexit\n"; + + // An emitted instruction pair, one issue cycle: its upper and lower + // instructions, and the pairs that can issue next. + struct Pair + { + std::string upper; + std::string lower; + std::vector next; + }; + + std::string mnemonic(const std::string& instruction) + { + return instruction.substr(0, instruction.find(' ')); + } + + // The register an instruction names first: what a load or an operation writes. + std::string destination(const std::string& instruction) + { + const std::string::size_type space = instruction.find(' '); + if (space == std::string::npos) + return std::string(); + const std::string::size_type comma = instruction.find(',', space); + return instruction.substr(space + 1, comma == std::string::npos ? std::string::npos : comma - space - 1); + } + + bool isBranch(const std::string& instruction) + { + const std::string op = mnemonic(instruction); + return op == "b" || op == "bal" || op.compare(0, 2, "ib") == 0 || op == "jr" || op == "jalr"; + } + + // The emitted pairs and the control flow between them: a branch takes + // effect after its delay slot, and the pair after one with the E bit is + // the last to run. + std::vector parse(const std::string& vsm, std::map& labels) + { + std::vector pairs; + std::vector pending; + std::stringstream stream(vsm); + std::string line; + while (std::getline(stream, line)) + { + if (line.compare(0, 20, std::string(20, ' ')) == 0) + { + Pair pair; + const std::string::size_type start = line.find_first_not_of(' '); + const std::string::size_type split = line.find(" ", start); + pair.upper = line.substr(start, split == std::string::npos ? std::string::npos : split - start); + if (split != std::string::npos) + pair.lower = line.substr(line.find_first_not_of(' ', split)); + for (unsigned int i = 0; i < pending.size(); ++i) + labels[pending[i]] = static_cast(pairs.size()); + pending.clear(); + pairs.push_back(pair); + } + else if (!line.empty() && line[0] != ' ' && line[0] != '\t' && line[line.size() - 1] == ':') + pending.push_back(line.substr(0, line.size() - 1)); + } + + for (unsigned int i = 0; i < pairs.size(); ++i) + { + if (i > 0 && pairs[i - 1].upper.find("[E]") != std::string::npos) + continue; + if (i > 0 && isBranch(pairs[i - 1].lower)) + { + const std::string& branch = pairs[i - 1].lower; + const std::map::const_iterator target = labels.find(branch.substr(branch.rfind(' ') + 1)); + if (target != labels.end()) + pairs[i].next.push_back(target->second); + if (mnemonic(branch).compare(0, 2, "ib") == 0 && i + 1 < pairs.size()) + pairs[i].next.push_back(i + 1); + } + else if (i + 1 < pairs.size()) + pairs[i].next.push_back(i + 1); + } + return pairs; + } + + // The first pair with this instruction in either half. + unsigned int find(const std::vector& pairs, const std::string& op) + { + for (unsigned int i = 0; i < pairs.size(); ++i) + { + if (mnemonic(pairs[i].upper) == op || mnemonic(pairs[i].lower) == op) + return i; + } + return static_cast(pairs.size()); + } + + // The fewest cycles from a pair for which `produces` holds to the + // consumer over any path, or -1 when none lies within `limit`. + int nearestProducer(const std::vector& pairs, unsigned int consumer, bool (*produces)(const Pair&), int limit) + { + std::vector< std::vector > previous(pairs.size()); + for (unsigned int i = 0; i < pairs.size(); ++i) + { + for (unsigned int n = 0; n < pairs[i].next.size(); ++n) + previous[pairs[i].next[n]].push_back(i); + } + + int nearest = -1; + std::map seen; + std::vector< std::pair > work; + for (unsigned int p = 0; p < previous[consumer].size(); ++p) + work.push_back(std::make_pair(previous[consumer][p], 1)); + while (!work.empty()) + { + const unsigned int i = work.back().first; + const int distance = work.back().second; + work.pop_back(); + if (distance >= limit || (seen.count(i) && seen[i] <= distance)) + continue; + seen[i] = distance; + if (produces(pairs[i])) + { + if (nearest < 0 || distance < nearest) + nearest = distance; + continue; + } + for (unsigned int p = 0; p < previous[i].size(); ++p) + work.push_back(std::make_pair(previous[i][p], distance + 1)); + } + return nearest; + } + + bool writesClipFlags(const Pair& pair) + { + return mnemonic(pair.upper).compare(0, 5, "clipw") == 0; + } + + bool writesMacFlags(const Pair& pair) + { + return mnemonic(pair.upper) != "nop"; + } + + bool loadsVF01(const Pair& pair) + { + return mnemonic(pair.lower) == "lq" && destination(pair.lower) == "VF01"; + } + + bool loadsVI02(const Pair& pair) + { + return mnemonic(pair.lower).compare(0, 3, "ilw") == 0 && destination(pair.lower) == "VI02"; + } + + // The lane move: a lower instruction with a four-cycle latency. + bool movesIntoVF02(const Pair& pair) + { + return mnemonic(pair.lower).compare(0, 4, "move") == 0 && destination(pair.lower) == "VF02"; + } + + // The field move, which the emitter may turn into an upper max. + bool writesFieldsOfVF02(const Pair& pair) + { + return mnemonic(pair.upper) != "nop" && destination(pair.upper) == "VF02"; + } + + ::test::RunResult compile(const char* source) + { + return ::test::run_openvcl(std::vector(), source); + } +} + +TEST_CASE("openvcl: the clip flags are read four cycles after the clipw on every path into a join") +{ + ::test::RunResult result = compile(kClipJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int fcand = find(code, "fcand"); + REQUIRE(fcand < code.size()); + CHECK(nearestProducer(code, fcand, writesClipFlags, 4) == -1); +} + +TEST_CASE("openvcl: the MAC flags are read four cycles after the operation on every path into a join") +{ + ::test::RunResult result = compile(kMacJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int fmand = find(code, "fmand"); + REQUIRE(fmand < code.size()); + CHECK(nearestProducer(code, fmand, writesMacFlags, 4) == -1); +} + +TEST_CASE("openvcl: a register loaded before a label or a branch is read four cycles later on every path") +{ + const char* sources[] = { kLoadBeforeJoin, kLoadBeforeBranch, kLoadBeforeLoop }; + for (unsigned int i = 0; i < 3; ++i) + { + ::test::RunResult result = compile(sources[i]); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int add = find(code, "add.xyzw"); + REQUIRE(add < code.size()); + CHECK(nearestProducer(code, add, loadsVF01, 4) == -1); + } +} + +TEST_CASE("openvcl: a VI loaded before a join is tested five cycles later on every path") +{ + ::test::RunResult result = compile(kIntegerLoadBeforeJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int ibne = find(code, "ibne"); + REQUIRE(ibne < code.size()); + CHECK(nearestProducer(code, ibne, loadsVI02, 5) == -1); +} + +TEST_CASE("openvcl: a vertex colour written lane by lane before a join is stored after the lanes land") +{ + ::test::RunResult result = compile(kMaskedWritesBeforeJoin); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + const unsigned int clear = find(code, "move"); + const unsigned int branch = find(code, "ibeq"); + const unsigned int store = find(code, "sq"); + REQUIRE(store < code.size()); + CHECK(clear < branch); + CHECK(branch < store); + CHECK(nearestProducer(code, store, movesIntoVF02, 5) == -1); + CHECK(nearestProducer(code, store, writesFieldsOfVF02, 4) == -1); +} + +TEST_CASE("openvcl: a join with nothing in flight on either path gets no padding") +{ + ::test::RunResult result = compile(kJoinWithoutHazard); + REQUIRE(result.exit_code == 0); + + std::map labels; + const std::vector code = parse(result.stdout_data, labels); + REQUIRE(labels.count("lJoin") == 1); + CHECK(mnemonic(code[labels["lJoin"]].lower) == "isw.x"); +}