Pad the code after a label for what the branches to it leave in flight.

src/CodeGenerator.cpp

@@ -951,6 +951,7 @@ CodeGenerator::CodeGenerator()
 	m_strictScheduleSlots = false;
 	m_enableUpperMoves = false;
 	m_ignoredImplicitWawResources = VU_RESOURCE_NONE;
+	m_branchTargetStatesGrew = false;
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1056,22 +1057,57 @@ bool CodeGenerator::beginProcess(const std::list<Token>& tokens)
 	m_ignoredImplicitWawResources = VU_RESOURCE_NONE;
 
 	const bool emitTypedScheduleDirectly = m_strictScheduleSlots || !m_knownLoopOptimizations;
+	VuScheduledProgram scheduledProgram;
 	if( emitTypedScheduleDirectly )
 	{
 		if( !appliedGenericSoftwarePipeline )
 		{
 			fillBranchDelaySlots(workTokens);
 			fillDeadFallthroughBranchDelaySlots(workTokens);
 		}
-		if( !emitStrictScheduledProgram(workTokens, exitWritten) )
-			return false;
+		scheduledProgram = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(workTokens);
 	}
 	else
 	{
 		std::list<Token> scheduledTokens = scheduleVuTokensReadySetWithFlagLiveness(workTokens);
 		workTokens.swap(scheduledTokens);
 		fillBranchDelaySlots(workTokens);
 		fillDeadFallthroughBranchDelaySlots(workTokens);
+	}
+
+	// The schedule pads each block for the results still in flight from the
+	// code before it in the program, not for those a branch into it leaves
+	// in flight. Each branch records what its target inherits, and the
+	// program is emitted again with that merged in at the labels, until the
+	// records stop growing: padding only ever moves a branch further from
+	// the writes before it, so they can grow a bounded number of times.
+	const std::list<Token> emittedTokens = workTokens;
+	const size_t headerLines = m_codeLines.size();
+	for( unsigned int pass = 0; ; ++pass )
+	{
+		if( pass == 64 )
+		{
+			Error::Display( Error( "Internal error: latencies carried to branch targets did not settle" ) );
+			return false;
+		}
+
+		m_codeLines.resize(headerLines);
+		m_sourceComments.clear();
+		m_currentCycle = 0;
+		m_latencyTracker.reset();
+		m_branchTargetStatesGrew = false;
+		exitWritten = true;
+
+	if( emitTypedScheduleDirectly )
+	{
+		if( !emitStrictScheduledProgram(workTokens, scheduledProgram, exitWritten) )
+			return false;
+	}
+	else
+	{
+		// the lookahead below takes the tokens it emits out of the list
+		workTokens.clear();
+		workTokens.insert(workTokens.end(), emittedTokens.begin(), emittedTokens.end());
 
 		for( std::list<Token>::iterator k = workTokens.begin(); k != workTokens.end(); )
 		{
@@ -1087,7 +1123,7 @@ bool CodeGenerator::beginProcess(const std::list<Token>& tokens)
 
 		//handle label
 		if(token.label().length()>0)
-			m_codeLines.push_back(token.label() + ":");
+			addLabelLine(token.label());
 
 		if(!token.operand())
 		{
@@ -1444,6 +1480,10 @@ bool CodeGenerator::beginProcess(const std::list<Token>& tokens)
 	}
 	}
 
+		if( !m_branchTargetStatesGrew )
+			break;
+	}
+
 	if( !exitWritten )
 	{
 		// Auto-terminate with an exit pair if the last instruction wasn't closed
@@ -1493,6 +1533,17 @@ void CodeGenerator::addSourceComment( const Token& token )
 	m_sourceComments.push_back( std::make_pair( m_codeLines.size(), s.str() ) );
 }
 
+void CodeGenerator::addLabelLine( const std::string& label )
+{
+	m_codeLines.push_back( label + ":" );
+
+	// the code after the label waits for what the branches to it leave in
+	// flight, as well as for what the code before it does
+	std::map<std::string, VuLatencyTracker>::const_iterator state = m_branchTargetStates.find( label );
+	if( state != m_branchTargetStates.end() )
+		m_latencyTracker.merge( state->second, m_currentCycle, m_currentCycle );
+}
+
 void CodeGenerator::emitWaitQ()
 {
 	const int nextCycle = m_currentCycle + 1;
@@ -1610,6 +1661,7 @@ void CodeGenerator::emitSingleToken( const Token& token )
 			addNopLine();
 			m_currentCycle++;
 		}
+		recordBranchTargetState( token );
 	}
 	else
 	{
@@ -1685,6 +1737,7 @@ void CodeGenerator::emitBranchWithDelayFiller( const Token& branch, const Token&
 	m_codeLines.push_back(fillerLine);
 	recordRegisterWrites(filler, m_currentCycle);
 	m_currentCycle++;
+	recordBranchTargetState( branch );
 }
 
 void CodeGenerator::emitPairedTokens( const Token& a, const Token& b )
@@ -1713,6 +1766,8 @@ void CodeGenerator::emitPairedTokens( const Token& a, const Token& b )
 		addNopLine();
 		m_currentCycle++;
 	}
+	recordBranchTargetState( a );
+	recordBranchTargetState( b );
 }
 
 void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler )
@@ -1764,6 +1819,8 @@ void CodeGenerator::emitPairedBranchWithDelayFiller( const Token& a, const Token
 	m_codeLines.push_back(fillerLine);
 	recordRegisterWrites(filler, m_currentCycle);
 	m_currentCycle++;
+	recordBranchTargetState( a );
+	recordBranchTargetState( b );
 }
 
 namespace
@@ -1983,7 +2040,7 @@ bool CodeGenerator::prepareStrictScheduledToken( const Token& token,
                                                  bool allowBranchDelayFiller )
 {
 	if( token.label().length() > 0 )
-		m_codeLines.push_back(token.label() + ":");
+		addLabelLine(token.label());
 
 	if( !token.operand() )
 		return false;
@@ -2054,9 +2111,8 @@ void CodeGenerator::emitStrictScheduledPadding( const VuScheduledIssueSlot& slot
 	}
 }
 
-bool CodeGenerator::emitStrictScheduledProgram( const std::list<Token>& tokens, bool& exitWritten )
+bool CodeGenerator::emitStrictScheduledProgram( const std::list<Token>& tokens, const VuScheduledProgram& program, bool& exitWritten )
 {
-	VuScheduledProgram program = scheduleVuProgramReadyIssueSlotsWithFlagLiveness(tokens);
 	std::vector<const VuScheduledIssueSlot*> slots;
 	for( std::vector<VuScheduledBasicBlock>::const_iterator block = program.blocks.begin();
 	     block != program.blocks.end();
@@ -2134,7 +2190,7 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list<Token>& tokens,
 			if( bake )
 			{
 				// Emit the MAIN_LOOP label.
-				m_codeLines.push_back( bake->mainLabelTok->label() + ":" );
+				addLabelLine( bake->mainLabelTok->label() );
 
 				// Smart-tail substitution: if the last cycle has an empty
 				// lower lane, place the back-edge branch there to save a
@@ -2230,6 +2286,14 @@ bool CodeGenerator::emitStrictScheduledProgram( const std::list<Token>& tokens,
 			emitUpperWithWait(*first, slot.paddingKind == VU_SCHEDULED_PADDING_WAITQ);
 			continue;
 		}
+
+		// the schedule's padding covers the block's own latencies; this adds
+		// what a branch into the block left in flight
+		if( firstEmits )
+			padForReadHazards(*first, secondEmits ? second : NULL);
+		else if( secondEmits )
+			padForReadHazards(*second, NULL);
+
 		const Token* branch = branchTokenInScheduledSlot(slot);
 		const Token* delayFiller = NULL;
 		unsigned int delayFillerSlot = i + 1;
@@ -2366,6 +2430,18 @@ void CodeGenerator::recordRegisterWrites( const Token& token, int issueCycle )
 	m_latencyTracker.recordWrites( token, issueCycle, emitsAsUpperMove( token ) );
 }
 
+void CodeGenerator::recordBranchTargetState( const Token& branch )
+{
+	std::string label;
+	if( vuTokenBranchDelaySlots( branch ) == 0 || !branchTargetLabel( branch, label ) )
+		return;
+
+	// the branch and its delay slot are emitted: the target's first
+	// instruction issues at m_currentCycle
+	if( m_branchTargetStates[label].merge( m_latencyTracker, -m_currentCycle, 0 ) )
+		m_branchTargetStatesGrew = true;
+}
+
 unsigned int CodeGenerator::ignoredImplicitWawResourcesForRemaining( std::list<Token>::const_iterator begin,
                                                                      std::list<Token>::const_iterator end ) const
 {

src/CodeGenerator.h

@@ -69,6 +69,7 @@ class CodeGenerator
 
 	void addNopLine();
 	void addSourceComment( const Token& token );
+	void addLabelLine( const std::string& label );
 	void emitWaitQ();
 	void emitWaitP();
 	void emitUpperWithWait( const Token& token, bool waitQ );
@@ -78,12 +79,13 @@ class CodeGenerator
 	void emitBranchWithDelayFiller( const Token& branch, const Token& filler );
 	void emitPairedTokens( const Token& a, const Token& b );
 	void emitPairedBranchWithDelayFiller( const Token& a, const Token& b, const Token& filler );
-	bool emitStrictScheduledProgram( const std::list<Token>& tokens, bool& exitWritten );
+	bool emitStrictScheduledProgram( const std::list<Token>& tokens, const VuScheduledProgram& program, bool& exitWritten );
 	void emitStrictScheduledPadding( const VuScheduledIssueSlot& slot );
 	bool prepareStrictScheduledToken( const Token& token, bool& exitWritten, bool allowBranchDelayFiller );
 	int readHazardDelay( const Token& token, const Token* partner ) const;
 	void padForReadHazards( const Token& token, const Token* partner );
 	void recordRegisterWrites( const Token& token, int issueCycle );
+	void recordBranchTargetState( const Token& branch );
 	unsigned int ignoredImplicitWawResourcesForRemaining( std::list<Token>::const_iterator begin,
 	                                                       std::list<Token>::const_iterator end ) const;
 	void fillPreIncrementStoreBranchDelaySlots( std::list<Token>& tokens ) const;
@@ -208,6 +210,14 @@ class CodeGenerator
 	int m_currentCycle;
 	VuLatencyTracker m_latencyTracker;
 
+	// What the code at each label inherits from the branches to it: the
+	// latencies still pending when the branch is taken, relative to the cycle
+	// the target's first instruction issues, as the most any of them leaves.
+	// The program is emitted again with these merged in at the labels, until
+	// they stop growing.
+	std::map<std::string, VuLatencyTracker> m_branchTargetStates;
+	bool m_branchTargetStatesGrew;
+
 	// 9.G-1h-4a-2: out-of-band MAIN-body ranges produced by the
 	// generic kernel rewrite. Populated when OPENVCL_USE_GENERIC_KERNEL_REWRITE
 	// is set and at least one plan is rewritten. Consumed in 4a-3.

src/VuLatencyTracker.cpp

@@ -31,6 +31,14 @@ namespace
 		const VuInstructionInfo* info = findVuInstructionInfo( mnemonic );
 		return info ? static_cast<int>( info->latency ) : fallback;
 	}
+
+	bool mergeCycle( int& cycle, int other, int pendingAfter )
+	{
+		if( other <= cycle )
+			return false;
+		cycle = other;
+		return other > pendingAfter;
+	}
 }
 
 VuLatencyTracker::VuLatencyTracker()
@@ -48,6 +56,44 @@ void VuLatencyTracker::reset()
 	m_registerProducerMnemonic.clear();
 }
 
+bool VuLatencyTracker::merge( const VuLatencyTracker& other, int shift, int now )
+{
+	bool changed = false;
+	changed = mergeCycle( m_qReadyCycle, other.m_qReadyCycle + shift, now ) || changed;
+	changed = mergeCycle( m_pReadyCycle, other.m_pReadyCycle + shift, now ) || changed;
+	// the flags are read four cycles after they are written
+	changed = mergeCycle( m_lastFMACCycle, other.m_lastFMACCycle + shift, now - 4 ) || changed;
+	changed = mergeCycle( m_lastClipwCycle, other.m_lastClipwCycle + shift, now - 4 ) || changed;
+
+	for( std::map<std::string, int>::const_iterator i = other.m_registerReadyCycle.begin();
+	     i != other.m_registerReadyCycle.end(); ++i )
+	{
+		const int ready = i->second + shift;
+		std::map<std::string, std::string>::const_iterator producer =
+		    other.m_registerProducerMnemonic.find( i->first );
+		std::map<std::string, int>::iterator mine = m_registerReadyCycle.find( i->first );
+		if( mine == m_registerReadyCycle.end() )
+		{
+			m_registerReadyCycle[i->first] = ready;
+			if( producer != other.m_registerProducerMnemonic.end() )
+				m_registerProducerMnemonic[i->first] = producer->second;
+			changed = (ready > now) || changed;
+			continue;
+		}
+
+		changed = mergeCycle( mine->second, ready, now ) || changed;
+
+		// a bypass only applies when the same instruction writes the register
+		// on every path
+		std::map<std::string, std::string>::iterator mineProducer = m_registerProducerMnemonic.find( i->first );
+		if( mineProducer != m_registerProducerMnemonic.end()
+		    && (producer == other.m_registerProducerMnemonic.end() || producer->second != mineProducer->second) )
+			m_registerProducerMnemonic.erase( mineProducer );
+	}
+
+	return changed;
+}
+
 int VuLatencyTracker::readHazardDelay( const Token& token,
                                        const Token* partner,
                                        int currentCycle ) const

src/VuLatencyTracker.h

@@ -21,6 +21,11 @@ class VuLatencyTracker
 	VuLatencyTracker();
 
 	void reset();
+
+	// Takes on the pending latencies of another tracker, whose cycle c is
+	// this tracker's cycle c + shift: each ready cycle becomes the later of
+	// the two. Returns whether anything still pending at cycle `now` grew.
+	bool merge( const VuLatencyTracker& other, int shift, int now );
 	int readHazardDelay( const Token& token, const Token* partner, int currentCycle ) const;
 	void recordWrites( const Token& token, int issueCycle, bool forceMacFlagWrite = false );
 

test/CMakeLists.txt

@@ -80,6 +80,7 @@ set(OPENVCL_TEST_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_pairing.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_literal_register_conflict.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_flag_latency.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_cross_block_latency.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_emit_source.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_continuation_lifetime.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/integration/test_two_address_coalescing.cpp