From cc2ae66540cdf62a9f6b9d4daf5cb4b82d229fa3 Mon Sep 17 00:00:00 2001
From: DomClark <mrdomclark@gmail.com>
Date: Tue, 14 Aug 2018 20:52:48 +0100
Subject: [PATCH] Fix hang when updateInOutCount called from processReplacing

Ignore requests to change the I/O count from within processReplacing and print a warning instead; the shared memory is in use so it can't be reallocated. Add a special case to return immediately if the I/O count hasn't changed at all; this will prevent spurious warnings when the plugin is only updating the latency and should reduce unnecessary reallocations in general.
---
 plugins/vst_base/RemoteVstPlugin.cpp | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/plugins/vst_base/RemoteVstPlugin.cpp b/plugins/vst_base/RemoteVstPlugin.cpp
index 5a73bd10e..e0b51381e 100644
--- a/plugins/vst_base/RemoteVstPlugin.cpp
+++ b/plugins/vst_base/RemoteVstPlugin.cpp
@@ -118,6 +118,7 @@ class RemoteVstPlugin;
 RemoteVstPlugin * __plugin = NULL;
 
 HWND __MessageHwnd = NULL;
+DWORD __processingThreadId = 0;
 
 
 
@@ -251,7 +252,7 @@ public:
 	}
 
 	// has to be called as soon as input- or output-count changes
-	void updateInOutCount();
+	int updateInOutCount();
 
 	inline void lockShm()
 	{
@@ -1438,8 +1439,21 @@ void RemoteVstPlugin::loadChunkFromFile( const std::string & _file, int _len )
 
 
 
-void RemoteVstPlugin::updateInOutCount()
+int RemoteVstPlugin::updateInOutCount()
 {
+	if( inputCount() == RemotePluginClient::inputCount() &&
+		outputCount() == RemotePluginClient::outputCount() )
+	{
+		return 1;
+	}
+	
+	if( GetCurrentThreadId() == __processingThreadId )
+	{
+		debugMessage( "Plugin requested I/O change from processing "
+			"thread. Request denied; stability may suffer.\n" );
+		return 0;
+	}
+
 	lockShm();
 
 	setShmIsValid( false );
@@ -1467,6 +1481,8 @@ void RemoteVstPlugin::updateInOutCount()
 	{
 		m_outputs = new float * [outputCount()];
 	}
+
+	return 1;
 }
 
 
@@ -1611,10 +1627,9 @@ intptr_t RemoteVstPlugin::hostCallback( AEffect * _effect, int32_t _opcode,
 			return 0;
 
 		case audioMasterIOChanged:
-			__plugin->updateInOutCount();
 			SHOW_CALLBACK( "amc: audioMasterIOChanged\n" );
-			// numInputs and/or numOutputs has changed
-			return 0;
+			// numInputs, numOutputs, and/or latency has changed
+			return __plugin->updateInOutCount();
 
 #ifdef OLD_VST_SDK
 		case audioMasterWantMidi:
@@ -1897,6 +1912,8 @@ void RemoteVstPlugin::processUIThreadMessages()
 
 DWORD WINAPI RemoteVstPlugin::processingThread( LPVOID _param )
 {
+	__processingThreadId = GetCurrentThreadId();
+
 	RemoteVstPlugin * _this = static_cast<RemoteVstPlugin *>( _param );
 
 	RemotePluginClient::message m;