Kill orphaned subprocesses on crash (#6366)

Co-authored-by: Lukas W <lukaswhl@gmail.com>

include/RemotePluginClient.h

@@ -27,6 +27,15 @@
 
 #include "RemotePluginBase.h"
 
+#ifndef LMMS_BUILD_WIN32
+#	include <condition_variable>
+#	include <mutex>
+#	include <thread>
+
+#	include <signal.h>
+#	include <unistd.h>
+#endif
+
 class RemotePluginClient : public RemotePluginBase
 {
 public:
@@ -126,6 +135,47 @@ private:
 	fpp_t m_bufferSize;
 } ;
 
+#ifndef LMMS_BUILD_WIN32
+class PollParentThread
+{
+public:
+	PollParentThread() :
+		m_stop{false},
+		m_thread{
+			[this]
+			{
+				using namespace std::literals::chrono_literals;
+				auto lock = std::unique_lock{m_mutex};
+				while (!m_cv.wait_for(lock, 500ms, [this] { return m_stop; }))
+				{
+					if (getppid() == 1)
+					{
+						kill(getpid(), SIGHUP);
+						break;
+					}
+				}
+			}
+		}
+	{ }
+
+	~PollParentThread()
+	{
+		{
+			const auto lock = std::unique_lock{m_mutex};
+			m_stop = true;
+		}
+		m_cv.notify_all();
+		m_thread.join();
+	}
+
+private:
+	bool m_stop;
+	std::mutex m_mutex;
+	std::condition_variable m_cv;
+	std::thread m_thread;
+};
+#endif
+
 #ifdef SYNC_WITH_SHM_FIFO
 RemotePluginClient::RemotePluginClient( key_t _shm_in, key_t _shm_out ) :
 	RemotePluginBase( new shmFifo( _shm_in ), new shmFifo( _shm_out ) ),

plugins/VstBase/RemoteVstPlugin.cpp

@@ -2473,6 +2473,11 @@ int main( int _argc, char * * _argv )
 		fprintf( stderr, "not enough arguments\n" );
 		return -1;
 	}
+
+#ifndef LMMS_BUILD_WIN32
+	const auto pollParentThread = PollParentThread{};
+#endif
+
 #ifndef NATIVE_LINUX_VST
 	OleInitialize(nullptr);
 #else

plugins/VstBase/RemoteVstPlugin/CMakeLists.txt

@@ -35,10 +35,13 @@ add_executable(${EXE_NAME} WIN32
 	${LMMS_COMMON_SRCS}
 )
 if(IS_WIN)
-    target_link_libraries(${EXE_NAME} ole32)
+	target_link_libraries(${EXE_NAME} ole32)
 else()
-    target_link_libraries(${EXE_NAME} pthread dl X11)
-ENDIF()
+	target_link_libraries(${EXE_NAME} dl X11)
+endif()
+if(NOT WIN32)
+	target_link_libraries(${EXE_NAME} pthread)
+endif()
 
 target_include_directories(${EXE_NAME}
 	PRIVATE 

plugins/ZynAddSubFx/RemoteZynAddSubFx.cpp

@@ -267,6 +267,10 @@ int main( int _argc, char * * _argv )
 		return -1;
 	}
 
+#ifndef LMMS_BUILD_WIN32
+	const auto pollParentThread = PollParentThread{};
+#endif
+
 #ifdef LMMS_BUILD_WIN32
 #ifndef __WINPTHREADS_VERSION
 	// (non-portable) initialization of statically linked pthread library

src/core/RemotePlugin.cpp

@@ -29,6 +29,10 @@
 #include <QDebug>
 #endif
 
+#ifdef LMMS_BUILD_WIN32
+#include <windows.h>
+#endif
+
 #include "BufferManager.h"
 #include "AudioEngine.h"
 #include "Engine.h"
@@ -43,6 +47,29 @@
 #include <sys/un.h>
 #endif
 
+#ifdef LMMS_BUILD_WIN32
+
+namespace {
+
+HANDLE getRemotePluginJob()
+{
+	static const auto job = []
+	{
+		const auto job = CreateJobObject(nullptr, nullptr);
+
+		auto limitInfo = JOBOBJECT_EXTENDED_LIMIT_INFORMATION{};
+		limitInfo.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+		SetInformationJobObject(job, JobObjectExtendedLimitInformation, &limitInfo, sizeof(limitInfo));
+
+		return job;
+	}();
+
+	return job;
+}
+
+} // namespace
+
+#endif // LMMS_BUILD_WIN32
 
 // simple helper thread monitoring our RemotePlugin - if process terminates
 // unexpectedly invalidate plugin so LMMS doesn't lock up
@@ -56,18 +83,41 @@ ProcessWatcher::ProcessWatcher( RemotePlugin * _p ) :
 
 void ProcessWatcher::run()
 {
-	m_plugin->m_process.start( m_plugin->m_exec, m_plugin->m_args );
-	exec();
-	m_plugin->m_process.moveToThread( m_plugin->thread() );
-	while( !m_quit && m_plugin->messagesLeft() )
-	{
-		msleep( 200 );
-	}
-	if( !m_quit )
-	{
-		fprintf( stderr,
-				"remote plugin died! invalidating now.\n" );
+	auto& process = m_plugin->m_process;
+	process.start(m_plugin->m_exec, m_plugin->m_args);
 
+#ifdef LMMS_BUILD_WIN32
+	// Add the process to our job so it is killed if we crash
+	if (process.waitForStarted(-1))
+	{
+		if (const auto processHandle = OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, false, process.processId()))
+		{
+			// Ensure the process is still running, otherwise the handle we
+			// obtained may be for a different process that happened to reuse
+			// the same process id.
+			// QProcess::state() alone is insufficient as it only returns a
+			// cached state variable that is updated asynchronously. To query
+			// the process itself, we can use QProcess::waitForFinished() with a
+			// zero timeout, but that too is insufficient as it fails if the
+			// process has already finished. Therefore, we check both.
+			if (!process.waitForFinished(0) && process.state() == QProcess::Running)
+			{
+				AssignProcessToJobObject(getRemotePluginJob(), processHandle);
+			}
+			CloseHandle(processHandle);
+		}
+	}
+#endif // LMMS_BUILD_WIN32
+
+	exec();
+	process.moveToThread(m_plugin->thread());
+	while (!m_quit && m_plugin->messagesLeft())
+	{
+		msleep(200);
+	}
+	if (!m_quit)
+	{
+		fprintf(stderr, "remote plugin died! invalidating now.\n");
 		m_plugin->invalidate();
 	}
 }