set "child subreaper" process attribute on Linux (#7315)

When launching the wine VST process various wrappers may be involved,
when they exit the VST process becomes orphaned. This breaks the
PollParentThread mechanism which is responsible for cleaning up
processes in case of a crash. Because of this 64bit VST process exits
prematurely, in other words 64bit VST is currently broken in a typical
wine configuration.

A solution suggested by Lukas W is to set the PR_SET_CHILD_SUBREAPER
flag which makes the kernel reparent such process to lmms and
PollParentThread then works as intended.

Co-authored-by: Lukas W <lukaswhl@gmail.com>

CMakeLists.txt

@@ -154,6 +154,7 @@ CHECK_INCLUDE_FILES(sys/types.h LMMS_HAVE_SYS_TYPES_H)
 CHECK_INCLUDE_FILES(sys/ipc.h LMMS_HAVE_SYS_IPC_H)
 CHECK_INCLUDE_FILES(sys/time.h LMMS_HAVE_SYS_TIME_H)
 CHECK_INCLUDE_FILES(sys/times.h LMMS_HAVE_SYS_TIMES_H)
+CHECK_INCLUDE_FILES(sys/prctl.h LMMS_HAVE_SYS_PRCTL_H)
 CHECK_INCLUDE_FILES(sched.h LMMS_HAVE_SCHED_H)
 CHECK_INCLUDE_FILES(sys/soundcard.h LMMS_HAVE_SYS_SOUNDCARD_H)
 CHECK_INCLUDE_FILES(soundcard.h LMMS_HAVE_SOUNDCARD_H)

src/core/main.cpp

@@ -55,6 +55,10 @@
 #include <unistd.h>
 #endif
 
+#ifdef LMMS_HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+
 #include <csignal>
 
 #include "MainApplication.h"
@@ -293,6 +297,15 @@ int main( int argc, char * * argv )
 	qInstallMessageHandler(consoleMessageHandler);
 #endif
 
+#if defined(LMMS_HAVE_SYS_PRCTL_H) && defined(PR_SET_CHILD_SUBREAPER)
+	// Set the "child subreaper" attribute so that plugin child processes remain as lmms'
+	// children even when some wrapper process exits, as it may happen with wine
+	if (prctl(PR_SET_CHILD_SUBREAPER, 1))
+	{
+		perror("prctl(PR_SET_CHILD_SUBREAPER)");
+	}
+#endif
+
 	// initialize memory managers
 	NotePlayHandleManager::init();
 

src/lmmsconfig.h.in

@@ -45,6 +45,7 @@
 #cmakedefine LMMS_HAVE_SEMAPHORE_H
 #cmakedefine LMMS_HAVE_SYS_TIME_H
 #cmakedefine LMMS_HAVE_SYS_TIMES_H
+#cmakedefine LMMS_HAVE_SYS_PRCTL_H
 #cmakedefine LMMS_HAVE_SCHED_H
 #cmakedefine LMMS_HAVE_SYS_SOUNDCARD_H
 #cmakedefine LMMS_HAVE_SOUNDCARD_H