Enable 64bit VSTs on Linux

* Remove trial-and-error approach of detecting VST's machine types. Read PE
  headers instead.
* Add RemoteVstPlugin64 to AppImage

cmake/linux/package_linux.sh.in

@@ -126,7 +126,7 @@ EOL
 
 chmod +x "${APPDIR}usr/bin/lmms"
 
-# Per https://github.com/probonopd/linuxdeployqt/issues/129 
+# Per https://github.com/probonopd/linuxdeployqt/issues/129
 unset LD_LIBRARY_PATH
 
 # Ensure linuxdeployqt can find shared objects
@@ -139,13 +139,16 @@ fi
 
 # Move executables so linuxdeployqt can find them
 ZYNLIB="${APPDIR}usr/lib/lmms/RemoteZynAddSubFx"
-VSTLIB="${APPDIR}usr/lib/lmms/RemoteVstPlugin32.exe.so"
+VSTLIB32="${APPDIR}usr/lib/lmms/RemoteVstPlugin32.exe.so"
+VSTLIB64="${APPDIR}usr/lib/lmms/RemoteVstPlugin64.exe.so"
 
 ZYNBIN="${APPDIR}usr/bin/RemoteZynAddSubFx"
-VSTBIN="${APPDIR}usr/bin/RemoteVstPlugin32.exe.so"
+VSTBIN32="${APPDIR}usr/bin/RemoteVstPlugin32.exe.so"
+VSTBIN64="${APPDIR}usr/bin/RemoteVstPlugin64.exe.so"
 
 mv "$ZYNLIB" "$ZYNBIN"
-mv "$VSTLIB" "$VSTBIN"
+mv "$VSTLIB32" "$VSTBIN32"
+mv "$VSTLIB64" "$VSTBIN64"
 
 # Patch the desktop file
 sed -i 's/.*Exec=.*/Exec=lmms.real/' "$DESKTOPFILE"
@@ -171,7 +174,8 @@ success "Bundled and relinked dependencies"
 
 # Link to original location so lmms can find them
 ln -sr  "$ZYNBIN" "$ZYNLIB"
-ln -sr  "$VSTBIN" "$VSTLIB"
+ln -sr  "$VSTBIN32" "$VSTLIB32"
+ln -sr  "$VSTBIN64" "$VSTLIB64"
 
 # Remove wine library conflict
 rm -f "${APPDIR}/usr/lib/libwine.so.1"

include/RemotePlugin.h

@@ -854,12 +854,11 @@ protected:
 	}
 
 
+	bool m_failed;
 private:
 	void resizeSharedProcessingMemory();
 
 
-	bool m_failed;
-
 	QProcess m_process;
 	ProcessWatcher m_watcher;
 

plugins/vst_base/RemoteVstPlugin.cpp

@@ -811,11 +811,6 @@ bool RemoteVstPlugin::load( const std::string & _plugin_file )
 	{
 		DWORD error = GetLastError();
 		debugMessage( "LoadLibrary failed: " + GetErrorAsString(error) );
-		// give VstPlugin class a chance to start 32 bit version of RemoteVstPlugin
-		if( GetLastError() == ERROR_BAD_EXE_FORMAT )
-		{
-			sendMessage( IdVstBadDllFormat );
-		}
 		return false;
 	}
 

plugins/vst_base/VstPlugin.cpp

@@ -26,6 +26,7 @@
 
 #include "communication.h"
 
+#include <QtCore/QtEndian>
 #include <QtCore/QDebug>
 #include <QDir>
 #include <QFileInfo>
@@ -68,6 +69,54 @@
 #	include <X11/Xlib.h>
 #endif
 
+namespace PE
+{
+// Utilities for reading PE file machine type
+// See specification at https://msdn.microsoft.com/library/windows/desktop/ms680547(v=vs.85).aspx
+
+enum MachineType : uint16_t
+{
+	IMAGE_FILE_MACHINE_UNKNOWN = 0x0,
+	IMAGE_FILE_MACHINE_AMD64 = 0x8664,
+	IMAGE_FILE_MACHINE_I386 = 0x14c,
+};
+
+class FileInfo
+{
+public:
+	FileInfo(QString filePath)
+		: m_file(filePath)
+	{
+		m_file.open(QFile::ReadOnly);
+		m_map = m_file.map(0, m_file.size());
+		if (m_map == nullptr) {
+			throw std::runtime_error("Cannot map file");
+		}
+	}
+	~FileInfo()
+	{
+		m_file.unmap(m_map);
+	}
+
+	MachineType machineType()
+	{
+		int32_t peOffset = qFromLittleEndian(* reinterpret_cast<int32_t*>(m_map + 0x3C));
+		uchar* peSignature = m_map + peOffset;
+		if (memcmp(peSignature, "PE\0\0", 4)) {
+			throw std::runtime_error("Invalid PE file");
+		}
+		uchar * coffHeader = peSignature + 4;
+		uint16_t machineType = qFromLittleEndian(* reinterpret_cast<uint16_t*>(coffHeader));
+		return static_cast<MachineType>(machineType);
+	}
+
+private:
+	QFile m_file;
+	uchar* m_map;
+};
+
+}
+
 
 VstPlugin::VstPlugin( const QString & _plugin ) :
 	m_plugin( _plugin ),
@@ -75,22 +124,32 @@ VstPlugin::VstPlugin( const QString & _plugin ) :
 	m_embedMethod( gui
 			? ConfigManager::inst()->vstEmbedMethod()
 			: "headless" ),
-	m_badDllFormat( false ),
 	m_version( 0 ),
 	m_currentProgram()
 {
 	setSplittedChannels( true );
 
-#ifdef LMMS_BUILD_WIN64
-	tryLoad( "RemoteVstPlugin64" );
-	if( m_badDllFormat )
-	{
-		m_badDllFormat = false;
-#endif
-		tryLoad( "RemoteVstPlugin32" );
-#ifdef LMMS_BUILD_WIN64
+	PE::MachineType machineType;
+	try {
+		PE::FileInfo peInfo(_plugin);
+		machineType = peInfo.machineType();
+	} catch (std::runtime_error& e) {
+		qCritical() << "Error while determining PE file's machine type: " << e.what();
+		machineType = PE::IMAGE_FILE_MACHINE_UNKNOWN;
+	}
+
+	switch(machineType)
+	{
+	case PE::IMAGE_FILE_MACHINE_AMD64:
+		tryLoad( "RemoteVstPlugin64" );
+		break;
+	case PE::IMAGE_FILE_MACHINE_I386:
+		tryLoad( "RemoteVstPlugin32" );
+		break;
+	default:
+		m_failed = true;
+		return;
 	}
-#endif
 
 	setTempo( Engine::getSong()->getTempo() );
 
@@ -326,10 +385,6 @@ bool VstPlugin::processMessage( const message & _m )
 {
 	switch( _m.id )
 	{
-	case IdVstBadDllFormat:
-		m_badDllFormat = true;
-		break;
-
 	case IdVstPluginWindowID:
 		m_pluginWindowID = _m.getInt();
 		if( m_embedMethod == "none" )

plugins/vst_base/VstPlugin.h

@@ -142,8 +142,6 @@ private:
 	QSize m_pluginGeometry;
 	const QString m_embedMethod;
 
-	bool m_badDllFormat;
-
 	QString m_name;
 	int m_version;
 	QString m_vendorString;

plugins/vst_base/communication.h

@@ -70,7 +70,6 @@ enum VstRemoteMessageIDs
 
 	// remoteVstPlugin -> vstPlugin
 	IdVstFailedLoadingPlugin,
-	IdVstBadDllFormat,
 	IdVstPluginWindowID,
 	IdVstPluginEditorGeometry,
 	IdVstPluginName,

src/core/RemotePlugin.cpp

@@ -63,7 +63,10 @@ void ProcessWatcher::run()
 	{
 		fprintf( stderr,
 				"remote plugin died! invalidating now.\n" );
+
+#ifndef SYNC_WITH_SHM_FIFO
 		m_plugin->invalidate();
+#endif
 	}
 }
 
@@ -464,9 +467,6 @@ void RemotePlugin::processFinished( int exitCode,
 	{
 		qCritical() << "Remote plugin exit code: " << exitCode;
 	}
-#ifndef SYNC_WITH_SHM_FIFO
-	invalidate();
-#endif
 }
 
 void RemotePlugin::processErrored( QProcess::ProcessError err )