Enable 64bit VSTs on Linux

* Remove trial-and-error approach of detecting VST's machine types. Read PE
  headers instead.
* Add RemoteVstPlugin64 to AppImage

plugins/vst_base/RemoteVstPlugin.cpp

@@ -811,11 +811,6 @@ bool RemoteVstPlugin::load( const std::string & _plugin_file )
 	{
 		DWORD error = GetLastError();
 		debugMessage( "LoadLibrary failed: " + GetErrorAsString(error) );
-		// give VstPlugin class a chance to start 32 bit version of RemoteVstPlugin
-		if( GetLastError() == ERROR_BAD_EXE_FORMAT )
-		{
-			sendMessage( IdVstBadDllFormat );
-		}
 		return false;
 	}
 

plugins/vst_base/VstPlugin.cpp

@@ -26,6 +26,7 @@
 
 #include "communication.h"
 
+#include <QtCore/QtEndian>
 #include <QtCore/QDebug>
 #include <QDir>
 #include <QFileInfo>
@@ -68,6 +69,54 @@
 #	include <X11/Xlib.h>
 #endif
 
+namespace PE
+{
+// Utilities for reading PE file machine type
+// See specification at https://msdn.microsoft.com/library/windows/desktop/ms680547(v=vs.85).aspx
+
+enum MachineType : uint16_t
+{
+	IMAGE_FILE_MACHINE_UNKNOWN = 0x0,
+	IMAGE_FILE_MACHINE_AMD64 = 0x8664,
+	IMAGE_FILE_MACHINE_I386 = 0x14c,
+};
+
+class FileInfo
+{
+public:
+	FileInfo(QString filePath)
+		: m_file(filePath)
+	{
+		m_file.open(QFile::ReadOnly);
+		m_map = m_file.map(0, m_file.size());
+		if (m_map == nullptr) {
+			throw std::runtime_error("Cannot map file");
+		}
+	}
+	~FileInfo()
+	{
+		m_file.unmap(m_map);
+	}
+
+	MachineType machineType()
+	{
+		int32_t peOffset = qFromLittleEndian(* reinterpret_cast<int32_t*>(m_map + 0x3C));
+		uchar* peSignature = m_map + peOffset;
+		if (memcmp(peSignature, "PE\0\0", 4)) {
+			throw std::runtime_error("Invalid PE file");
+		}
+		uchar * coffHeader = peSignature + 4;
+		uint16_t machineType = qFromLittleEndian(* reinterpret_cast<uint16_t*>(coffHeader));
+		return static_cast<MachineType>(machineType);
+	}
+
+private:
+	QFile m_file;
+	uchar* m_map;
+};
+
+}
+
 
 VstPlugin::VstPlugin( const QString & _plugin ) :
 	m_plugin( _plugin ),
@@ -75,22 +124,32 @@ VstPlugin::VstPlugin( const QString & _plugin ) :
 	m_embedMethod( gui
 			? ConfigManager::inst()->vstEmbedMethod()
 			: "headless" ),
-	m_badDllFormat( false ),
 	m_version( 0 ),
 	m_currentProgram()
 {
 	setSplittedChannels( true );
 
-#ifdef LMMS_BUILD_WIN64
-	tryLoad( "RemoteVstPlugin64" );
-	if( m_badDllFormat )
-	{
-		m_badDllFormat = false;
-#endif
-		tryLoad( "RemoteVstPlugin32" );
-#ifdef LMMS_BUILD_WIN64
+	PE::MachineType machineType;
+	try {
+		PE::FileInfo peInfo(_plugin);
+		machineType = peInfo.machineType();
+	} catch (std::runtime_error& e) {
+		qCritical() << "Error while determining PE file's machine type: " << e.what();
+		machineType = PE::IMAGE_FILE_MACHINE_UNKNOWN;
+	}
+
+	switch(machineType)
+	{
+	case PE::IMAGE_FILE_MACHINE_AMD64:
+		tryLoad( "RemoteVstPlugin64" );
+		break;
+	case PE::IMAGE_FILE_MACHINE_I386:
+		tryLoad( "RemoteVstPlugin32" );
+		break;
+	default:
+		m_failed = true;
+		return;
 	}
-#endif
 
 	setTempo( Engine::getSong()->getTempo() );
 
@@ -326,10 +385,6 @@ bool VstPlugin::processMessage( const message & _m )
 {
 	switch( _m.id )
 	{
-	case IdVstBadDllFormat:
-		m_badDllFormat = true;
-		break;
-
 	case IdVstPluginWindowID:
 		m_pluginWindowID = _m.getInt();
 		if( m_embedMethod == "none" )

plugins/vst_base/VstPlugin.h

@@ -142,8 +142,6 @@ private:
 	QSize m_pluginGeometry;
 	const QString m_embedMethod;
 
-	bool m_badDllFormat;
-
 	QString m_name;
 	int m_version;
 	QString m_vendorString;

plugins/vst_base/communication.h

@@ -70,7 +70,6 @@ enum VstRemoteMessageIDs
 
 	// remoteVstPlugin -> vstPlugin
 	IdVstFailedLoadingPlugin,
-	IdVstBadDllFormat,
 	IdVstPluginWindowID,
 	IdVstPluginEditorGeometry,
 	IdVstPluginName,