From 9db8cbfb31f9b6d3e754887a6280770e3d3e2bcd Mon Sep 17 00:00:00 2001 From: Lukas W Date: Thu, 7 Jun 2018 11:07:40 +0200 Subject: [PATCH] Enable 64bit VSTs on Linux * Remove trial-and-error approach of detecting VST's machine types. Read PE headers instead. * Add RemoteVstPlugin64 to AppImage --- cmake/linux/package_linux.sh.in | 14 +++-- include/RemotePlugin.h | 3 +- plugins/vst_base/RemoteVstPlugin.cpp | 5 -- plugins/vst_base/VstPlugin.cpp | 83 +++++++++++++++++++++++----- plugins/vst_base/VstPlugin.h | 2 - plugins/vst_base/communication.h | 1 - src/core/RemotePlugin.cpp | 6 +- 7 files changed, 82 insertions(+), 32 deletions(-) diff --git a/cmake/linux/package_linux.sh.in b/cmake/linux/package_linux.sh.in index 16866e4d4..907c7a999 100644 --- a/cmake/linux/package_linux.sh.in +++ b/cmake/linux/package_linux.sh.in @@ -126,7 +126,7 @@ EOL chmod +x "${APPDIR}usr/bin/lmms" -# Per https://github.com/probonopd/linuxdeployqt/issues/129 +# Per https://github.com/probonopd/linuxdeployqt/issues/129 unset LD_LIBRARY_PATH # Ensure linuxdeployqt can find shared objects @@ -139,13 +139,16 @@ fi # Move executables so linuxdeployqt can find them ZYNLIB="${APPDIR}usr/lib/lmms/RemoteZynAddSubFx" -VSTLIB="${APPDIR}usr/lib/lmms/RemoteVstPlugin32.exe.so" +VSTLIB32="${APPDIR}usr/lib/lmms/RemoteVstPlugin32.exe.so" +VSTLIB64="${APPDIR}usr/lib/lmms/RemoteVstPlugin64.exe.so" ZYNBIN="${APPDIR}usr/bin/RemoteZynAddSubFx" -VSTBIN="${APPDIR}usr/bin/RemoteVstPlugin32.exe.so" +VSTBIN32="${APPDIR}usr/bin/RemoteVstPlugin32.exe.so" +VSTBIN64="${APPDIR}usr/bin/RemoteVstPlugin64.exe.so" mv "$ZYNLIB" "$ZYNBIN" -mv "$VSTLIB" "$VSTBIN" +mv "$VSTLIB32" "$VSTBIN32" +mv "$VSTLIB64" "$VSTBIN64" # Patch the desktop file sed -i 's/.*Exec=.*/Exec=lmms.real/' "$DESKTOPFILE" @@ -171,7 +174,8 @@ success "Bundled and relinked dependencies" # Link to original location so lmms can find them ln -sr "$ZYNBIN" "$ZYNLIB" -ln -sr "$VSTBIN" "$VSTLIB" +ln -sr "$VSTBIN32" "$VSTLIB32" +ln -sr "$VSTBIN64" "$VSTLIB64" # Remove wine library conflict rm -f "${APPDIR}/usr/lib/libwine.so.1" diff --git a/include/RemotePlugin.h b/include/RemotePlugin.h index 67f0c3807..57f6b379b 100644 --- a/include/RemotePlugin.h +++ b/include/RemotePlugin.h @@ -854,12 +854,11 @@ protected: } + bool m_failed; private: void resizeSharedProcessingMemory(); - bool m_failed; - QProcess m_process; ProcessWatcher m_watcher; diff --git a/plugins/vst_base/RemoteVstPlugin.cpp b/plugins/vst_base/RemoteVstPlugin.cpp index 1a7c43b59..5fa7fe4ea 100644 --- a/plugins/vst_base/RemoteVstPlugin.cpp +++ b/plugins/vst_base/RemoteVstPlugin.cpp @@ -785,11 +785,6 @@ bool RemoteVstPlugin::load( const std::string & _plugin_file ) { DWORD error = GetLastError(); debugMessage( "LoadLibrary failed: " + GetErrorAsString(error) ); - // give VstPlugin class a chance to start 32 bit version of RemoteVstPlugin - if( GetLastError() == ERROR_BAD_EXE_FORMAT ) - { - sendMessage( IdVstBadDllFormat ); - } return false; } diff --git a/plugins/vst_base/VstPlugin.cpp b/plugins/vst_base/VstPlugin.cpp index e629041ca..bdfddf936 100644 --- a/plugins/vst_base/VstPlugin.cpp +++ b/plugins/vst_base/VstPlugin.cpp @@ -26,6 +26,7 @@ #include "communication.h" +#include #include #include #include @@ -64,6 +65,54 @@ #include "templates.h" #include "FileDialog.h" +namespace PE +{ +// Utilities for reading PE file machine type +// See specification at https://msdn.microsoft.com/library/windows/desktop/ms680547(v=vs.85).aspx + +enum MachineType : uint16_t +{ + IMAGE_FILE_MACHINE_UNKNOWN = 0x0, + IMAGE_FILE_MACHINE_AMD64 = 0x8664, + IMAGE_FILE_MACHINE_I386 = 0x14c, +}; + +class FileInfo +{ +public: + FileInfo(QString filePath) + : m_file(filePath) + { + m_file.open(QFile::ReadOnly); + m_map = m_file.map(0, m_file.size()); + if (m_map == nullptr) { + throw std::runtime_error("Cannot map file"); + } + } + ~FileInfo() + { + m_file.unmap(m_map); + } + + MachineType machineType() + { + int32_t peOffset = qFromLittleEndian(* reinterpret_cast(m_map + 0x3C)); + uchar* peSignature = m_map + peOffset; + if (memcmp(peSignature, "PE\0\0", 4)) { + throw std::runtime_error("Invalid PE file"); + } + uchar * coffHeader = peSignature + 4; + uint16_t machineType = qFromLittleEndian(* reinterpret_cast(coffHeader)); + return static_cast(machineType); + } + +private: + QFile m_file; + uchar* m_map; +}; + +} + class vstSubWin : public QMdiSubWindow { public: @@ -93,22 +142,32 @@ VstPlugin::VstPlugin( const QString & _plugin ) : m_embedMethod( gui ? ConfigManager::inst()->vstEmbedMethod() : "headless" ), - m_badDllFormat( false ), m_version( 0 ), m_currentProgram() { setSplittedChannels( true ); -#ifdef LMMS_BUILD_WIN64 - tryLoad( "RemoteVstPlugin64" ); - if( m_badDllFormat ) - { - m_badDllFormat = false; -#endif - tryLoad( "RemoteVstPlugin32" ); -#ifdef LMMS_BUILD_WIN64 + PE::MachineType machineType; + try { + PE::FileInfo peInfo(_plugin); + machineType = peInfo.machineType(); + } catch (std::runtime_error& e) { + qCritical() << "Error while determining PE file's machine type: " << e.what(); + machineType = PE::IMAGE_FILE_MACHINE_UNKNOWN; + } + + switch(machineType) + { + case PE::IMAGE_FILE_MACHINE_AMD64: + tryLoad( "RemoteVstPlugin64" ); + break; + case PE::IMAGE_FILE_MACHINE_I386: + tryLoad( "RemoteVstPlugin32" ); + break; + default: + m_failed = true; + return; } -#endif setTempo( Engine::getSong()->getTempo() ); @@ -390,10 +449,6 @@ bool VstPlugin::processMessage( const message & _m ) { switch( _m.id ) { - case IdVstBadDllFormat: - m_badDllFormat = true; - break; - case IdVstPluginWindowID: m_pluginWindowID = _m.getInt(); break; diff --git a/plugins/vst_base/VstPlugin.h b/plugins/vst_base/VstPlugin.h index 8d2ca01f9..df9117685 100644 --- a/plugins/vst_base/VstPlugin.h +++ b/plugins/vst_base/VstPlugin.h @@ -139,8 +139,6 @@ private: QSize m_pluginGeometry; const QString m_embedMethod; - bool m_badDllFormat; - QString m_name; int m_version; QString m_vendorString; diff --git a/plugins/vst_base/communication.h b/plugins/vst_base/communication.h index cfe136627..756fd7557 100644 --- a/plugins/vst_base/communication.h +++ b/plugins/vst_base/communication.h @@ -70,7 +70,6 @@ enum VstRemoteMessageIDs // remoteVstPlugin -> vstPlugin IdVstFailedLoadingPlugin, - IdVstBadDllFormat, IdVstPluginWindowID, IdVstPluginEditorGeometry, IdVstPluginName, diff --git a/src/core/RemotePlugin.cpp b/src/core/RemotePlugin.cpp index 408b8631f..c00d26be9 100644 --- a/src/core/RemotePlugin.cpp +++ b/src/core/RemotePlugin.cpp @@ -63,7 +63,10 @@ void ProcessWatcher::run() { fprintf( stderr, "remote plugin died! invalidating now.\n" ); + +#ifndef SYNC_WITH_SHM_FIFO m_plugin->invalidate(); +#endif } } @@ -464,9 +467,6 @@ void RemotePlugin::processFinished( int exitCode, { qCritical() << "Remote plugin exit code: " << exitCode; } -#ifndef SYNC_WITH_SHM_FIFO - invalidate(); -#endif } void RemotePlugin::processErrored( QProcess::ProcessError err )