Merge pull request #3786 from LMMS/fix/qt5-vst

Add all the Qt5 Linux VST implementations

.gitmodules

@@ -1,3 +1,6 @@
+[submodule "src/3rdparty/qt5-x11embed"]
+	path = src/3rdparty/qt5-x11embed
+	url = https://github.com/Lukas-W/qt5-x11embed.git
 [submodule "src/3rdparty/rpmalloc/rpmalloc"]
 	path = src/3rdparty/rpmalloc/rpmalloc
 	url = https://github.com/rampantpixels/rpmalloc.git

.travis.yml

@@ -12,6 +12,7 @@ matrix:
         - env: TARGET_OS=win64
         - os: osx
           osx_image: xcode8.2
+        - env: QT5=
         - env: QT5=True
         - env: QT5=True TARGET_OS=win32
         - env: QT5=True TARGET_OS=win64

.travis/linux..install.sh

@@ -2,11 +2,13 @@
 
 PACKAGES="cmake libsndfile-dev fftw3-dev libvorbis-dev libogg-dev libmp3lame-dev
 	libasound2-dev libjack-dev libsdl-dev libsamplerate0-dev libstk0-dev stk
-	libfluidsynth-dev portaudio19-dev wine-dev g++-multilib libfltk1.3-dev
+	libfluidsynth-dev portaudio19-dev g++-multilib libfltk1.3-dev
 	libgig-dev libsoundio-dev"
 
+VST_PACKAGES="wine-dev libqt5x11extras5-dev qtbase5-private-dev libxcb-util0-dev libxcb-keysyms1-dev"
+
 # Help with unmet dependencies
-PACKAGES="$PACKAGES libjack0"
+PACKAGES="$PACKAGES $VST_PACKAGES libjack0"
 
 if [ $QT5 ]; then
 	PACKAGES="$PACKAGES qt59base qt59translations qt59tools"

CMakeLists.txt

@@ -156,6 +156,11 @@ IF(WANT_QT5)
 		Qt5::Xml
 	)
 
+	IF(LMMS_BUILD_LINUX)
+		FIND_PACKAGE(Qt5X11Extras REQUIRED)
+		LIST(APPEND QT_LIBRARIES Qt5::X11Extras)
+	ENDIF()
+
 	# Resolve Qt5::qmake to full path for use in packaging scripts
 	GET_TARGET_PROPERTY(QT_QMAKE_EXECUTABLE "${Qt5Core_QMAKE_EXECUTABLE}" IMPORTED_LOCATION)
 

include/ConfigManager.h

@@ -32,9 +32,9 @@
 #include <QtCore/QPair>
 #include <QtCore/QStringList>
 #include <QtCore/QVector>
+#include <QtCore/QObject>
 
 #include "export.h"
-#include "MemoryManager.h"
 
 class LmmsCore;
 
@@ -51,9 +51,9 @@ const QString TRACK_ICON_PATH = "track_icons/";
 const QString LOCALE_PATH = "locale/";
 
 
-class EXPORT ConfigManager
+class EXPORT ConfigManager : public QObject
 {
-	MM_OPERATORS
+	Q_OBJECT
 public:
 	static inline ConfigManager * inst()
 	{
@@ -210,6 +210,9 @@ public:
 		return m_recentlyOpenedProjects;
 	}
 
+	static QStringList availabeVstEmbedMethods();
+	QString vstEmbedMethod() const;
+
 	// returns true if the working dir (e.g. ~/lmms) exists on disk
 	bool hasWorkingDir() const;
 
@@ -242,6 +245,8 @@ public:
 	// creates the working directory & subdirectories on disk.
 	void createWorkingDir();
 
+signals:
+	void valueChanged( QString cls, QString attribute, QString value );
 
 private:
 	static ConfigManager * s_instanceOfMe;

include/RemotePlugin.h

@@ -414,6 +414,7 @@ private:
 enum RemoteMessageIDs
 {
 	IdUndefined,
+	IdHostInfoGotten,
 	IdInitDone,
 	IdQuit,
 	IdSampleRateInformation,
@@ -427,6 +428,8 @@ enum RemoteMessageIDs
 	IdChangeInputOutputCount,
 	IdShowUI,
 	IdHideUI,
+	IdToggleUI,
+	IdIsUIVisible,
 	IdSaveSettingsToString,
 	IdSaveSettingsToFile,
 	IdLoadSettingsFromString,
@@ -781,7 +784,13 @@ public:
 #endif
 	}
 
-	bool init( const QString &pluginExecutable, bool waitForInitDoneMsg );
+	bool init( const QString &pluginExecutable, bool waitForInitDoneMsg, QStringList extraArgs = {} );
+
+	inline void waitForHostInfoGotten()
+	{
+		m_failed = waitForMessage( IdHostInfoGotten ).id
+							!= IdHostInfoGotten;
+	}
 
 	inline void waitForInitDone( bool _busyWaiting = true )
 	{
@@ -801,18 +810,21 @@ public:
 		unlock();
 	}
 
-	void showUI()
+
+	virtual void toggleUI()
 	{
 		lock();
-		sendMessage( IdShowUI );
+		sendMessage( IdToggleUI );
 		unlock();
 	}
 
-	void hideUI()
+	int isUIVisible()
 	{
 		lock();
-		sendMessage( IdHideUI );
+		sendMessage( IdIsUIVisible );
 		unlock();
+		message m = waitForMessage( IdIsUIVisible );
+		return m.id != IdIsUIVisible ? -1 : m.getInt() ? 1 : 0;
 	}
 
 	inline bool failed() const
@@ -830,6 +842,9 @@ public:
 		m_commMutex.unlock();
 	}
 
+public slots:
+	virtual void showUI();
+	virtual void hideUI();
 
 protected:
 	inline void setSplittedChannels( bool _on )
@@ -1206,6 +1221,7 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 		m_vstSyncData = (VstSyncData *) m_shmQtID.data();
 		m_bufferSize = m_vstSyncData->m_bufferSize;
 		m_sampleRate = m_vstSyncData->m_sampleRate;
+		sendMessage( IdHostInfoGotten );
 		return;
 	}
 #else
@@ -1233,6 +1249,7 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 			{
 				m_bufferSize = m_vstSyncData->m_bufferSize;
 				m_sampleRate = m_vstSyncData->m_sampleRate;
+				sendMessage( IdHostInfoGotten );
 
 				// detach segment
 				if( shmdt(m_vstSyncData) == -1 )
@@ -1248,6 +1265,12 @@ RemotePluginClient::RemotePluginClient( const char * socketPath ) :
 	// if attaching shared memory fails
 	sendMessage( IdSampleRateInformation );
 	sendMessage( IdBufferSizeInformation );
+	if( waitForMessage( IdBufferSizeInformation ).id
+						!= IdBufferSizeInformation )
+	{
+		fprintf( stderr, "Could not get buffer size information\n" );
+	}
+	sendMessage( IdHostInfoGotten );
 }
 
 

include/SetupDialog.h

@@ -204,7 +204,8 @@ private:
 	MswMap m_midiIfaceSetupWidgets;
 	trMap m_midiIfaceNames;
 
-
+	QComboBox* m_vstEmbedComboBox;
+	QString m_vstEmbedMethod;
 } ;
 
 

plugins/VstEffect/VstEffect.cpp

@@ -55,7 +55,6 @@ Plugin::Descriptor PLUGIN_EXPORT vsteffect_plugin_descriptor =
 VstEffect::VstEffect( Model * _parent,
 			const Descriptor::SubPluginFeatures::Key * _key ) :
 	Effect( &vsteffect_plugin_descriptor, _parent, _key ),
-	m_plugin( NULL ),
 	m_pluginMutex(),
 	m_key( *_key ),
 	m_vstControls( this )
@@ -73,7 +72,6 @@ VstEffect::VstEffect( Model * _parent,
 
 VstEffect::~VstEffect()
 {
-	closePlugin();
 }
 
 
@@ -128,22 +126,20 @@ void VstEffect::openPlugin( const QString & _plugin )
 		VstPlugin::tr( "Loading plugin" ),
 		VstPlugin::tr( "Please wait while loading VST plugin..." ),
 			PLUGIN_NAME::getIconPixmap( "logo", 24, 24 ), 0 );
-	m_pluginMutex.lock();
-	m_plugin = new VstPlugin( _plugin );
+
+	QMutexLocker ml( &m_pluginMutex ); Q_UNUSED( ml );
+	m_plugin = QSharedPointer<VstPlugin>(new VstPlugin( _plugin ));
 	if( m_plugin->failed() )
 	{
-		m_pluginMutex.unlock();
-		closePlugin();
+		m_plugin.clear();
 		delete tf;
 		collectErrorForUI( VstPlugin::tr( "The VST plugin %1 could not be loaded." ).arg( _plugin ) );
 		return;
 	}
 
-	VstPlugin::connect( Engine::getSong(), SIGNAL( tempoChanged( bpm_t ) ), m_plugin, SLOT( setTempo( bpm_t ) ) );
+	VstPlugin::connect( Engine::getSong(), SIGNAL( tempoChanged( bpm_t ) ), m_plugin.data(), SLOT( setTempo( bpm_t ) ) );
 	m_plugin->setTempo( Engine::getSong()->getTempo() );
 
-	m_pluginMutex.unlock();
-
 	delete tf;
 
 	m_key.attributes["file"] = _plugin;
@@ -151,21 +147,6 @@ void VstEffect::openPlugin( const QString & _plugin )
 
 
 
-void VstEffect::closePlugin()
-{
-	m_pluginMutex.lock();
-	if( m_plugin && m_plugin->pluginWidget() != NULL )
-	{
-		delete m_plugin->pluginWidget();
-	}
-	delete m_plugin;
-	m_plugin = NULL;
-	m_pluginMutex.unlock();
-}
-
-
-
-
 
 extern "C"
 {

plugins/VstEffect/VstEffect.h

@@ -25,13 +25,14 @@
 #ifndef _VST_EFFECT_H
 #define _VST_EFFECT_H
 
-#include <QMutex>
+#include <QtCore/QMutex>
+#include <QtCore/QSharedPointer>
 
 #include "Effect.h"
-#include "VstPlugin.h"
 #include "VstEffectControlDialog.h"
 #include "VstEffectControls.h"
 
+class VstPlugin;
 
 class VstEffect : public Effect
 {
@@ -58,7 +59,7 @@ private:
 	void openPlugin( const QString & _plugin );
 	void closePlugin();
 
-	VstPlugin * m_plugin;
+	QSharedPointer<VstPlugin> m_plugin;
 	QMutex m_pluginMutex;
 	EffectKey m_key;
 

plugins/VstEffect/VstEffectControlDialog.cpp

@@ -30,6 +30,7 @@
 #include "VstEffectControlDialog.h"
 #include "VstEffect.h"
 
+#include "ConfigManager.h"
 #include "PixmapButton.h"
 #include "embed.h"
 #include "ToolTip.h"
@@ -52,32 +53,43 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 	l->setVerticalSpacing( 2 );
 	l->setHorizontalSpacing( 2 );
 
+	bool embed_vst = false;
+
 	if( _ctl != NULL && _ctl->m_effect != NULL &&
 					_ctl->m_effect->m_plugin != NULL )
 	{
 		m_plugin = _ctl->m_effect->m_plugin;
-		m_plugin->showEditor( NULL, true );
-		m_pluginWidget = m_plugin->pluginWidget();
+		embed_vst = m_plugin->embedMethod() != "none";
+
+		if (embed_vst) {
+			m_plugin->createUI( nullptr, true );
+			m_pluginWidget = m_plugin->pluginWidget( false );
 
 #ifdef LMMS_BUILD_WIN32
-
-		if( !m_pluginWidget )
-		{
-			m_pluginWidget = m_plugin->pluginWidget( false );
-		}
+			if( !m_pluginWidget )
+			{
+				m_pluginWidget = m_plugin->pluginWidget( false );
+			}
 #endif
+
+		}
 	}
 
-	if( m_pluginWidget )
+	if ( m_plugin && (!embed_vst || m_pluginWidget) )
 	{
-		setWindowTitle( m_pluginWidget->windowTitle() );
-		setMinimumWidth( 250 );
+		setWindowTitle( m_plugin->name() );
 
 		QPushButton * btn = new QPushButton( tr( "Show/hide" ) );
-		btn->setCheckable( true );
-		connect( btn, SIGNAL( toggled( bool ) ),
-					m_pluginWidget, SLOT( setVisible( bool ) ) );
-		emit btn->click();
+
+		if (embed_vst) {
+			btn->setCheckable( true );
+			btn->setChecked( true );
+			connect( btn, SIGNAL( toggled( bool ) ),
+						SLOT( togglePluginUI( bool ) ) );
+		} else {
+			connect( btn, SIGNAL( clicked( bool ) ),
+						SLOT( togglePluginUI( bool ) ) );
+		}
 
 		btn->setMinimumWidth( 78 );
 		btn->setMaximumWidth( 78 );
@@ -206,8 +218,13 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 		m_savePresetButton->setMinimumHeight( 21 );
 		m_savePresetButton->setMaximumHeight( 21 );
 
-		int newSize = m_pluginWidget->width() + 20;
-		newSize = (newSize < 250) ? 250 : newSize;
+		int newSize = 0;
+
+		if (embed_vst) {
+			newSize = m_pluginWidget->width() + 20;
+		}
+		newSize = std::max(newSize, 250);
+
 		QWidget* resize = new QWidget(this);
 		resize->resize( newSize, 10 );
 		QWidget* space0 = new QWidget(this);
@@ -219,7 +236,9 @@ VstEffectControlDialog::VstEffectControlDialog( VstEffectControls * _ctl ) :
 		l->addItem( new QSpacerItem( newSize - 20, 30, QSizePolicy::Fixed,
 						QSizePolicy::Fixed ), 1, 0 );
 		l->addWidget( resize, 2, 0, 1, 1, Qt::AlignCenter );
-		l->addWidget( m_pluginWidget, 3, 0, 1, 1, Qt::AlignCenter );
+		if (embed_vst) {
+			l->addWidget( m_pluginWidget, 3, 0, 1, 1, Qt::AlignCenter );
+		}
 		l->setRowStretch( 5, 1 );
 		l->setColumnStretch( 1, 1 );
 
@@ -264,3 +283,15 @@ VstEffectControlDialog::~VstEffectControlDialog()
 	//delete m_pluginWidget;
 }
 
+
+
+
+void VstEffectControlDialog::togglePluginUI( bool checked )
+{
+	if( !m_plugin ) {
+		return;
+	}
+
+	m_plugin->toggleUI();
+}
+

plugins/VstEffect/VstEffectControlDialog.h

@@ -31,6 +31,7 @@
 #include <QObject>
 #include <QPainter>
 #include <QLabel>
+#include <QSharedPointer>
 
 
 class VstEffectControls;
@@ -59,9 +60,12 @@ private:
 	PixmapButton * m_managePluginButton;
 	PixmapButton * m_savePresetButton;
 
-	VstPlugin * m_plugin;
+	QSharedPointer<VstPlugin> m_plugin;
 
 	QLabel * tbLabel;
+
+private slots:
+	void togglePluginUI( bool checked );
 } ;
 
 #endif

plugins/vestige/vestige.cpp

@@ -32,6 +32,7 @@
 #include <QMenu>
 #include <QDomElement>
 
+#include "ConfigManager.h"
 #include "BufferManager.h"
 #include "Engine.h"
 #include "gui_templates.h"
@@ -89,6 +90,10 @@ vestigeInstrument::vestigeInstrument( InstrumentTrack * _instrument_track ) :
 	// now we need a play-handle which cares for calling play()
 	InstrumentPlayHandle * iph = new InstrumentPlayHandle( this, _instrument_track );
 	Engine::mixer()->addPlayHandle( iph );
+
+	connect( ConfigManager::inst(), SIGNAL( valueChanged(QString,QString,QString) ),
+			 this, SLOT( handleConfigChange(QString, QString, QString) ),
+			 Qt::QueuedConnection );
 }
 
 
@@ -170,6 +175,22 @@ void vestigeInstrument::setParameter( void )
 	}
 }
 
+void vestigeInstrument::handleConfigChange(QString cls, QString attr, QString value)
+{
+    Q_UNUSED(cls); Q_UNUSED(attr); Q_UNUSED(value);
+    // Disabled for consistency with VST effects that don't implement this. (#3786)
+    // if ( cls == "ui" && attr == "vstembedmethod" )
+    // {
+    // 	reloadPlugin();
+    // }
+}
+
+void vestigeInstrument::reloadPlugin()
+{
+	closePlugin();
+	loadFile( m_pluginDLL );
+}
+
 
 
 
@@ -262,7 +283,7 @@ void vestigeInstrument::loadFile( const QString & _file )
 		return;
 	}
 
-	m_plugin->showEditor( NULL, false );
+	m_plugin->showUI();
 
 	if( set_ch_name )
 	{
@@ -366,10 +387,6 @@ void vestigeInstrument::closePlugin( void )
 	}
 
 	m_pluginMutex.lock();
-	if( m_plugin )
-	{
-		delete m_plugin->pluginWidget();
-	}
 	delete m_plugin;
 	m_plugin = NULL;
 	m_pluginMutex.unlock();
@@ -739,19 +756,7 @@ void VestigeInstrumentView::toggleGUI( void )
 	{
 		return;
 	}
-	QWidget * w = m_vi->m_plugin->pluginWidget();
-	if( w == NULL )
-	{
-		return;
-	}
-	if( w->isHidden() )
-	{
-		w->show();
-	}
-	else
-	{
-		w->hide();
-	}
+	m_vi->m_plugin->toggleUI();
 }
 
 

plugins/vestige/vestige.h

@@ -74,6 +74,8 @@ public:
 
 protected slots:
 	void setParameter( void );
+	void handleConfigChange( QString cls, QString attr, QString value );
+	void reloadPlugin();
 
 private:
 	void closePlugin( void );

plugins/vst_base/CMakeLists.txt

@@ -29,6 +29,10 @@ SET(REMOTE_VST_PLUGIN_FILEPATH "RemoteVstPlugin" CACHE STRING "Relative file pat
 ADD_DEFINITIONS(-DREMOTE_VST_PLUGIN_FILEPATH="${REMOTE_VST_PLUGIN_FILEPATH}")
 BUILD_PLUGIN(vstbase vst_base.cpp VstPlugin.cpp VstPlugin.h communication.h MOCFILES VstPlugin.h)
 
+IF(LMMS_BUILD_LINUX AND WANT_QT5)
+	TARGET_LINK_LIBRARIES(vstbase qx11embedcontainer)
+ENDIF()
+
 IF(LMMS_BUILD_LINUX AND NOT WANT_VST_NOWINE)
 
 IF(LMMS_HOST_X86_64)
@@ -43,21 +47,44 @@ SET(WINE_CXX_FLAGS "" CACHE STRING "Extra flags passed to wineg++")
 STRING(REPLACE "include/wine" "include" WINE_INCLUDE_BASE_DIR ${WINE_INCLUDE_DIR})
 STRING(REPLACE "lib/libwine.so" "lib" WINE_LIBRARY_DIR ${WINE_LIBRARY})
 STRING(REPLACE " " ";" WINE_BUILD_FLAGS ${CMAKE_CXX_FLAGS} " " ${CMAKE_EXE_LINKER_FLAGS} " " ${WINE_CXX_FLAGS})
+
+SET(WINE_CXX_ARGS
+	-I${CMAKE_BINARY_DIR}
+	-I${CMAKE_SOURCE_DIR}/include
+	-I${WINE_INCLUDE_BASE_DIR}
+	-L${WINE_LIBRARY_DIR}
+	${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp
+	-std=c++0x
+	-mwindows -lpthread ${EXTRA_FLAGS} -fno-omit-frame-pointer
+	${WINE_BUILD_FLAGS}
+	-o ../RemoteVstPlugin
+)
+
+# winegcc fails if winebuild is not in path
+GET_FILENAME_COMPONENT(WINE_BINDIR ${WINE_CXX} PATH)
+FIND_PROGRAM(WINEBUILD winebuild NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+IF(NOT WINEBUILD)
+	IF(CMAKE_VERSION VERSION_LESS 3.1)
+		MESSAGE(WARNING "winebuild is not in PATH. Building RemoteVstPlugin may fail.")
+	ELSE()
+		SET(WINE_CXX_ARGS -E env PATH=$ENV{PATH}:${WINE_BINDIR} ${WINE_CXX} ${WINE_CXX_ARGS})
+		SET(WINE_CXX "${CMAKE_COMMAND}")
+	ENDIF()
+ENDIF()
+
+set(ENV{PATH} "$ENV{PATH}:${WINE_BINDIR}")
+
 ADD_CUSTOM_COMMAND(
 	SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp"
+	COMMAND export
+		ARGS "PATH=$PATH:${WINE_BINDIR}"
 	COMMAND ${WINE_CXX}
-	ARGS -I${CMAKE_BINARY_DIR}
-		-I${CMAKE_SOURCE_DIR}/include
-		-I${WINE_INCLUDE_BASE_DIR}
-		-L${WINE_LIBRARY_DIR}
-		${CMAKE_CURRENT_SOURCE_DIR}/RemoteVstPlugin.cpp
-		-ansi -mwindows -lpthread ${EXTRA_FLAGS} -fno-omit-frame-pointer
-		${WINE_BUILD_FLAGS}
-		-o ../RemoteVstPlugin
+	ARGS ${WINE_CXX_ARGS}
 	# Ensure correct file extension
 	COMMAND sh -c "mv ../RemoteVstPlugin.exe ../RemoteVstPlugin || true"
 	TARGET vstbase
 	OUTPUTS ../RemoteVstPlugin
+	VERBATIM
 )
 
 SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ../RemoteVstPlugin.exe.so)

plugins/vst_base/RemoteVstPlugin.cpp

@@ -63,17 +63,10 @@
 #define USE_WS_PREFIX
 #include <windows.h>
 
-#if defined(LMMS_BUILD_WIN32) || defined(LMMS_BUILD_WIN64)
-#include "basename.c"
-#else
-#include <libgen.h>
-#endif
-
-
 #include <vector>
 #include <queue>
 #include <string>
-
+#include <iostream>
 
 #include <aeffectx.h>
 
@@ -105,14 +98,18 @@ struct ERect
 #ifndef USE_QT_SHMEM
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <sys/types.h>
 #include <sys/ipc.h>
 #include <sys/shm.h>
 #endif
 
+using namespace std;
+
 static VstHostLanguages hlang = LanguageEnglish;
 
+static bool EMBED = false;
+static bool EMBED_X11 = false;
+static bool EMBED_WIN32 = false;
 
 class RemoteVstPlugin;
 
@@ -136,6 +133,7 @@ public:
 
 	void init( const std::string & _plugin_file );
 	void initEditor();
+	void destroyEditor();
 
 	virtual void process( const sampleFrame * _in, sampleFrame * _out );
 
@@ -356,6 +354,7 @@ private:
 	int m_windowHeight;
 
 	bool m_initialized;
+	bool m_registeredWindowClass;
 
 	pthread_mutex_t m_pluginLock;
 	bool m_processing;
@@ -401,7 +400,6 @@ RemoteVstPlugin::RemoteVstPlugin( key_t _shm_in, key_t _shm_out ) :
 RemoteVstPlugin::RemoteVstPlugin( const char * socketPath ) :
 	RemotePluginClient( socketPath ),
 #endif
-	m_shortName( "" ),
 	m_libInst( NULL ),
 	m_plugin( NULL ),
 	m_window( NULL ),
@@ -409,6 +407,7 @@ RemoteVstPlugin::RemoteVstPlugin( const char * socketPath ) :
 	m_windowWidth( 0 ),
 	m_windowHeight( 0 ),
 	m_initialized( false ),
+	m_registeredWindowClass( false ),
 	m_pluginLock(),
 	m_processing( false ),
 	m_messageList(),
@@ -424,7 +423,6 @@ RemoteVstPlugin::RemoteVstPlugin( const char * socketPath ) :
 	m_in( NULL ),
 	m_shmID( -1 ),
 	m_vstSyncData( NULL )
-
 {
 	pthread_mutex_init( &m_pluginLock, NULL );
 	pthread_mutex_init( &m_shmLock, NULL );
@@ -491,14 +489,7 @@ RemoteVstPlugin::RemoteVstPlugin( const char * socketPath ) :
 
 RemoteVstPlugin::~RemoteVstPlugin()
 {
-	if( m_window != NULL )
-	{
-		pluginDispatch( effEditClose );
-#ifdef LMMS_BUILD_LINUX
-		CloseWindow( m_window );
-#endif
-		m_window = NULL;
-	}
+	destroyEditor();
 	pluginDispatch( effMainsChanged, 0, 0 );
 	pluginDispatch( effClose );
 #ifndef USE_QT_SHMEM
@@ -535,22 +526,48 @@ RemoteVstPlugin::~RemoteVstPlugin()
 
 bool RemoteVstPlugin::processMessage( const message & _m )
 {
+	if (! EMBED)
+	{
+		switch( _m.id )
+		{
+		case IdShowUI:
+			initEditor();
+			return true;
+
+		case IdHideUI:
+			destroyEditor();
+			return true;
+
+		case IdToggleUI:
+			if( m_window )
+			{
+				destroyEditor();
+			}
+			else
+			{
+				initEditor();
+			}
+			return true;
+
+		case IdIsUIVisible:
+			sendMessage( message( IdIsUIVisible )
+						 .addInt( m_window ? 1 : 0 ) );
+			return true;
+		}
+	}
+	else if (EMBED && _m.id == IdShowUI)
+	{
+		ShowWindow( m_window, SW_SHOWNORMAL );
+		UpdateWindow( m_window );
+		return true;
+	}
+
 	switch( _m.id )
 	{
 		case IdVstLoadPlugin:
 			init( _m.getString() );
 			break;
 
-#ifdef LMMS_BUILD_WIN32
-		case IdVstPluginWindowInformation:
-		{
-			HWND top = FindWindowEx( NULL, NULL, NULL,
-						_m.getString().c_str() );
-			m_window = FindWindowEx( top, NULL, NULL, NULL );
-			break;
-		}
-#endif
-
 		case IdVstSetTempo:
 			setBPM( _m.getInt() );
 			break;
@@ -688,9 +705,20 @@ void RemoteVstPlugin::init( const std::string & _plugin_file )
 
 
 
+static void close_check( int fd )
+{
+	if( close( fd ) )
+	{
+		perror( "close" );
+	}
+}
+
+
+
+
 void RemoteVstPlugin::initEditor()
 {
-	if( !( m_plugin->flags & effFlagsHasEditor ) )
+	if( m_window || !( m_plugin->flags & effFlagsHasEditor ) )
 	{
 		return;
 	}
@@ -704,38 +732,37 @@ void RemoteVstPlugin::initEditor()
 	}
 
 
-	WNDCLASS wc;
-	wc.style = CS_HREDRAW | CS_VREDRAW;
-	wc.lpfnWndProc = DefWindowProc;
-	wc.cbClsExtra = 0;
-	wc.cbWndExtra = 0;
-	wc.hInstance = hInst;
-	wc.hIcon = LoadIcon( NULL, IDI_APPLICATION );
-	wc.hCursor = LoadCursor( NULL, IDC_ARROW );
-	wc.hbrBackground = (HBRUSH) GetStockObject( BLACK_BRUSH );
-	wc.lpszMenuName = NULL;
-	wc.lpszClassName = "LVSL";
-
-	if( !RegisterClass( &wc ) )
+	if( !m_registeredWindowClass )
 	{
-		return;
+		WNDCLASS wc;
+		wc.style = CS_HREDRAW | CS_VREDRAW;
+		wc.lpfnWndProc = DefWindowProc;
+		wc.cbClsExtra = 0;
+		wc.cbWndExtra = 0;
+		wc.hInstance = hInst;
+		wc.hIcon = LoadIcon( NULL, IDI_APPLICATION );
+		wc.hCursor = LoadCursor( NULL, IDC_ARROW );
+		wc.hbrBackground = NULL;
+		wc.lpszMenuName = NULL;
+		wc.lpszClassName = "LVSL";
+
+		if( !RegisterClass( &wc ) )
+		{
+			return;
+		}
+		m_registeredWindowClass = true;
 	}
 
-#ifdef LMMS_BUILD_LINUX
-	//m_window = CreateWindowEx( 0, "LVSL", m_shortName.c_str(),
-	//		       ( WS_OVERLAPPEDWINDOW | WS_THICKFRAME ) & ~WS_MAXIMIZEBOX,
-	//		       0, 0, 10, 10, NULL, NULL, hInst, NULL );
+	DWORD dwStyle;
+	if (EMBED) {
+		dwStyle = WS_POPUP | WS_SYSMENU | WS_BORDER;
+	} else {
+		dwStyle = WS_OVERLAPPEDWINDOW & ~WS_MAXIMIZEBOX;
+	}
 
-	m_window = CreateWindowEx( 0 , "LVSL", m_shortName.c_str(),
-	   WS_POPUP | WS_SYSMENU | WS_BORDER , 0, 0, 10, 10, NULL, NULL, hInst, NULL);
-#else
-	m_windowID = 1;	// arbitrary value on win32 to signal
-			// vstPlugin-class that we have an editor
-
-	m_window = CreateWindowEx( 0, "LVSL", m_shortName.c_str(),
-					WS_CHILD, 0, 0, 10, 10,
-					m_window, NULL, hInst, NULL );
-#endif
+	m_window = CreateWindowEx( 0, "LVSL", pluginName(),
+		dwStyle,
+		0, 0, 10, 10, NULL, NULL, hInst, NULL );
 	if( m_window == NULL )
 	{
 		debugMessage( "initEditor(): cannot create editor window\n" );
@@ -756,17 +783,37 @@ void RemoteVstPlugin::initEditor()
 						SWP_NOMOVE | SWP_NOZORDER );
 	pluginDispatch( effEditTop );
 
-	ShowWindow( m_window, SW_SHOWNORMAL );
-	UpdateWindow( m_window );
+	if (! EMBED) {
+		ShowWindow( m_window, SW_SHOWNORMAL );
+	}
 
 #ifdef LMMS_BUILD_LINUX
 	m_windowID = (intptr_t) GetProp( m_window, "__wine_x11_whole_window" );
+#else
+	// 64-bit versions of Windows use 32-bit handles for interoperability
+	m_windowID = (intptr_t) m_window;
 #endif
 }
 
 
 
 
+void RemoteVstPlugin::destroyEditor()
+{
+	if( m_window == NULL )
+	{
+		return;
+	}
+
+	pluginDispatch( effEditClose );
+	// Destroying the window takes some time in Wine 1.8.5
+	DestroyWindow( m_window );
+	m_window = NULL;
+}
+
+
+
+
 bool RemoteVstPlugin::load( const std::string & _plugin_file )
 {
 	if( ( m_libInst = LoadLibrary( _plugin_file.c_str() ) ) == NULL )
@@ -779,10 +826,6 @@ bool RemoteVstPlugin::load( const std::string & _plugin_file )
 		return false;
 	}
 
-	char * tmp = strdup( _plugin_file.c_str() );
-	m_shortName = basename( tmp );
-	free( tmp );
-
 	typedef AEffect * ( __stdcall * mainEntryPointer )
 						( audioMasterCallback );
 	mainEntryPointer mainEntry = (mainEntryPointer)
@@ -1067,7 +1110,7 @@ void RemoteVstPlugin::saveChunkToFile( const std::string & _file )
 				fprintf( stderr,
 					"Error saving chunk to file.\n" );
 			}
-			close( fd );
+			close_check( fd );
 		}
 	}
 }
@@ -1392,7 +1435,7 @@ void RemoteVstPlugin::loadChunkFromFile( const std::string & _file, int _len )
 	{
 		fprintf( stderr, "Error loading chunk from file.\n" );
 	}
-	close( fd );
+	close_check( fd );
 
 	pluginDispatch( effSetChunk, 0, _len, chunk );
 
@@ -1970,6 +2013,12 @@ LRESULT CALLBACK RemoteVstPlugin::messageWndProc( HWND hwnd, UINT uMsg,
 				break;
 		}
 	}
+	else if( uMsg == WM_SYSCOMMAND && wParam == SC_CLOSE )
+	{
+		__plugin->destroyEditor();
+		return 0;
+	}
+
 	return DefWindowProc( hwnd, uMsg, wParam, lParam );
 }
 
@@ -1979,9 +2028,9 @@ LRESULT CALLBACK RemoteVstPlugin::messageWndProc( HWND hwnd, UINT uMsg,
 int main( int _argc, char * * _argv )
 {
 #ifdef SYNC_WITH_SHM_FIFO
-	if( _argc < 3 )
+	if( _argc < 4 )
 #else
-	if( _argc < 2 )
+	if( _argc < 3 )
 #endif
 	{
 		fprintf( stderr, "not enough arguments\n" );
@@ -2013,6 +2062,41 @@ int main( int _argc, char * * _argv )
 	}
 #endif
 
+	{
+	#ifdef SYNC_WITH_SHM_FIFO
+		int embedMethodIndex = 3;
+	#else
+		int embedMethodIndex = 2;
+	#endif
+		std::string embedMethod = _argv[embedMethodIndex];
+
+		if ( embedMethod == "none" )
+		{
+			cerr << "Starting detached." << endl;
+			EMBED = EMBED_X11 = EMBED_WIN32 = false;
+		}
+		else if ( embedMethod == "win32" )
+		{
+			cerr << "Starting using Win32-native embedding." << endl;
+			EMBED = EMBED_WIN32 = true; EMBED_X11= false;
+		}
+		else if ( embedMethod == "qt" )
+		{
+			cerr << "Starting using Qt-native embedding." << endl;
+			EMBED = true; EMBED_X11 = EMBED_WIN32 = false;
+		}
+		else if ( embedMethod == "xembed" )
+		{
+			cerr << "Starting using X11Embed protocol." << endl;
+			EMBED = EMBED_X11 = true; EMBED_WIN32 = false;
+		}
+		else
+		{
+			cerr << "Unknown embed method " << embedMethod << ". Starting detached instead." << endl;
+			EMBED = EMBED_X11 = EMBED_WIN32 = false;
+		}
+	}
+
 	// constructor automatically will process messages until it receives
 	// a IdVstLoadPlugin message and processes it
 #ifdef SYNC_WITH_SHM_FIFO

plugins/vst_base/VstPlugin.cpp

@@ -2,7 +2,7 @@
  * VstPlugin.cpp - implementation of VstPlugin class
  *
  * Copyright (c) 2005-2014 Tobias Doerffel <tobydox/at/users.sourceforge.net>
- * 
+ *
  * This file is part of LMMS - https://lmms.io
  *
  * This program is free software; you can redistribute it and/or
@@ -24,6 +24,7 @@
 
 #include "VstPlugin.h"
 
+#include <QtCore/QDebug>
 #include <QDir>
 #include <QFileInfo>
 #include <QLocale>
@@ -31,18 +32,26 @@
 #include <QCloseEvent>
 #include <QMdiArea>
 #include <QMdiSubWindow>
+
 #ifdef LMMS_BUILD_LINUX
-#if QT_VERSION < 0x050000
-#include <QX11EmbedContainer>
-#include <QX11Info>
+#	if QT_VERSION < 0x050000
+#		include <QX11EmbedContainer>
+#		include <QX11Info>
+#	else
+#		include "X11EmbedContainer.h"
+#		include <QWindow>
+#	endif
 #endif
-#else
-#include <QLayout>
+
+#if QT_VERSION >= 0x050000
+#	include <QWindow>
 #endif
+
 #include <QDomDocument>
 
 #ifdef LMMS_BUILD_WIN32
-#include <windows.h>
+#	include <windows.h>
+#	include <QLayout>
 #endif
 
 #include "ConfigManager.h"
@@ -52,8 +61,6 @@
 #include "Song.h"
 #include "templates.h"
 #include "FileDialog.h"
-#include <QLayout>
-
 
 class vstSubWin : public QMdiSubWindow
 {
@@ -78,24 +85,13 @@ public:
 } ;
 
 
-
-
 VstPlugin::VstPlugin( const QString & _plugin ) :
-	RemotePlugin(),
-	JournallingObject(),
 	m_plugin( _plugin ),
-	m_pluginWidget( NULL ),
 	m_pluginWindowID( 0 ),
+	m_embedMethod( ConfigManager::inst()->vstEmbedMethod() ),
 	m_badDllFormat( false ),
-	m_name(),
 	m_version( 0 ),
-	m_vendorString(),
-	m_productString(),
-	m_currentProgramName(),
-	m_allProgramNames(),
-	p_name(),
-	m_currentProgram(),
-	m_idleTimer()
+	m_currentProgram()
 {
 	setSplittedChannels( true );
 
@@ -126,6 +122,8 @@ VstPlugin::VstPlugin( const QString & _plugin ) :
 
 VstPlugin::~VstPlugin()
 {
+	delete m_pluginSubWindow;
+	delete m_pluginWidget;
 }
 
 
@@ -133,29 +131,15 @@ VstPlugin::~VstPlugin()
 
 void VstPlugin::tryLoad( const QString &remoteVstPluginExecutable )
 {
-	init( remoteVstPluginExecutable, false );
+	init( remoteVstPluginExecutable, false, {m_embedMethod} );
+
+	waitForHostInfoGotten();
+	if( failed() )
+	{
+		return;
+	}
 
 	lock();
-#ifdef LMMS_BUILD_WIN32
-	QWidget * helper = new QWidget;
-	QHBoxLayout * l = new QHBoxLayout( helper );
-	QWidget * target = new QWidget( helper );
-	l->setSpacing( 0 );
-	l->setMargin( 0 );
-	l->addWidget( target );
-
-	static int k = 0;
-	const QString t = QString( "vst%1%2" ).arg( GetCurrentProcessId()<<10 ).
-								arg( ++k );
-	helper->setWindowTitle( t );
-
-	// we've to call that for making sure, Qt created the windows
-	(void) helper->winId();
-	(void) target->winId();
-
-	sendMessage( message( IdVstPluginWindowInformation ).
-					addString( QSTR_TO_STDSTR( t ) ) );
-#endif
 
 	VstHostLanguages hlang = LanguageEnglish;
 	switch( QLocale::system().language() )
@@ -183,95 +167,6 @@ void VstPlugin::tryLoad( const QString &remoteVstPluginExecutable )
 	waitForInitDone();
 
 	unlock();
-
-#ifdef LMMS_BUILD_WIN32
-	if( !failed() && m_pluginWindowID )
-	{
-		target->setFixedSize( m_pluginGeometry );
-		vstSubWin * sw = new vstSubWin(
-					gui->mainWindow()->workspace() );
-		sw->setWidget( helper );
-		helper->setWindowTitle( name() );
-		m_pluginWidget = helper;
-	}
-	else
-	{
-		delete helper;
-	}
-#endif
-}
-
-
-
-
-void VstPlugin::showEditor( QWidget * _parent, bool isEffect )
-{
-	QWidget * w = pluginWidget();
-	if( w )
-	{
-#ifdef LMMS_BUILD_WIN32
-		// hide sw, plugin window wrapper on win32
-		// this is obtained from pluginWidget()
-		if( isEffect )
-		{
-			w->setWindowFlags( Qt::FramelessWindowHint );
-			w->setAttribute( Qt::WA_TranslucentBackground );
-		}
-		else
-		{
-			w->setWindowFlags( Qt::WindowCloseButtonHint );
-		}
-#endif
-		w->show();
-		return;
-	}
-
-#ifdef LMMS_BUILD_LINUX
-	if( m_pluginWindowID == 0 )
-	{
-		return;
-	}
-
-	m_pluginWidget = new QWidget( _parent );
-	m_pluginWidget->setFixedSize( m_pluginGeometry );
-	m_pluginWidget->setWindowTitle( name() );
-	if( _parent == NULL )
-	{
-		vstSubWin * sw = new vstSubWin(
-					gui->mainWindow()->workspace() );
-		if( isEffect )
-		{
-			sw->setAttribute( Qt::WA_TranslucentBackground );
-			sw->setWindowFlags( Qt::FramelessWindowHint );
-			sw->setWidget( m_pluginWidget );
-#if QT_VERSION < 0x050000
-			QX11EmbedContainer * xe = new QX11EmbedContainer( sw );
-			xe->embedClient( m_pluginWindowID );
-			xe->setFixedSize( m_pluginGeometry );
-			xe->show();
-#endif
-		} 
-		else
-		{
-			sw->setWindowFlags( Qt::WindowCloseButtonHint );
-			sw->setWidget( m_pluginWidget );
-
-#if QT_VERSION < 0x050000
-			QX11EmbedContainer * xe = new QX11EmbedContainer( sw );
-			xe->embedClient( m_pluginWindowID );
-			xe->setFixedSize( m_pluginGeometry );
-			xe->move( 4, 24 );
-			xe->show();
-#endif
-		}
-	}
-
-#endif
-
-	if( m_pluginWidget )
-	{
-		m_pluginWidget->show();
-	}
 }
 
 
@@ -289,18 +184,27 @@ void VstPlugin::hideEditor()
 
 
 
+void VstPlugin::toggleEditor()
+{
+	QWidget * w = pluginWidget();
+	if( w )
+	{
+		w->setVisible( !w->isVisible() );
+	}
+}
+
+
+
+
 void VstPlugin::loadSettings( const QDomElement & _this )
 {
-	if( pluginWidget() != NULL )
+	if( _this.attribute( "guivisible" ).toInt() )
 	{
-		if( _this.attribute( "guivisible" ).toInt() )
-		{
-			showEditor( NULL, false );
-		}
-		else
-		{
-			hideEditor();
-		}
+		showUI();
+	}
+	else
+	{
+		hideUI();
 	}
 
 	const int num_params = _this.attribute( "numparams" ).toInt();
@@ -334,9 +238,20 @@ void VstPlugin::loadSettings( const QDomElement & _this )
 
 void VstPlugin::saveSettings( QDomDocument & _doc, QDomElement & _this )
 {
-	if( pluginWidget() != NULL )
+	if ( m_embedMethod != "none" )
 	{
-		_this.setAttribute( "guivisible", pluginWidget()->isVisible() );
+		if( pluginWidget() != NULL )
+		{
+			_this.setAttribute( "guivisible", pluginWidget()->isVisible() );
+		}
+	}
+	else
+	{
+		int visible = isUIVisible();
+		if ( visible != -1 )
+		{
+			_this.setAttribute( "guivisible", visible );
+		}
 	}
 
 	// try to save all settings in a chunk
@@ -361,6 +276,18 @@ void VstPlugin::saveSettings( QDomDocument & _doc, QDomElement & _this )
 	_this.setAttribute( "program", currentProgram() );
 }
 
+void VstPlugin::toggleUI()
+{
+	if ( m_embedMethod == "none" )
+	{
+		RemotePlugin::toggleUI();
+	}
+	else if (pluginWidget())
+	{
+		toggleEditor();
+	}
+}
+
 
 
 
@@ -432,6 +359,23 @@ void VstPlugin::setParameterDump( const QMap<QString, QString> & _pdump )
 	unlock();
 }
 
+QWidget *VstPlugin::pluginWidget(bool _top_widget)
+{
+	if ( m_embedMethod == "none" || !m_pluginWidget )
+	{
+		return nullptr;
+	}
+
+	if ( _top_widget && m_pluginWidget->parentWidget() == m_pluginSubWindow )
+	{
+		return m_pluginSubWindow;
+	}
+	else
+	{
+		return m_pluginWidget;
+	}
+}
+
 
 
 
@@ -439,17 +383,17 @@ bool VstPlugin::processMessage( const message & _m )
 {
 	switch( _m.id )
 	{
-		case IdVstBadDllFormat:
-			m_badDllFormat = true;
-			break;
+	case IdVstBadDllFormat:
+		m_badDllFormat = true;
+		break;
 
-		case IdVstPluginWindowID:
-			m_pluginWindowID = _m.getInt();
-			break;
+	case IdVstPluginWindowID:
+		m_pluginWindowID = _m.getInt();
+		break;
 
-		case IdVstPluginEditorGeometry:
-			m_pluginGeometry = QSize( _m.getInt( 0 ),
-							_m.getInt( 1 ) );
+	case IdVstPluginEditorGeometry:
+		m_pluginGeometry = QSize( _m.getInt( 0 ),
+								  _m.getInt( 1 ) );
 			break;
 
 		case IdVstPluginName:
@@ -624,6 +568,46 @@ void VstPlugin::idleUpdate()
 	unlock();
 }
 
+void VstPlugin::showUI()
+{
+	if ( m_embedMethod == "none" )
+	{
+		RemotePlugin::showUI();
+	}
+	else
+	{
+		if (! pluginWidget()) {
+			createUI( NULL, false );
+		}
+
+		QWidget * w = pluginWidget();
+		if( w )
+		{
+			w->show();
+		}
+	}
+}
+
+void VstPlugin::hideUI()
+{
+	if ( m_embedMethod == "none" )
+	{
+		RemotePlugin::hideUI();
+	}
+	else if ( pluginWidget() != nullptr )
+	{
+		hideEditor();
+	}
+}
+
+// X11Embed only
+void VstPlugin::handleClientEmbed()
+{
+	lock();
+	sendMessage( IdShowUI );
+	unlock();
+}
+
 
 
 void VstPlugin::loadChunk( const QByteArray & _chunk )
@@ -667,6 +651,114 @@ QByteArray VstPlugin::saveChunk()
 	return a;
 }
 
+void VstPlugin::createUI( QWidget * parent, bool isEffect )
+{
+	if( m_pluginWindowID == 0 )
+	{
+		return;
+	}
+
+	QWidget* container = nullptr;
+	m_pluginSubWindow = new vstSubWin( gui->mainWindow()->workspace() );
+	auto sw = m_pluginSubWindow.data();
+
+#if QT_VERSION >= 0x050100
+	if (m_embedMethod == "qt" )
+	{
+		QWindow* vw = QWindow::fromWinId(m_pluginWindowID);
+		container = QWidget::createWindowContainer(vw, sw );
+		container->installEventFilter(this);
+	} else
+#endif
+
+#ifdef LMMS_BUILD_WIN32
+	if (m_embedMethod == "win32" )
+	{
+		QWidget * helper = new QWidget;
+		QHBoxLayout * l = new QHBoxLayout( helper );
+		QWidget * target = new QWidget( helper );
+		l->setSpacing( 0 );
+		l->setMargin( 0 );
+		l->addWidget( target );
+
+		// we've to call that for making sure, Qt created the windows
+		helper->winId();
+		HWND targetHandle = (HWND)target->winId();
+		HWND pluginHandle = (HWND)(intptr_t)m_pluginWindowID;
+
+		DWORD style = GetWindowLong(pluginHandle, GWL_STYLE);
+		style = style & ~(WS_POPUP);
+		style = style | WS_CHILD;
+		SetWindowLong(pluginHandle, GWL_STYLE, style);
+		SetParent(pluginHandle, targetHandle);
+
+		DWORD threadId = GetWindowThreadProcessId(pluginHandle, NULL);
+		DWORD currentThreadId = GetCurrentThreadId();
+		AttachThreadInput(currentThreadId, threadId, true);
+
+		container = helper;
+		RemotePlugin::showUI();
+
+	} else
+#endif
+
+#ifdef LMMS_BUILD_LINUX
+	if (m_embedMethod == "xembed" )
+	{
+		QX11EmbedContainer * embedContainer = new QX11EmbedContainer( sw );
+		connect(embedContainer, SIGNAL(clientIsEmbedded()), this, SLOT(handleClientEmbed()));
+		embedContainer->embedClient( m_pluginWindowID );
+		container = embedContainer;
+	} else
+#endif
+	{
+		qCritical() << "Unknown embed method" << m_embedMethod;
+		delete m_pluginSubWindow;
+		return;
+	}
+
+	container->setFixedSize( m_pluginGeometry );
+	container->setWindowTitle( name() );
+
+	if( parent == NULL )
+	{
+		m_pluginWidget = container;
+
+		sw->setWidget(container);
+
+		if( isEffect )
+		{
+			sw->setAttribute( Qt::WA_TranslucentBackground );
+			sw->setWindowFlags( Qt::FramelessWindowHint );
+		}
+		else
+		{
+			sw->setWindowFlags( Qt::WindowCloseButtonHint );
+		}
+	};
+
+	container->setFixedSize( m_pluginGeometry );
+}
+
+bool VstPlugin::eventFilter(QObject *obj, QEvent *event)
+{
+#if QT_VERSION >= 0x050100
+	if (embedMethod() == "qt" && obj == m_pluginWidget)
+	{
+		if (event->type() == QEvent::Show) {
+			RemotePlugin::showUI();
+		}
+		qDebug() << obj << event;
+	}
+#endif
+	return false;
+}
+
+QString VstPlugin::embedMethod() const
+{
+	return m_embedMethod;
+}
+
 
 
 

plugins/vst_base/VstPlugin.h

@@ -35,6 +35,8 @@
 #include "JournallingObject.h"
 #include "communication.h"
 
+class vstSubWin;
+
 
 class PLUGIN_EXPORT VstPlugin : public RemotePlugin, public JournallingObject
 {
@@ -52,8 +54,8 @@ public:
 		return m_pluginWindowID != 0;
 	}
 
-	void showEditor( QWidget * _parent = NULL, bool isEffect = false );
 	void hideEditor();
+	void toggleEditor();
 
 	inline const QString & name() const
 	{
@@ -91,17 +93,7 @@ public:
 	void setParameterDump( const QMap<QString, QString> & _pdump );
 
 
-	inline QWidget * pluginWidget( bool _top_widget = true )
-	{
-		if( _top_widget && m_pluginWidget )
-		{
-			if( m_pluginWidget->parentWidget() )
-			{
-				return m_pluginWidget->parentWidget();
-			}
-		}
-		return m_pluginWidget;
-	}
+	QWidget * pluginWidget( bool _top_widget = true );
 
 	virtual void loadSettings( const QDomElement & _this );
 	virtual void saveSettings( QDomDocument & _doc, QDomElement & _this );
@@ -111,6 +103,12 @@ public:
 		return "vstplugin";
 	}
 
+	void toggleUI() override;
+
+	void createUI( QWidget *parent, bool isEffect );
+	bool eventFilter(QObject *obj, QEvent *event);
+
+	QString embedMethod() const;
 
 public slots:
 	void setTempo( bpm_t _bpm );
@@ -123,6 +121,10 @@ public slots:
 	void setParam( int i, float f );
 	void idleUpdate();
 
+	void showUI() override;
+	void hideUI() override;
+
+	void handleClientEmbed();
 
 private:
 	void loadChunk( const QByteArray & _chunk );
@@ -130,8 +132,10 @@ private:
 
 	QString m_plugin;
 	QPointer<QWidget> m_pluginWidget;
+	QPointer<vstSubWin> m_pluginSubWindow;
 	int m_pluginWindowID;
 	QSize m_pluginGeometry;
+	const QString m_embedMethod;
 
 	bool m_badDllFormat;
 

plugins/vst_base/basename.c

@@ -1,166 +0,0 @@
-/* basename.c
- *
- * $Id: basename.c,v 1.2 2007/03/08 23:15:58 keithmarshall Exp $
- *
- * Provides an implementation of the "basename" function, conforming
- * to SUSv3, with extensions to accommodate Win32 drive designators,
- * and suitable for use on native Microsoft(R) Win32 platforms.
- *
- * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
- *
- * This is free software.  You may redistribute and/or modify it as you
- * see fit, without restriction of copyright.
- *
- * This software is provided "as is", in the hope that it may be useful,
- * but WITHOUT WARRANTY OF ANY KIND, not even any implied warranty of
- * MERCHANTABILITY, nor of FITNESS FOR ANY PARTICULAR PURPOSE.  At no
- * time will the author accept any form of liability for any damages,
- * however caused, resulting from the use of this software.
- *
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <locale.h>
-
-#ifndef __cdecl  /* If compiling on any non-Win32 platform ... */
-#define __cdecl  /* this may not be defined.                   */
-#endif
-
-__cdecl char *basename( char *path )
-{
-  size_t len;
-  static char *retfail = NULL;
-
-  /* to handle path names for files in multibyte character locales,
-   * we need to set up LC_CTYPE to match the host file system locale
-   */
-
-  char *locale = setlocale( LC_CTYPE, NULL );
-  if( locale != NULL ) locale = strdup( locale );
-  setlocale( LC_CTYPE, "" );
-
-  if( path && *path )
-  {
-    /* allocate sufficient local storage space,
-     * in which to create a wide character reference copy of path
-     */
-
-    wchar_t refcopy[1 + (len = mbstowcs( NULL, path, 0 ))];
-
-    /* create the wide character reference copy of path,
-     * and step over the drive designator, if present ...
-     */
-
-    wchar_t *refpath = refcopy;
-    if( ((len = mbstowcs( refpath, path, len )) > 1) && (refpath[1] == L':') )
-    {
-      /* FIXME: maybe should confirm *refpath is a valid drive designator */
-
-      refpath += 2;
-    }
-
-    /* ensure that our wide character reference path is NUL terminated */
-
-    refcopy[ len ] = L'\0';
-
-    /* check again, just to ensure we still have a non-empty path name ... */
-
-    if( *refpath )
-    {
-      /* and, when we do, process it in the wide character domain ...
-       * scanning from left to right, to the char after the final dir separator
-       */
-
-      wchar_t *refname;
-      for( refname = refpath ; *refpath ; ++refpath )
-      {
-	if( (*refpath == L'/') || (*refpath == L'\\') )
-	{
-	  /* we found a dir separator ...
-	   * step over it, and any others which immediately follow it
-	   */
-
-	  while( (*refpath == L'/') || (*refpath == L'\\') )
-	    ++refpath;
-
-	  /* if we didn't reach the end of the path string ... */
-
-	  if( *refpath )
-
-	    /* then we have a new candidate for the base name */
-
-	    refname = refpath;
-
-	  /* otherwise ...
-	   * strip off any trailing dir separators which we found
-	   */
-
-	  else while(  (refpath > refname)
-	  &&          ((*--refpath == L'/') || (*refpath == L'\\'))   )
-	    *refpath = L'\0';
-	}
-      }
-
-      /* in the wide character domain ...
-       * refname now points at the resolved base name ...
-       */
-
-      if( *refname )
-      {
-	/* if it's not empty,
-	 * then we transform the full normalised path back into
-	 * the multibyte character domain, and skip over the dirname,
-	 * to return the resolved basename.
-	 */
-	
-	if( (len = wcstombs( path, refcopy, len )) != (size_t)(-1) )
-	  path[ len ] = '\0';
-	*refname = L'\0';
-	if( (len = wcstombs( NULL, refcopy, 0 )) != (size_t)(-1) )
-	  path += len;
-      }
-
-      else
-      {
-	/* the basename is empty, so return the default value of "/",
-	 * transforming from wide char to multibyte char domain, and
-	 * returning it in our own buffer.
-	 */
-
-	retfail = (char *) realloc( retfail, len = 1 + wcstombs( NULL, L"/", 0 ));
-	wcstombs( path = retfail, L"/", len );
-      }
-
-      /* restore the caller's locale, clean up, and return the result */
-
-      setlocale( LC_CTYPE, locale );
-      free( locale );
-      return( path );
-    }
-
-    /* or we had an empty residual path name, after the drive designator,
-     * in which case we simply fall through ...
-     */
-  }
-
-  /* and, if we get to here ...
-   * the path name is either NULL, or it decomposes to an empty string;
-   * in either case, we return the default value of "." in our own buffer,
-   * reloading it with the correct value, transformed from the wide char
-   * to the multibyte char domain, just in case the caller trashed it
-   * after a previous call.
-   */
-
-  retfail = (char *) realloc( retfail, len = 1 + wcstombs( NULL, L".", 0 ));
-  wcstombs( retfail, L".", len );
-
-  /* restore the caller's locale, clean up, and return the result */
-
-  setlocale( LC_CTYPE, locale );
-  free( locale );
-  return( retfail );
-}
-
-/* $RCSfile: basename.c,v $$Revision: 1.2 $: end of file */

plugins/vst_base/communication.h

@@ -30,6 +30,7 @@
 #include "RemotePlugin.h"
 
 
+
 struct VstParameterDumpItem
 {
 	int32_t index;
@@ -56,7 +57,6 @@ enum VstRemoteMessageIDs
 {
 	// vstPlugin -> remoteVstPlugin
 	IdVstLoadPlugin = IdUserBase,
-	IdVstPluginWindowInformation,
 	IdVstClosePlugin,
 	IdVstSetTempo,
 	IdVstSetLanguage,

src/3rdparty/CMakeLists.txt

@@ -1,4 +1,9 @@
 set(CMAKE_C_FLAGS "")
 set(CMAKE_CXX_FLAGS "")
 
+IF(QT5 AND LMMS_BUILD_LINUX)
+	set(BUILD_SHARED_LIBS OFF)
+	add_subdirectory(qt5-x11embed)
+ENDIF()
+
 ADD_SUBDIRECTORY(rpmalloc)

src/CMakeLists.txt

@@ -134,7 +134,7 @@ IF(LMMS_BUILD_HAIKU)
 	SET(EXTRA_LIBRARIES "-lnetwork")
 ENDIF()
 
-SET(LMMS_REQUIRED_LIBS
+SET(LMMS_REQUIRED_LIBS ${LMMS_REQUIRED_LIBS}
 	${CMAKE_THREAD_LIBS_INIT}
 	${QT_LIBRARIES}
 	${ASOUND_LIBRARY}

src/core/ConfigManager.cpp

@@ -186,6 +186,39 @@ QString ConfigManager::defaultVersion() const
 	return LMMS_VERSION;
 }
 
+QStringList ConfigManager::availabeVstEmbedMethods()
+{
+	QStringList methods;
+	methods.append("none");
+#if QT_VERSION >= 0x050100
+	methods.append("qt");
+#endif
+#ifdef LMMS_BUILD_WIN32
+	methods.append("win32");
+#endif
+#ifdef LMMS_BUILD_LINUX
+#if QT_VERSION >= 0x050000
+	if (static_cast<QGuiApplication*>(QApplication::instance())->
+		platformName() == "xcb")
+#else
+	if (qgetenv("QT_QPA_PLATFORM").isNull()
+		|| qgetenv("QT_QPA_PLATFORM") == "xcb")
+#endif
+	{
+		methods.append("xembed");
+	}
+#endif
+	return methods;
+}
+
+QString ConfigManager::vstEmbedMethod() const
+{
+	QStringList methods = availabeVstEmbedMethods();
+	QString defaultMethod = *(methods.end() - 1);
+	QString currentMethod = value( "ui", "vstembedmethod", defaultMethod );
+	return methods.contains(currentMethod) ? currentMethod : defaultMethod;
+}
+
 bool ConfigManager::hasWorkingDir() const
 {
 	return QDir( m_workingDir ).exists();
@@ -336,12 +369,15 @@ void ConfigManager::setValue( const QString & cls,
 {
 	if( m_settings.contains( cls ) )
 	{
-		for( stringPairVector::iterator it = m_settings[cls].begin();
-					it != m_settings[cls].end(); ++it )
+		for( QPair<QString, QString>& pair : m_settings[cls])
 		{
-			if( ( *it ).first == attribute )
+			if( pair.first == attribute )
 			{
-				( *it ).second = value;
+				if ( pair.second != value )
+				{
+					pair.second = value;
+					emit valueChanged( cls, attribute, value );
+				}
 				return;
 			}
 		}

src/core/RemotePlugin.cpp

@@ -164,8 +164,8 @@ RemotePlugin::~RemotePlugin()
 
 
 
-bool RemotePlugin::init( const QString &pluginExecutable,
-							bool waitForInitDoneMsg )
+bool RemotePlugin::init(const QString &pluginExecutable,
+							bool waitForInitDoneMsg , QStringList extraArgs)
 {
 	lock();
 	if( m_failed )
@@ -208,6 +208,7 @@ bool RemotePlugin::init( const QString &pluginExecutable,
 #else
 	args << m_socketFile;
 #endif
+	args << extraArgs;
 #ifndef DEBUG_REMOTE_PLUGIN
 	m_process.setProcessChannelMode( QProcess::ForwardedChannels );
 	m_process.setWorkingDirectory( QCoreApplication::applicationDirPath() );
@@ -392,6 +393,20 @@ void RemotePlugin::processMidiEvent( const MidiEvent & _e,
 	unlock();
 }
 
+void RemotePlugin::showUI()
+{
+	lock();
+	sendMessage( IdShowUI );
+	unlock();
+}
+
+void RemotePlugin::hideUI()
+{
+	lock();
+	sendMessage( IdHideUI );
+	unlock();
+}
+
 
 
 

src/gui/SetupDialog.cpp

@@ -67,8 +67,6 @@
 #include "MidiApple.h"
 #include "MidiDummy.h"
 
-
-
 inline void labelWidget( QWidget * _w, const QString & _txt )
 {
 	QLabel * title = new QLabel( _txt, _w );
@@ -137,12 +135,13 @@ SetupDialog::SetupDialog( ConfigTabs _tab_to_open ) :
 	m_displayWaveform(ConfigManager::inst()->value( "ui",
 						   "displaywaveform").toInt() ),
 	m_disableAutoQuit(ConfigManager::inst()->value( "ui",
-						   "disableautoquit").toInt() )
+						   "disableautoquit").toInt() ),
+	m_vstEmbedMethod( ConfigManager::inst()->vstEmbedMethod() )
 {
 	setWindowIcon( embed::getIconPixmap( "setup_general" ) );
 	setWindowTitle( tr( "Setup LMMS" ) );
 	setModal( true );
-	setFixedSize( 452, 520 );
+	setFixedSize( 452, 570 );
 
 	Engine::projectJournal()->setJournalling( false );
 
@@ -159,7 +158,7 @@ SetupDialog::SetupDialog( ConfigTabs _tab_to_open ) :
 	m_tabBar->setFixedWidth( 72 );
 
 	QWidget * ws = new QWidget( settings );
-	int wsHeight = 370;
+	int wsHeight = 420;
 #ifdef LMMS_HAVE_STK
 	wsHeight += 50;
 #endif
@@ -168,7 +167,7 @@ SetupDialog::SetupDialog( ConfigTabs _tab_to_open ) :
 #endif
 	ws->setFixedSize( 360, wsHeight );
 	QWidget * general = new QWidget( ws );
-	general->setFixedSize( 360, 240 );
+	general->setFixedSize( 360, 290 );
 	QVBoxLayout * gen_layout = new QVBoxLayout( general );
 	gen_layout->setSpacing( 0 );
 	gen_layout->setMargin( 0 );
@@ -335,6 +334,27 @@ SetupDialog::SetupDialog( ConfigTabs _tab_to_open ) :
 
 	misc_tw->setFixedHeight( YDelta*labelNumber + HeaderSize );
 
+	TabWidget* embed_tw = new TabWidget( tr( "PLUGIN EMBEDDING" ), general);
+	embed_tw->setFixedHeight( 48 );
+	m_vstEmbedComboBox = new QComboBox( embed_tw );
+	m_vstEmbedComboBox->move( XDelta, YDelta );
+
+	QStringList embedMethods = ConfigManager::availabeVstEmbedMethods();
+	m_vstEmbedComboBox->addItem( tr( "No embedding" ), "none" );
+	if( embedMethods.contains("qt") )
+	{
+		m_vstEmbedComboBox->addItem( tr( "Embed using Qt API" ), "qt" );
+	}
+	if( embedMethods.contains("win32") )
+	{
+		m_vstEmbedComboBox->addItem( tr( "Embed using native Win32 API" ), "win32" );
+	}
+	if( embedMethods.contains("xembed") )
+	{
+		m_vstEmbedComboBox->addItem( tr( "Embed using XEmbed protocol" ), "xembed" );
+	}
+	m_vstEmbedComboBox->setCurrentIndex( m_vstEmbedComboBox->findData( m_vstEmbedMethod ) );
+
 	TabWidget * lang_tw = new TabWidget( tr( "LANGUAGE" ), general );
 	lang_tw->setFixedHeight( 48 );
 	QComboBox * changeLang = new QComboBox( lang_tw );
@@ -380,13 +400,15 @@ SetupDialog::SetupDialog( ConfigTabs _tab_to_open ) :
 	gen_layout->addSpacing( 10 );
 	gen_layout->addWidget( misc_tw );
 	gen_layout->addSpacing( 10 );
+	gen_layout->addWidget( embed_tw );
+	gen_layout->addSpacing( 10 );
 	gen_layout->addWidget( lang_tw );
 	gen_layout->addStretch();
 
 
 
 	QWidget * paths = new QWidget( ws );
-	int pathsHeight = 370;
+	int pathsHeight = 420;
 #ifdef LMMS_HAVE_STK
 	pathsHeight += 55;
 #endif
@@ -1001,6 +1023,20 @@ SetupDialog::~SetupDialog()
 
 void SetupDialog::accept()
 {
+	if( m_warnAfterSetup )
+	{
+		QMessageBox::information( NULL, tr( "Restart LMMS" ),
+					tr( "Please note that most changes "
+						"won't take effect until "
+						"you restart LMMS!" ),
+					QMessageBox::Ok );
+	}
+
+	// Hide dialog before setting values. This prevents an obscure bug
+	// where non-embedded VST windows would steal focus and prevent LMMS
+	// from taking mouse input, rendering the application unusable.
+	QDialog::accept();
+
 	ConfigManager::inst()->setValue( "mixer", "framesperaudiobuffer",
 					QString::number( m_bufferSize ) );
 	ConfigManager::inst()->setValue( "mixer", "audiodev",
@@ -1044,6 +1080,12 @@ void SetupDialog::accept()
 	ConfigManager::inst()->setValue( "ui", "disableautoquit",
 					QString::number( m_disableAutoQuit ) );
 	ConfigManager::inst()->setValue( "app", "language", m_lang );
+	ConfigManager::inst()->setValue( "ui", "vstembedmethod",
+#if QT_VERSION >= 0x050000
+					m_vstEmbedComboBox->currentData().toString() );
+#else
+					m_vstEmbedComboBox->itemData(m_vstEmbedComboBox->currentIndex()).toString() );
+#endif
 
 
 	ConfigManager::inst()->setWorkingDir(QDir::fromNativeSeparators(m_workingDir));
@@ -1074,16 +1116,6 @@ void SetupDialog::accept()
 	}
 
 	ConfigManager::inst()->saveConfigFile();
-
-	QDialog::accept();
-	if( m_warnAfterSetup )
-	{
-		QMessageBox::information( NULL, tr( "Restart LMMS" ),
-					tr( "Please note that most changes "
-						"won't take effect until "
-						"you restart LMMS!" ),
-					QMessageBox::Ok );
-	}
 }
 
 

src/gui/SubWindow.cpp

@@ -97,7 +97,10 @@ void SubWindow::paintEvent( QPaintEvent * )
 {
 	QPainter p( this );
 	QRect rect( 0, 0, width(), m_titleBarHeight );
-	bool isActive = SubWindow::mdiArea()->activeSubWindow() == this;
+
+	bool isActive = mdiArea()
+			? mdiArea()->activeSubWindow() == this
+			: false;
 
 	p.fillRect( rect, isActive ? activeColor() : p.pen().brush() );