Add options to enable sanitizers (#6840)

* Add options to enable sanitizers

* Specify them as debugging options

* Apply suggestions made by Dom

* Fix linking issues
There were linking issues, most likely because we were
applying the sanitizer flags to unnecessary targets that
are part of the build. Now, we only focus on adding the
compiler flags to lmmsobjs as a PUBLIC dependency,
and selectively choose what targets we need to apply
the linker flags to as PRIVATE dependencies.

* Add UBSan

* Add status messages

* Remove GNU as supported compiler for MSan

* Revert to using add_<compile|link>_options again

* Fix sanitizer linkage within veal and cmt libraries
I suspect that our sanitizer flags were removed for these two CMake
targets. Instead of setting the properties directly, we call
target_compile_options and target_link_options
instead.

* Remove TODO comment

* Revert "Fix sanitizer linkage within veal and cmt libraries"

This reverts commit b04dce8d8c.

* Use CMAKE_<LANG>_FLAGS_<CONFIG> and apply fixes

* Remove quotes

* Add support for additional flags

* Disable vptr for UBSan

* Print "Debug using" in status for clarity

* Wrap ${supported_compilers} and ${status_flag} in quotes

* Replace semicolons with spaces in additional_flags

* Remove platform check for no-undefined flag
The platform check was added as an attempt
to make this branch compile with the veal
and cmt libraries. However, it seems to work
without it, so the problem must've been something
else occuring in these files.

* Undo change to FP exceptions status message

* Use spaces instead of tabs

* Remove -no-undefined flag for cmt and veal libraries

CMakeLists.txt

@@ -78,6 +78,10 @@ OPTION(WANT_VST_32	"Include 32-bit VST support" ON)
 OPTION(WANT_VST_64	"Include 64-bit VST support" ON)
 OPTION(WANT_WINMM	"Include WinMM MIDI support" OFF)
 OPTION(WANT_DEBUG_FPE	"Debug floating point exceptions" OFF)
+option(WANT_DEBUG_ASAN	"Enable AddressSanitizer" OFF)
+option(WANT_DEBUG_TSAN	"Enable ThreadSanitizer" OFF)
+option(WANT_DEBUG_MSAN	"Enable MemorySanitizer" OFF)
+option(WANT_DEBUG_UBSAN	"Enable UndefinedBehaviorSanitizer" OFF)
 OPTION(BUNDLE_QT_TRANSLATIONS	"Install Qt translation files for LMMS" OFF)
 
 
@@ -649,6 +653,31 @@ IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
 	ENDIF(WIN32)
 ENDIF()
 
+# add enabled sanitizers
+function(add_sanitizer sanitizer supported_compilers want_flag status_flag)
+	if(${want_flag})
+		if(CMAKE_CXX_COMPILER_ID MATCHES "${supported_compilers}")
+			set("${status_flag}" "Enabled" PARENT_SCOPE)
+			string(REPLACE ";" " " additional_flags "${ARGN}")
+			# todo CMake 3.13: use add_compile_options/add_link_options instead
+			set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fsanitize=${sanitizer} ${additional_flags}" PARENT_SCOPE)
+			set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=${sanitizer} ${additional_flags}" PARENT_SCOPE)
+		else()
+			set("${status_flag}" "Wanted but disabled due to unsupported compiler" PARENT_SCOPE)
+		endif()
+	else()
+		set("${status_flag}" "Disabled" PARENT_SCOPE)
+	endif()
+endfunction()
+
+add_sanitizer(address "GNU|Clang|MSVC" WANT_DEBUG_ASAN STATUS_DEBUG_ASAN)
+add_sanitizer(thread "GNU|Clang" WANT_DEBUG_TSAN STATUS_DEBUG_TSAN)
+add_sanitizer(memory "Clang" WANT_DEBUG_MSAN STATUS_DEBUG_MSAN -fno-omit-frame-pointer)
+# UBSan does not link with vptr enabled due to a problem with references from PeakControllerEffect
+# not being found by PeakController
+add_sanitizer(undefined "GNU|Clang" WANT_DEBUG_UBSAN STATUS_DEBUG_UBSAN -fno-sanitize=vptr)
+
+
 # use ccache
 include(CompileCache)
 
@@ -717,7 +746,6 @@ ADD_CUSTOM_TARGET(uninstall
 	COMMAND ${CMAKE_COMMAND} -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/uninstall.cmake"
 )
 
-
 #
 # display configuration information
 #
@@ -783,7 +811,11 @@ MESSAGE(
 MESSAGE(
 "Developer options\n"
 "-----------------------------------------\n"
-"* Debug FP exceptions         : ${STATUS_DEBUG_FPE}\n"
+"* Debug FP exceptions               : ${STATUS_DEBUG_FPE}\n"
+"* Debug using AddressSanitizer      : ${STATUS_DEBUG_ASAN}\n"
+"* Debug using ThreadSanitizer       : ${STATUS_DEBUG_TSAN}\n"
+"* Debug using MemorySanitizer       : ${STATUS_DEBUG_MSAN}\n"
+"* Debug using UBSanitizer           : ${STATUS_DEBUG_UBSAN}\n"
 )
 
 MESSAGE(

plugins/LadspaEffect/calf/CMakeLists.txt

@@ -36,9 +36,9 @@ TARGET_COMPILE_DEFINITIONS(veal PRIVATE DISABLE_OSC=1)
 
 SET(INLINE_FLAGS "")
 IF("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-	SET(INLINE_FLAGS "-finline-functions-called-once -finline-limit=80")
+	SET(INLINE_FLAGS -finline-functions-called-once -finline-limit=80)
 ENDIF()
-SET_TARGET_PROPERTIES(veal PROPERTIES COMPILE_FLAGS "-fexceptions -O2 -finline-functions ${INLINE_FLAGS}")
+target_compile_options(veal PRIVATE -fexceptions -O2 -finline-functions ${INLINE_FLAGS})
 
 if(LMMS_BUILD_WIN32)
 	add_custom_command(
@@ -50,5 +50,5 @@ if(LMMS_BUILD_WIN32)
 	)
 endif()
 IF(NOT LMMS_BUILD_APPLE AND NOT LMMS_BUILD_OPENBSD)
-	SET_TARGET_PROPERTIES(veal PROPERTIES LINK_FLAGS "${LINK_FLAGS} -shared -Wl,-no-undefined")
+	target_link_libraries(veal PRIVATE -shared)
 ENDIF()

plugins/LadspaEffect/cmt/CMakeLists.txt

@@ -5,7 +5,7 @@ ADD_LIBRARY(cmt MODULE ${SOURCES})
 INSTALL(TARGETS cmt LIBRARY DESTINATION "${PLUGIN_DIR}/ladspa")
 
 SET_TARGET_PROPERTIES(cmt PROPERTIES PREFIX "")
-SET_TARGET_PROPERTIES(cmt PROPERTIES COMPILE_FLAGS "-Wall -O3 -fno-strict-aliasing")
+target_compile_options(cmt PRIVATE -Wall -O3 -fno-strict-aliasing)
 
 if(LMMS_BUILD_WIN32)
 	add_custom_command(
@@ -18,10 +18,10 @@ if(LMMS_BUILD_WIN32)
 endif()
 
 if(NOT LMMS_BUILD_WIN32)
-	set_target_properties(cmt PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} -fPIC")
+	target_compile_options(cmt PRIVATE -fPIC)
 endif()
 
 IF(NOT LMMS_BUILD_APPLE AND NOT LMMS_BUILD_OPENBSD)
-	SET_TARGET_PROPERTIES(cmt PROPERTIES LINK_FLAGS "${LINK_FLAGS} -shared -Wl,-no-undefined")
+	target_link_libraries(cmt PRIVATE -shared)
 ENDIF()