From cd018c04ffa4438183a5617fb97b499700b45b88 Mon Sep 17 00:00:00 2001 From: Dominic Clark Date: Tue, 31 Oct 2023 05:19:00 +0000 Subject: [PATCH] Improve CI cache usage (#6868) * Update third-party actions to latest version * Use vcpkg in manifest mode * Only trim ccache after build * Use ccache with MSVC * Use Brewfile and cache Homebrew downloads * Use --print-config for ccache 3 * Attempt to make ccache actually work with MSVC * Zero ccache stats before building * Use SDL2 on macOS --- .github/workflows/build.yml | 115 ++++++++++++++++++++++--------- .gitignore | 1 + Brewfile | 20 ++++++ CMakeLists.txt | 29 +++++--- cmake/modules/CompileCache.cmake | 47 ++++++++----- vcpkg.json | 77 +++++++++++++++++++++ 6 files changed, 233 insertions(+), 56 deletions(-) create mode 100644 Brewfile create mode 100644 vcpkg.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 007842b82..e8cec8867 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,8 @@ jobs: -DUSE_WERROR=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_COMPILE_CACHE=ON - CCACHE_MAXSIZE: 500M + CCACHE_MAXSIZE: 0 + CCACHE_NOCOMPRESS: 1 MAKEFLAGS: -j2 steps: - name: Update and configure Git @@ -38,6 +39,7 @@ jobs: path: ~/.ccache - name: Configure run: | + ccache --zero-stats source /opt/qt5*/bin/qt5*-env.sh || true mkdir build && cd build cmake .. $CMAKE_OPTS -DCMAKE_INSTALL_PREFIX=./install @@ -56,12 +58,15 @@ jobs: with: name: linux path: build/lmms-*.AppImage - - name: Print ccache statistics + - name: Trim ccache and print statistics run: | + ccache --cleanup echo "[ccache config]" - ccache -p + ccache --print-config echo "[ccache stats]" - ccache -s + ccache --show-stats + env: + CCACHE_MAXSIZE: 500M macos: name: macos runs-on: macos-11 @@ -70,7 +75,8 @@ jobs: -DUSE_WERROR=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_COMPILE_CACHE=ON - CCACHE_MAXSIZE: 500M + CCACHE_MAXSIZE: 0 + CCACHE_NOCOMPRESS: 1 MAKEFLAGS: -j3 DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer steps: @@ -79,6 +85,15 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Clean up Homebrew download cache + run: rm -rf ~/Library/Caches/Homebrew/downloads + - name: Restore Homebrew download cache + uses: actions/cache/restore@v3 + with: + key: n/a - only restore from restore-keys + restore-keys: | + homebrew- + path: ~/Library/Caches/Homebrew/downloads - name: Cache ccache data uses: actions/cache@v3 with: @@ -89,12 +104,15 @@ jobs: path: ~/Library/Caches/ccache - name: Install dependencies run: | - brew install ccache fftw pkg-config libogg libvorbis lame libsndfile \ - libsamplerate jack sdl libgig libsoundio lilv lv2 stk \ - fluid-synth portaudio fltk qt@5 carla + brew bundle install --verbose npm install --location=global appdmg + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_UPGRADE: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 - name: Configure run: | + ccache --zero-stats mkdir build cmake -S . \ -B build \ @@ -117,12 +135,20 @@ jobs: with: name: macos path: build/lmms-*.dmg - - name: Print ccache statistics + - name: Trim ccache and print statistics run: | + ccache --cleanup echo "[ccache config]" - ccache -p + ccache --show-config echo "[ccache stats]" - ccache -s + ccache --show-stats --verbose + env: + CCACHE_MAXSIZE: 500MB + - name: Save Homebrew download cache + uses: actions/cache/save@v3 + with: + key: homebrew-${{ hashFiles('Brewfile.lock.json') }} + path: ~/Library/Caches/Homebrew/downloads mingw: strategy: fail-fast: false @@ -136,7 +162,8 @@ jobs: -DUSE_WERROR=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_COMPILE_CACHE=ON - CCACHE_MAXSIZE: 500M + CCACHE_MAXSIZE: 0 + CCACHE_NOCOMPRESS: 1 MAKEFLAGS: -j2 steps: - name: Update and configure Git @@ -161,6 +188,7 @@ jobs: path: ~/.ccache - name: Configure run: | + ccache --zero-stats mkdir build && cd build ../cmake/build_win${{ matrix.arch }}.sh - name: Build @@ -174,12 +202,15 @@ jobs: with: name: mingw${{ matrix.arch }} path: build/lmms-*.exe - - name: Print ccache statistics + - name: Trim ccache and print statistics run: | + ccache --cleanup echo "[ccache config]" - ccache -p + ccache --print-config echo "[ccache stats]" - ccache -s + ccache --show-stats + env: + CCACHE_MAXSIZE: 500M msvc: strategy: fail-fast: false @@ -189,6 +220,8 @@ jobs: runs-on: windows-2019 env: qt-version: '5.15.2' + CCACHE_MAXSIZE: 0 + CCACHE_NOCOMPRESS: 1 steps: - name: Check out uses: actions/checkout@v3 @@ -196,49 +229,60 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache vcpkg dependencies + id: cache-deps uses: actions/cache@v3 with: - key: vcpkg-${{ matrix.arch }}-${{ github.ref }}-${{ github.run_id }} + key: vcpkg-${{ matrix.arch }}-${{ hashFiles('vcpkg.json') }} restore-keys: | - vcpkg-${{ matrix.arch }}-${{ github.ref }}- vcpkg-${{ matrix.arch }}- - path: C:\vcpkg\installed + path: build\vcpkg_installed + - name: Cache ccache data + uses: actions/cache@v3 + with: + key: "ccache-${{ github.job }}-${{ matrix.arch }}-${{ github.ref }}\ + -${{ github.run_id }}" + restore-keys: | + ccache-${{ github.job }}-${{ matrix.arch }}-${{ github.ref }}- + ccache-${{ github.job }}-${{ matrix.arch }}- + path: ~\AppData\Local\ccache + - name: Install tools + run: choco install ccache - name: Install 64-bit Qt if: matrix.arch == 'x64' - uses: jurplel/install-qt-action@64bdb64f2c14311d23733a8463e5fcbc65e8775e + uses: jurplel/install-qt-action@b3ea5275e37b734d027040e2c7fe7a10ea2ef946 with: version: ${{ env.qt-version }} arch: win64_msvc2019_64 archives: qtbase qtsvg qttools cache: true - name: Install 32-bit Qt - uses: jurplel/install-qt-action@64bdb64f2c14311d23733a8463e5fcbc65e8775e + uses: jurplel/install-qt-action@b3ea5275e37b734d027040e2c7fe7a10ea2ef946 with: version: ${{ env.qt-version }} arch: win32_msvc2019 archives: qtbase qtsvg qttools cache: true set-env: ${{ matrix.arch == 'x86' }} - - name: Install dependencies - run: | - vcpkg install ` - --triplet=${{ matrix.arch }}-windows ` - --host-triplet=${{ matrix.arch }}-windows ` - --recurse ` - fftw3 fltk fluidsynth[sndfile] libsamplerate libsndfile libstk ` - lilv lv2 portaudio sdl2 - name: Set up build environment - uses: ilammy/msvc-dev-cmd@d8610e2b41c6d0f0c3b4c46dad8df0fd826c68e1 + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 with: arch: ${{ matrix.arch }} - name: Configure run: | - mkdir build + ccache --zero-stats + mkdir build -Force cmake -S . ` -B build ` -G Ninja ` --toolchain C:/vcpkg/scripts/buildsystems/vcpkg.cmake ` - -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_BUILD_TYPE=RelWithDebInfo ` + -DUSE_COMPILE_CACHE=ON ` + -DVCPKG_TARGET_TRIPLET="${{ matrix.arch }}-windows" ` + -DVCPKG_HOST_TRIPLET="${{ matrix.arch }}-windows" ` + -DVCPKG_MANIFEST_INSTALL="${{ env.should_install_manifest }}" + env: + should_install_manifest: + ${{ steps.cache-deps.outputs.cache-hit == 'true' && 'NO' || 'YES' }} - name: Build run: cmake --build build - name: Build tests @@ -250,3 +294,12 @@ jobs: with: name: msvc-${{ matrix.arch }} path: build\lmms-*.exe + - name: Trim ccache and print statistics + run: | + ccache --cleanup + echo "[ccache config]" + ccache --show-config + echo "[ccache stats]" + ccache --show-stats --verbose + env: + CCACHE_MAXSIZE: 500MB diff --git a/.gitignore b/.gitignore index ee289379f..1b855f204 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /plugins/ZynAddSubFx/zynaddsubfx/doc/Makefile /plugins/ZynAddSubFx/zynaddsubfx/doc/gen/Makefile /data/locale/*.qm +Brewfile.lock.json diff --git a/Brewfile b/Brewfile new file mode 100644 index 000000000..1bfbd7b01 --- /dev/null +++ b/Brewfile @@ -0,0 +1,20 @@ +brew "carla" +brew "ccache" +brew "fftw" +brew "fltk" +brew "fluid-synth" +brew "jack" +brew "lame" +brew "libgig" +brew "libogg" +brew "libsamplerate" +brew "libsndfile" +brew "libsoundio" +brew "libvorbis" +brew "lilv" +brew "lv2" +brew "pkg-config" +brew "portaudio" +brew "qt@5" +brew "sdl2" +brew "stk" diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b6d3b4ff..9ef6aaba3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,20 +1,31 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.9) +# Set the given policy to NEW. If it does not exist, it will not be set. If it +# is already set to NEW (most likely due to predating the minimum required CMake +# version), a developer warning is emitted indicating that the policy need no +# longer be explicitly set. +function(enable_policy_if_exists id) + if(POLICY "${id}") + cmake_policy(GET "${id}" current_value) + if(current_value STREQUAL "NEW") + message(AUTHOR_WARNING "${id} is now set to NEW by default, and no longer needs to be explicitly set.") + else() + cmake_policy(SET "${id}" NEW) + endif() + endif() +endfunction() + +# Needed for the SWH Ladspa plugins. See below. +enable_policy_if_exists(CMP0074) # find_package() uses _ROOT variables. +# Needed for ccache support with MSVC +enable_policy_if_exists(CMP0141) # MSVC debug information format flags are selected by an abstraction. + PROJECT(lmms) SET(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules" ${CMAKE_MODULE_PATH}) SET(LMMS_BINARY_DIR ${CMAKE_BINARY_DIR}) SET(LMMS_SOURCE_DIR ${CMAKE_SOURCE_DIR}) -# CMAKE_POLICY Section -IF(COMMAND CMAKE_POLICY) - # TODO: Keep CMP0074 but remove this condition when cmake 3.12+ is guaranteed - IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.12) - # Needed for the SWH Ladspa plugins. See below. - CMAKE_POLICY(SET CMP0074 NEW) # find_package() uses _ROOT variables - ENDIF() -ENDIF(COMMAND CMAKE_POLICY) - # Import of windows.h breaks min()/max() ADD_DEFINITIONS(-DNOMINMAX) diff --git a/cmake/modules/CompileCache.cmake b/cmake/modules/CompileCache.cmake index ed4622bd9..56486e24f 100644 --- a/cmake/modules/CompileCache.cmake +++ b/cmake/modules/CompileCache.cmake @@ -1,25 +1,40 @@ -option(USE_COMPILE_CACHE "Use ccache or clcache for compilation" OFF) +option(USE_COMPILE_CACHE "Use a compiler cache for compilation" OFF) # Compatibility for old option name if(USE_CCACHE) set(USE_COMPILE_CACHE ON) endif() -if(USE_COMPILE_CACHE) - if(MSVC) - set(CACHE_TOOL_NAME clcache) - elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|AppleClang|Clang)") - set(CACHE_TOOL_NAME ccache) - else() - message(WARNING "Compile cache only available with MSVC or GNU") +if(NOT USE_COMPILE_CACHE) + return() +endif() + +if(NOT CMAKE_CXX_COMPILER_ID MATCHES "(GNU|AppleClang|Clang|MSVC)") + message(WARNING "Compiler cache only available with MSVC or GNU") + return() +endif() + +set(CACHE_TOOL_NAME ccache) +find_program(CACHE_TOOL "${CACHE_TOOL_NAME}") +if(NOT CACHE_TOOL) + message(WARNING "USE_COMPILE_CACHE enabled, but no ${CACHE_TOOL_NAME} found") + return() +endif() + +if(MSVC) + # ccache doesn't support debug information in the PDB format. Setting the + # debug information format requires CMP0141, introduced with CMake 3.25, to + # be set to NEW prior to the initial `project` command. + if(CMAKE_VERSION VERSION_LESS "3.25") + message(WARNING "Use of compiler cache with MSVC requires at least CMake 3.25") + return() endif() - find_program(CACHE_TOOL ${CACHE_TOOL_NAME}) - if (CACHE_TOOL) - message(STATUS "Using ${CACHE_TOOL} found for caching") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CACHE_TOOL}) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CACHE_TOOL}) - else() - message(WARNING "USE_COMPILE_CACHE enabled, but no ${CACHE_TOOL_NAME} found") - endif() + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$<$:Embedded>") endif() + +message(STATUS "Using ${CACHE_TOOL} for compiler caching") + +# TODO CMake 3.21: Use CMAKE___LAUNCHER variables instead +set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CACHE_TOOL}") +set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CACHE_TOOL}") diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 000000000..48a3e3c28 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "dependencies": [ + { + "name": "fftw3", + "default-features": false, + "features": [ + "sse", + "sse2", + "avx", + "avx2" + ] + }, + { + "name": "fltk", + "default-features": false + }, + { + "name": "fluidsynth", + "default-features": false, + "features": [ + "sndfile" + ] + }, + { + "name": "libogg", + "default-features": false + }, + { + "name": "libsamplerate", + "default-features": false + }, + { + "name": "libsndfile", + "default-features": false, + "features": [ + "external-libs", + "mpeg" + ] + }, + { + "name": "libstk", + "default-features": false + }, + { + "name": "libvorbis", + "default-features": false + }, + { + "name": "lilv", + "default-features": false + }, + { + "name": "lv2", + "default-features": false + }, + { + "name": "mp3lame", + "default-features": false + }, + { + "name": "portaudio", + "default-features": false + }, + { + "name": "sdl2", + "default-features": false, + "features": [ + "base" + ] + }, + { + "name": "zlib", + "default-features": false + } + ] +}