hashtags/lmms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Replace MemoryManager implementation with rpmalloc (#3873)

Browse Source 
* Replace MemoryManager implementation with rpmalloc
    Fixes #3865
* Travis: Specify OSX image for Qt5 build
This commit is contained in:
Lukas W
2017-10-18 17:02:40 +02:00
committed by GitHub
parent 20ea945cee
commit 8d6cb120b9
15 changed files with 105 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "src/3rdparty/rpmalloc/rpmalloc"]
path = src/3rdparty/rpmalloc/rpmalloc
url = https://github.com/rampantpixels/rpmalloc.git

1
.travis.yml
View File

@@ -16,6 +16,7 @@ matrix:
- env: QT5=True TARGET_OS=win32
- env: QT5=True TARGET_OS=win64
- os: osx
osx_image: xcode8.2
env: QT5=True
before_install:
- . ${TRAVIS_BUILD_DIR}/.travis/${TRAVIS_OS_NAME}.${TARGET_OS}.before_install.sh

1
include/AutomatableModel.h
View File

@@ -25,6 +25,7 @@
#ifndef AUTOMATABLE_MODEL_H
#define AUTOMATABLE_MODEL_H
#include <QtCore/QMap>
#include <QtCore/QMutex>
#include "JournallingObject.h"

108
include/MemoryManager.h
View File

@@ -1,6 +1,7 @@
/*
* MemoryManager.h - A lightweight, generic memory manager for LMMS
* MemoryManager.h
*
* Copyright (c) 2017 Lukas W <lukaswhl/at/gmail.com>
* Copyright (c) 2014 Vesa Kivimäki
* Copyright (c) 2007-2014 Tobias Doerffel <tobydox/at/users.sourceforge.net>
*
@@ -26,81 +27,22 @@
#ifndef MEMORY_MANAGER_H
#define MEMORY_MANAGER_H
#include <QtCore/QVector>
#include <QtCore/QMutex>
#include <QtCore/QHash>
#include "MemoryHelper.h"
#include <cstddef>
#include <vector>
#include "export.h"
class QReadWriteLock;
const int MM_CHUNK_SIZE = 64; // granularity of managed memory
const int MM_INITIAL_CHUNKS = 1024 * 1024; // how many chunks to allocate at startup - TODO: make configurable
const int MM_INCREMENT_CHUNKS = 16 * 1024; // min. amount of chunks to increment at a time
struct MemoryPool
{
void * m_pool;
char * m_free;
size_t m_chunks;
QMutex m_mutex;
MemoryPool() :
m_pool( NULL ),
m_free( NULL ),
m_chunks( 0 )
{}
MemoryPool( size_t chunks ) :
m_chunks( chunks )
{
m_free = reinterpret_cast<char*>( MemoryHelper::alignedMalloc( chunks ) );
memset( m_free, 1, chunks );
}
MemoryPool( const MemoryPool & mp ) :
m_pool( mp.m_pool ),
m_free( mp.m_free ),
m_chunks( mp.m_chunks ),
m_mutex()
{}
MemoryPool & operator = ( const MemoryPool & mp )
{
m_pool = mp.m_pool;
m_free = mp.m_free;
m_chunks = mp.m_chunks;
return *this;
}
void * getChunks( int chunksNeeded );
void releaseChunks( void * ptr, int chunks );
};
struct PtrInfo
{
int chunks;
MemoryPool * memPool;
};
typedef QVector<MemoryPool> MemoryPoolVector;
typedef QHash<void*, PtrInfo> PointerInfoMap;
class EXPORT MemoryManager
{
public:
static bool init();
struct ThreadGuard
{
ThreadGuard();
~ThreadGuard();
};
static void * alloc( size_t size );
static void free( void * ptr );
static int extend( int chunks ); // returns index of created pool (for use by alloc)
static void cleanup();
private:
static MemoryPoolVector s_memoryPools;
static QReadWriteLock s_poolMutex;
static PointerInfoMap s_pointerInfo;
static QMutex s_pointerMutex;
};
template<typename T>
@@ -147,32 +89,4 @@ static void operator delete[] ( void * ptr ) \
// and just for symmetry...
#define MM_FREE( ptr ) MemoryManager::free( ptr )
// for debugging purposes
#define MM_OPERATORS_DEBUG \
public: \
static void * operator new ( size_t size ) \
{ \
qDebug( "MM_OPERATORS_DEBUG: new called for %d bytes", size ); \
return MemoryManager::alloc( size ); \
} \
static void * operator new[] ( size_t size ) \
{ \
qDebug( "MM_OPERATORS_DEBUG: new[] called for %d bytes", size ); \
return MemoryManager::alloc( size ); \
} \
static void operator delete ( void * ptr ) \
{ \
qDebug( "MM_OPERATORS_DEBUG: delete called for %p", ptr ); \
MemoryManager::free( ptr ); \
} \
static void operator delete[] ( void * ptr ) \
{ \
qDebug( "MM_OPERATORS_DEBUG: delete[] called for %p", ptr ); \
MemoryManager::free( ptr ); \
}
#endif

1
include/PluginFactory.h
View File

@@ -28,6 +28,7 @@
#include <memory>
#include <QtCore/QFileInfo>
#include <QtCore/QHash>
#include <QtCore/QList>
#include "export.h"

2
plugins/carlabase/carla.h
View File

@@ -25,6 +25,8 @@
#ifndef CARLA_H
#define CARLA_H
#include <QtCore/QMutex>
#include "CarlaNative.h"
#include "Instrument.h"

4
src/3rdparty/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,4 @@
set(CMAKE_C_FLAGS "")
set(CMAKE_CXX_FLAGS "")
ADD_SUBDIRECTORY(rpmalloc)

30
src/3rdparty/rpmalloc/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,30 @@
set(CMAKE_C_FLAGS "-std=c11")
add_library(rpmalloc STATIC
rpmalloc/rpmalloc/rpmalloc.c
rpmalloc/rpmalloc/rpmalloc.h
)
target_include_directories(rpmalloc PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/rpmalloc/rpmalloc
)
if (NOT LMMS_BUILD_WIN32)
target_compile_definitions(rpmalloc
PRIVATE -D_GNU_SOURCE
)
endif()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
target_compile_definitions(rpmalloc
PRIVATE -DENABLE_ASSERTS=1 -DENABLE_VALIDATE_ARGS=1
)
endif()
option(LMMS_ENABLE_MALLOC_STATS "Enables statistics for rpmalloc" OFF)
if (LMMS_ENABLE_MALLOC_STATS)
target_compile_definitions(rpmalloc
PRIVATE -DENABLE_STATISTICS=1
)
endif()

1
src/3rdparty/rpmalloc/rpmalloc vendored Submodule

Submodule src/3rdparty/rpmalloc/rpmalloc added at 2e0479192b

12
src/CMakeLists.txt
View File

@@ -1,3 +1,5 @@
ADD_SUBDIRECTORY(3rdparty)
CONFIGURE_FILE("lmmsconfig.h.in" "${CMAKE_BINARY_DIR}/lmmsconfig.h")
CONFIGURE_FILE("lmmsversion.h.in" "${CMAKE_BINARY_DIR}/lmmsversion.h")
@@ -147,7 +149,9 @@ SET(LMMS_REQUIRED_LIBS
${SAMPLERATE_LIBRARIES}
${SNDFILE_LIBRARIES}
${EXTRA_LIBRARIES}
rpmalloc
)
# Expose required libs for tests binary
SET(LMMS_REQUIRED_LIBS ${LMMS_REQUIRED_LIBS} PARENT_SCOPE)
@@ -155,6 +159,14 @@ TARGET_LINK_LIBRARIES(lmms
${LMMS_REQUIRED_LIBS}
)
FOREACH(LIB ${LMMS_REQUIRED_LIBS})
GET_TARGET_PROPERTY(INCLUDE_DIRS ${LIB} INTERFACE_INCLUDE_DIRECTORIES)
if (INCLUDE_DIRS)
TARGET_INCLUDE_DIRECTORIES(lmmsobjs PRIVATE ${INCLUDE_DIRS})
ENDIF()
ENDFOREACH()
# Required libs for debug msys builds
IF(LMMS_BUILD_MSYS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
TARGET_LINK_LIBRARIES(lmms QtCore4 QtGui4 QtXml4)

216
src/core/MemoryManager.cpp
View File

@@ -1,8 +1,7 @@
/*
* MemoryManager.cpp - A lightweight, generic memory manager for LMMS
* MemoryManager.cpp
*
* Copyright (c) 2014 Vesa Kivimäki
* Copyright (c) 2007-2014 Tobias Doerffel <tobydox/at/users.sourceforge.net>
* Copyright (c) 2017 Lukas W <lukaswhl/at/gmail.com>
*
* This file is part of LMMS - https://lmms.io
*
@@ -25,197 +24,54 @@
#include "MemoryManager.h"
#include <QReadWriteLock>
#include <QtCore/QtGlobal>
#include "rpmalloc.h"
/// Global static object handling rpmalloc intializing and finalizing
struct MemoryManagerGlobalGuard {
MemoryManagerGlobalGuard() {
rpmalloc_initialize();
}
~MemoryManagerGlobalGuard() {
rpmalloc_finalize();
}
} static mm_global_guard;
MemoryPoolVector MemoryManager::s_memoryPools;
QReadWriteLock MemoryManager::s_poolMutex;
PointerInfoMap MemoryManager::s_pointerInfo;
QMutex MemoryManager::s_pointerMutex;
bool MemoryManager::init()
{
s_memoryPools.reserve( 64 );
s_pointerInfo.reserve( 4096 );
// construct first MemoryPool and allocate memory
MemoryPool m ( MM_INITIAL_CHUNKS );
m.m_pool = MemoryHelper::alignedMalloc( MM_INITIAL_CHUNKS * MM_CHUNK_SIZE );
s_memoryPools.append( m );
return true;
namespace {
static thread_local size_t thread_guard_depth;
}
void * MemoryManager::alloc( size_t size )
MemoryManager::ThreadGuard::ThreadGuard()
{
if( !size )
{
return NULL;
}
int requiredChunks = size / MM_CHUNK_SIZE + ( size % MM_CHUNK_SIZE > 0 ? 1 : 0 );
MemoryPool * mp = NULL;
void * ptr = NULL;
MemoryPoolVector::iterator it = s_memoryPools.begin();
s_poolMutex.lockForRead();
while( it != s_memoryPools.end() && !ptr )
{
ptr = ( *it ).getChunks( requiredChunks );
if( ptr )
{
mp = &( *it );
}
++it;
}
s_poolMutex.unlock();
if( ptr )
{
s_pointerMutex.lock();
PtrInfo p;
p.chunks = requiredChunks;
p.memPool = mp;
s_pointerInfo[ptr] = p;
s_pointerMutex.unlock();
return ptr;
}
// can't find enough chunks in existing pools, so
// create a new pool that is guaranteed to have enough chunks
int moreChunks = qMax( requiredChunks, MM_INCREMENT_CHUNKS );
int i = MemoryManager::extend( moreChunks );
mp = &s_memoryPools[i];
ptr = s_memoryPools[i].getChunks( requiredChunks );
if( ptr )
{
s_pointerMutex.lock();
PtrInfo p;
p.chunks = requiredChunks;
p.memPool = mp;
s_pointerInfo[ptr] = p;
s_pointerMutex.unlock();
return ptr;
}
// still no luck? something is horribly wrong
qFatal( "MemoryManager.cpp: Couldn't allocate memory: %d chunks asked", requiredChunks );
return NULL;
}
void MemoryManager::free( void * ptr )
{
if( !ptr )
{
return; // Null pointer deallocations are OK but do not need to be handled
}
// fetch info on the ptr and remove
s_pointerMutex.lock();
if( ! s_pointerInfo.contains( ptr ) ) // if we have no info on ptr, fail loudly
{
qFatal( "MemoryManager: Couldn't find pointer info for pointer: %p", ptr );
}
PtrInfo p = s_pointerInfo[ptr];
s_pointerInfo.remove( ptr );
s_pointerMutex.unlock();
p.memPool->releaseChunks( ptr, p.chunks );
}
int MemoryManager::extend( int chunks )
{
MemoryPool m ( chunks );
m.m_pool = MemoryHelper::alignedMalloc( chunks * MM_CHUNK_SIZE );
s_poolMutex.lockForWrite();
s_memoryPools.append( m );
int i = s_memoryPools.size() - 1;
s_poolMutex.unlock();
return i;
}
void MemoryManager::cleanup()
{
for( MemoryPoolVector::iterator it = s_memoryPools.begin(); it != s_memoryPools.end(); ++it )
{
MemoryHelper::alignedFree( ( *it ).m_pool );
MemoryHelper::alignedFree( ( *it ).m_free );
if (thread_guard_depth++ == 0) {
rpmalloc_thread_initialize();
}
}
void * MemoryPool::getChunks( int chunksNeeded )
MemoryManager::ThreadGuard::~ThreadGuard()
{
if( chunksNeeded > m_chunks ) // not enough chunks in this pool?
{
return NULL;
if (--thread_guard_depth == 0) {
rpmalloc_thread_finalize();
}
}
m_mutex.lock();
static thread_local MemoryManager::ThreadGuard local_mm_thread_guard{};
// now find out if we have a long enough sequence of chunks in this pool
char last = 0;
intptr_t n = 0;
intptr_t index = -1;
bool found = false;
for( int i = 0; i < m_chunks; ++i )
{
if( m_free[i] )
{
if( !last )
{
index = i;
}
++n;
if( n >= chunksNeeded )
{
found = true;
break;
}
}
else
{
n = 0;
}
last = m_free[i];
}
if( found ) // if enough chunks found, return pointer to chunks
{
// set chunk flags to false so we know the chunks are in use
for( intptr_t i = 0; i < chunksNeeded; ++i )
{
m_free[ index + i ] = 0;
}
m_mutex.unlock();
return (char*)m_pool + ( index * MM_CHUNK_SIZE );
}
m_mutex.unlock();
return NULL; // out of stock, come again tomorrow!
void* MemoryManager::alloc(size_t size)
{
// Reference local thread guard to ensure it is initialized.
// Compilers may optimize the instance away otherwise.
Q_UNUSED(&local_mm_thread_guard);
Q_ASSERT_X(rpmalloc_is_thread_initialized(), "MemoryManager::alloc", "Thread not initialized");
return rpmalloc(size);
}
void MemoryPool::releaseChunks( void * ptr, int chunks )
void MemoryManager::free(void * ptr)
{
m_mutex.lock();
intptr_t start = ( (intptr_t)ptr - (intptr_t)m_pool ) / MM_CHUNK_SIZE;
if( start < 0 )
{
qFatal( "MemoryManager: error at releaseChunks() - corrupt pointer info?" );
}
memset( &m_free[ start ], 1, chunks );
m_mutex.unlock();
Q_UNUSED(&local_mm_thread_guard);
Q_ASSERT_X(rpmalloc_is_thread_initialized(), "MemoryManager::free", "Thread not initialized");
return rpfree(ptr);
}

1
src/core/Mixer.cpp
View File

@@ -36,6 +36,7 @@
#include "NotePlayHandle.h"
#include "ConfigManager.h"
#include "SamplePlayHandle.h"
#include "MemoryHelper.h"
// platform-specific audio-interface-classes
#include "AudioAlsa.h"

1
src/core/MixerWorkerThread.cpp
View File

@@ -153,6 +153,7 @@ void MixerWorkerThread::startAndWaitForJobs()
void MixerWorkerThread::run()
{
MemoryManager::ThreadGuard mmThreadGuard; Q_UNUSED(mmThreadGuard);
disable_denormals();
QMutex m;

1
src/core/ProjectRenderer.cpp
View File

@@ -163,6 +163,7 @@ void ProjectRenderer::startProcessing()
void ProjectRenderer::run()
{
MemoryManager::ThreadGuard mmThreadGuard; Q_UNUSED(mmThreadGuard);
#if 0
#ifdef LMMS_BUILD_LINUX
#ifdef LMMS_HAVE_SCHED_H

5
src/core/main.cpp
View File

@@ -57,7 +57,6 @@
#include <signal.h>
#include "MainApplication.h"
#include "MemoryManager.h"
#include "ConfigManager.h"
#include "NotePlayHandle.h"
#include "embed.h"
@@ -203,7 +202,6 @@ void fileCheck( QString &file )
int main( int argc, char * * argv )
{
// initialize memory managers
MemoryManager::init();
NotePlayHandleManager::init();
// intialize RNG
@@ -930,9 +928,6 @@ int main( int argc, char * * argv )
Engine::destroy();
}
// cleanup memory managers
MemoryManager::cleanup();
// ProjectRenderer::updateConsoleProgress() doesn't return line after render
if( coreOnly )
{