Replace LocklessAllocator with new MemoryPool class

MemoryPool maintains a free list of pre-allocated entries stored in a
libcds MPMC Queue. In contrast to LocklessAllocator, it's a lot faster,
less (and less complex) code to maintain for us and it doesn't fail when
it's full.

.gitmodules

@@ -31,6 +31,9 @@
 [submodule "src/3rdparty/mingw-std-threads"]
 	path = src/3rdparty/mingw-std-threads
 	url = https://github.com/meganz/mingw-std-threads.git
+[submodule "src/3rdparty/libcds"]
+	path = src/3rdparty/libcds
+	url = https://github.com/khizmax/libcds.git
 [submodule "doc/wiki"]
 	path = doc/wiki
 	url = https://github.com/lmms/lmms.wiki.git

cmake/modules/CheckSubmodules.cmake

@@ -42,8 +42,8 @@ SET(ENV{LC_ALL} "C")
 SET(ENV{LANG} "en_US")
 
 # Assume alpha-numeric paths
-STRING(REGEX MATCHALL "path = [-0-9A-Za-z/]+" SUBMODULE_LIST ${SUBMODULE_DATA})
-STRING(REGEX MATCHALL "url = [.:%-0-9A-Za-z/]+" SUBMODULE_URL_LIST ${SUBMODULE_DATA})
+STRING(REGEX MATCHALL "path = [-0-9A-Za-z/_]+" SUBMODULE_LIST ${SUBMODULE_DATA})
+STRING(REGEX MATCHALL "url = [.:%-0-9A-Za-z/_]+" SUBMODULE_URL_LIST ${SUBMODULE_DATA})
 
 FOREACH(_part ${SUBMODULE_LIST})
 	STRING(REPLACE "path = " "" SUBMODULE_PATH ${_part})

include/LocklessAllocator.h

@@ -1,82 +0,0 @@
-/*
- * LocklessAllocator.h - allocator with lockless alloc and free
- *
- * Copyright (c) 2016 Javier Serrano Polo <javier@jasp.net>
- *
- * This file is part of LMMS - https://lmms.io
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public
- * License along with this program (see COPYING); if not, write to the
- * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301 USA.
- *
- */
-
-#ifndef LOCKLESS_ALLOCATOR_H
-#define LOCKLESS_ALLOCATOR_H
-
-#include <atomic>
-#include <stddef.h>
-
-class LocklessAllocator
-{
-public:
-	LocklessAllocator( size_t nmemb, size_t size );
-	virtual ~LocklessAllocator();
-	void * alloc();
-	void free( void * ptr );
-
-
-private:
-	char * m_pool;
-	size_t m_capacity;
-	size_t m_elementSize;
-
-	std::atomic_int * m_freeState;
-	size_t m_freeStateSets;
-
-	std::atomic_int m_available;
-	std::atomic_int m_startIndex;
-
-} ;
-
-
-
-
-template<typename T>
-class LocklessAllocatorT : private LocklessAllocator
-{
-public:
-	LocklessAllocatorT( size_t nmemb ) :
-		LocklessAllocator( nmemb, sizeof( T ) )
-	{
-	}
-
-	virtual ~LocklessAllocatorT()
-	{
-	}
-
-	T * alloc()
-	{
-		return (T *)LocklessAllocator::alloc();
-	}
-
-	void free( T * ptr )
-	{
-		LocklessAllocator::free( ptr );
-	}
-
-} ;
-
-
-#endif

include/LocklessList.h

@@ -25,7 +25,7 @@
 #ifndef LOCKLESS_LIST_H
 #define LOCKLESS_LIST_H
 
-#include "LocklessAllocator.h"
+#include "MemoryPool.h"
 
 #include <atomic>
 
@@ -41,7 +41,7 @@ public:
 
 	LocklessList( size_t size ) :
 		m_first(nullptr),
-		m_allocator(new LocklessAllocatorT<Element>(size))
+		m_allocator(new MemoryPool<Element>(size))
 	{
 	}
 
@@ -52,7 +52,7 @@ public:
 
 	void push( T value )
 	{
-		Element * e = m_allocator->alloc();
+		Element * e = m_allocator->allocate();
 		e->value = value;
 		e->next = m_first.load(std::memory_order_relaxed);
 
@@ -81,14 +81,13 @@ public:
 
 	void free( Element * e )
 	{
-		m_allocator->free( e );
+		m_allocator->destroy( e );
 	}
 
 
 private:
 	std::atomic<Element*> m_first;
-	LocklessAllocatorT<Element> * m_allocator;
-
+	MemoryPool<Element> * m_allocator;
 } ;
 
 

include/MemoryPool.h

@@ -0,0 +1,82 @@
+/*
+ * MemoryPool.h
+ *
+ * Copyright (c) 2018 Lukas W <lukaswhl/at/gmail.com>
+ *
+ * This file is part of LMMS - https://lmms.io
+ *
+ * This file is licensed under the MIT license. See LICENSE.MIT.txt file in the
+ * project root for details.
+ *
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+
+class _MemoryPool_Private;
+
+class _MemoryPool_Base
+{
+public:
+	_MemoryPool_Base(size_t size, size_t nmemb);
+	virtual ~_MemoryPool_Base();
+	void * allocate();
+	void * allocate_bounded();
+	void deallocate(void * ptr);
+
+private:
+	const std::unique_ptr<_MemoryPool_Private> _imp;
+};
+
+/// Thread-safe, lockless memory pool. Only supports allocate(n) with n=1. When
+/// the pool is exhausted, MmAllocator is used.
+template<typename T>
+class MemoryPool : private _MemoryPool_Base
+{
+public:
+	using value_type = T;
+	template<class U> struct rebind { typedef MemoryPool<U> other; };
+
+	MemoryPool(size_t nmemb) :	_MemoryPool_Base(sizeof(T), nmemb) {}
+
+	T * allocate(size_t n = 1)
+	{
+		if (n != 1) { throw std::bad_alloc{}; }
+		return reinterpret_cast<T*>(_MemoryPool_Base::allocate());
+	}
+
+	T * allocate_bounded()
+	{
+		return reinterpret_cast<T*>(_MemoryPool_Base::allocate_bounded());
+	}
+
+	void deallocate(T * ptr, size_t n = 1)
+	{
+		_MemoryPool_Base::deallocate(ptr);
+	}
+
+	template<class... Args>
+	T * construct(Args&&... args)
+	{
+		T* buffer = allocate();
+		return ::new ((void*)buffer) T(std::forward<Args>(args)...);
+	}
+
+	template<class... Args >
+	T * construct_bounded(Args&&... args)
+	{
+		T* buffer = allocate_bounded();
+		if (buffer) {
+			::new ((void*)buffer) T(std::forward<Args>(args)...);
+		}
+		return buffer;
+	}
+
+	void destroy(T* ptr)
+	{
+		ptr->~T();
+		deallocate(ptr);
+	}
+} ;

include/libcds.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "NiftyCounter.h"
+
+namespace _cdslib
+{
+	void init();
+	void deinit();
+	void thread_init();
+	void thread_deinit();
+
+	static NiftyCounter<init, deinit> _counter;
+	static NiftyCounterTL<thread_init, thread_init> _thread_counter;
+}
+

src/3rdparty/CMakeLists.txt

@@ -10,3 +10,5 @@ ENDIF()
 
 ADD_SUBDIRECTORY(rpmalloc)
 ADD_SUBDIRECTORY(weakjack)
+
+ADD_SUBDIRECTORY(libcds)

src/CMakeLists.txt

@@ -168,6 +168,7 @@ SET(LMMS_REQUIRED_LIBS ${LMMS_REQUIRED_LIBS}
 	${SNDFILE_LIBRARIES}
 	${EXTRA_LIBRARIES}
 	rpmalloc
+	cds-s
 )
 
 # Expose required libs for tests binary

src/core/CMakeLists.txt

@@ -30,8 +30,8 @@ set(LMMS_SRCS
 	core/LadspaControl.cpp
 	core/LadspaManager.cpp
 	core/LfoController.cpp
-	core/LocklessAllocator.cpp
 	core/Memory.cpp
+	core/MemoryPool.cpp
 	core/MeterModel.cpp
 	core/MicroTimer.cpp
 	core/Mixer.cpp
@@ -99,5 +99,7 @@ set(LMMS_SRCS
 	core/midi/MidiTime.cpp
 	core/midi/MidiWinMM.cpp
 
+	core/libcds.cpp
+
 	PARENT_SCOPE
 )

src/core/LocklessAllocator.cpp

@@ -1,164 +0,0 @@
-/*
- * LocklessAllocator.cpp - allocator with lockless alloc and free
- *
- * Copyright (c) 2016 Javier Serrano Polo <javier@jasp.net>
- *
- * This file is part of LMMS - https://lmms.io
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public
- * License along with this program (see COPYING); if not, write to the
- * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301 USA.
- *
- */
-
-#include "LocklessAllocator.h"
-
-#include <algorithm>
-#include <stdio.h>
-
-#include "lmmsconfig.h"
-
-#ifndef LMMS_BUILD_WIN32
-#include <strings.h>
-#endif
-
-static const size_t SIZEOF_SET = sizeof( int ) * 8;
-
-
-static size_t align( size_t size, size_t alignment )
-{
-	size_t misalignment = size % alignment;
-	if( misalignment )
-	{
-		size += alignment - misalignment;
-	}
-	return size;
-}
-
-
-
-
-LocklessAllocator::LocklessAllocator( size_t nmemb, size_t size )
-{
-	m_capacity = align( nmemb, SIZEOF_SET );
-	m_elementSize = align( size, sizeof( void * ) );
-	m_pool = new char[m_capacity * m_elementSize];
-
-	m_freeStateSets = m_capacity / SIZEOF_SET;
-	m_freeState = new std::atomic_int[m_freeStateSets];
-	std::fill(m_freeState, m_freeState + m_freeStateSets, 0);
-
-	m_available = m_capacity;
-	m_startIndex = 0;
-}
-
-
-
-
-LocklessAllocator::~LocklessAllocator()
-{
-	int available = m_available;
-	if( available != m_capacity )
-	{
-		fprintf( stderr, "LocklessAllocator: "
-				"Destroying with elements still allocated\n" );
-	}
-
-	delete[] m_pool;
-	delete[] m_freeState;
-}
-
-
-
-
-#ifdef LMMS_BUILD_WIN32
-static int ffs( int i )
-{
-	if( !i )
-	{
-		return 0;
-	}
-	for( int j = 0;; )
-	{
-		if( i & 1 << j++ )
-		{
-			return j;
-		}
-	}
-}
-#endif
-
-
-
-
-void * LocklessAllocator::alloc()
-{
-	// Some of these CAS loops could probably use relaxed atomics, as discussed
-	// in http://en.cppreference.com/w/cpp/atomic/atomic/compare_exchange.
-	// Let's use sequentially-consistent ops to be safe for now.
-	int available = m_available.load();
-	do
-	{
-		if( !available )
-		{
-			fprintf( stderr, "LocklessAllocator: No free space\n" );
-			return NULL;
-		}
-	}
-	while (!m_available.compare_exchange_weak(available, available - 1));
-
-	const size_t startIndex = m_startIndex++ % m_freeStateSets;
-	for (size_t set = startIndex;; set = ( set + 1 ) % m_freeStateSets)
-	{
-		for (int freeState = m_freeState[set]; freeState != -1;)
-		{
-			int bit = ffs( ~freeState ) - 1;
-			if (m_freeState[set].compare_exchange_weak(freeState,
-							freeState | 1 << bit ) )
-			{
-				return m_pool + ( SIZEOF_SET * set + bit )
-								* m_elementSize;
-			}
-		}
-	}
-}
-
-
-
-
-void LocklessAllocator::free( void * ptr )
-{
-	ptrdiff_t diff = (char *)ptr - m_pool;
-	if( diff < 0 || diff % m_elementSize )
-	{
-invalid:
-		fprintf( stderr, "LocklessAllocator: Invalid pointer\n" );
-		return;
-	}
-	size_t offset = diff / m_elementSize;
-	if( offset >= m_capacity )
-	{
-		goto invalid;
-	}
-	size_t set = offset / SIZEOF_SET;
-	int bit = offset % SIZEOF_SET;
-	int mask = 1 << bit;
-	int prevState = m_freeState[set].fetch_and(~mask);
-	if ( !( prevState & mask ) )
-	{
-		fprintf( stderr, "LocklessAllocator: Block not in use\n" );
-		return;
-	}
-	++m_available;
-}

src/core/MemoryPool.cpp

@@ -0,0 +1,121 @@
+/*
+ * MemoryPool.cpp
+ *
+ * Copyright (c) 2018 Lukas W <lukaswhl/at/gmail.com>
+ *
+ * This file is part of LMMS - https://lmms.io
+ *
+ * This file is licensed under the MIT license. See LICENSE.MIT.txt file in the
+ * project root for details.
+ *
+ */
+
+#include "MemoryPool.h"
+
+#include <QtCore/QDebug>
+
+#include <cds/container/vyukov_mpmc_cycle_queue.h>
+#include "libcds.h"
+
+#include "Memory.h"
+
+class _MemoryPool_Private : MmAllocator<char>
+{
+	using Alloc = MmAllocator<char>;
+public:
+	_MemoryPool_Private(size_t size, size_t nmemb)
+		: m_freelist(nmemb)
+		, m_elementSize(size)
+		, m_numElms(nmemb)
+	{
+		m_buffer = new char[m_elementSize * m_numElms];
+		for (size_t i = 0; i < m_numElms; i++) {
+			m_freelist.push(m_buffer + (i * m_elementSize));
+		}
+	}
+
+	~_MemoryPool_Private()
+	{
+		char* ptr = nullptr;
+		while (m_freelist.pop(ptr)) {
+			if (! is_from_pool(ptr)) {
+				Alloc::deallocate(ptr, m_elementSize);
+			}
+		}
+		delete[] m_buffer;
+	}
+
+	void * allocate()
+	{
+		void* ptr = allocate_bounded();
+		if (ptr) {
+			return ptr;
+		} else {
+			qWarning() << "MemoryPool exhausted";
+			return Alloc::allocate(m_elementSize);
+		}
+	}
+
+	void * allocate_bounded()
+	{
+		char* ptr = nullptr;
+		m_freelist.pop(ptr);
+		return ptr;
+	}
+
+	void deallocate(void * ptr)
+	{
+		if (is_from_pool(ptr)) {
+			bool pushed = m_freelist.push(reinterpret_cast<char*>(ptr));
+			assert(pushed);
+		} else {
+			do_deallocate(ptr);
+		}
+	}
+
+private:
+	void* do_allocate()
+	{
+		return Alloc::allocate(m_elementSize);
+	}
+	void do_deallocate(void* ptr)
+	{
+		Alloc::deallocate(reinterpret_cast<char*>(ptr), m_elementSize);
+	}
+
+	bool is_from_pool(void* ptr)
+	{
+		auto buff = reinterpret_cast<uintptr_t>(m_buffer);
+		auto p = reinterpret_cast<uintptr_t>(ptr);
+		return p >= buff && p < (buff + (m_elementSize * m_numElms));
+	}
+
+	const size_t m_elementSize;
+	const size_t m_numElms;
+
+	char* m_buffer;
+	cds::container::VyukovMPMCCycleQueue<char*> m_freelist;
+};
+
+_MemoryPool_Base::_MemoryPool_Base( size_t size, size_t nmemb )
+	: _imp(new _MemoryPool_Private(size, nmemb))
+{}
+
+_MemoryPool_Base::~_MemoryPool_Base()
+{}
+
+void * _MemoryPool_Base::allocate()
+{
+	return _imp->allocate();
+}
+
+void *_MemoryPool_Base::allocate_bounded()
+{
+	return _imp->allocate_bounded();
+}
+
+void _MemoryPool_Base::deallocate(void * ptr)
+{
+	return _imp->deallocate(ptr);
+}
+

src/core/libcds.cpp

@@ -0,0 +1,36 @@
+#include "libcds.h"
+
+#include <cds/init.h>
+#include <cds/gc/hp.h>
+
+#include <memory>
+#include "stdshims.h"
+
+namespace _cdslib
+{
+
+static std::unique_ptr<cds::gc::HP> hpGC;
+
+void init()
+{
+	cds::Initialize();
+	hpGC = make_unique<cds::gc::HP>();
+}
+
+void deinit()
+{
+	hpGC.reset();
+	cds::Terminate();
+}
+
+void thread_init()
+{
+	cds::threading::Manager::attachThread();
+}
+
+void thread_deinit()
+{
+	cds::threading::Manager::detachThread();
+}
+
+}

tests/CMakeLists.txt

@@ -17,6 +17,7 @@ ADD_EXECUTABLE(tests
 	src/core/AutomatableModelTest.cpp
 	src/core/ProjectVersionTest.cpp
 	src/core/RelativePathsTest.cpp
+	src/core/MemoryPoolTest.cpp
 
 	src/tracks/AutomationTrackTest.cpp
 )

tests/src/core/MemoryPoolTest.cpp

@@ -0,0 +1,58 @@
+/*
+ * MemoryPoolTest.cpp
+ *
+ * Copyright (c) 2018 Lukas W <lukaswhl/at/gmail.com>
+ *
+ * This file is part of LMMS - https://lmms.io
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program (see COPYING); if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301 USA.
+ *
+ */
+
+#include "QTestSuite.h"
+
+#include "MemoryPool.h"
+
+#include <array>
+#include <stack>
+
+class MemoryPoolTest : QTestSuite
+{
+	Q_OBJECT
+private slots:
+	void MemoryPoolTests()
+	{
+		using T = std::array<char, 16>;
+		MemoryPool<T> pool(256);
+
+		std::stack<T*> ptrs;
+
+		for (int i=0; i < 256; i++) {
+			ptrs.push(pool.allocate_bounded());
+			QVERIFY(ptrs.top());
+		}
+		QCOMPARE(pool.allocate_bounded(), nullptr);
+		ptrs.push(pool.allocate());
+		QVERIFY(ptrs.top());
+
+		while (!ptrs.empty()) {
+			pool.deallocate(ptrs.top());
+			ptrs.pop();
+		}
+	}
+} MemoryPoolTests;
+
+#include "MemoryPoolTest.moc"